This commit is contained in:
cwx committed 2023-05-16 07:19:02 +08:00
1 parent b59e241fa1
commit 3c3f8846cb
9 files changed
+430 -37

No files matched your search

+248
View File
@@ -0,0 +1,248 @@
import json,re
from opencc import OpenCC
from queue import Queue
from utils.OldUtils import OldUtils
class Comic:
# ComicInfo.xml and ComicChapter.json bengin
# value origin node dep required
dict_chapter = [None,None,"Title","章节名",True]
dict_comic_name = [None,None,"Series","漫画名",True]
dict_number = [None,None,"Number","编号",True]
dict_comic_names = [None,None,"SeriesGroup","别名",False]
dict_dep = [None,None,"Summary","概述",True]
dict_year = [None,None,"Year","年",False]
dict_month = [None,None,"Month","月",False]
dict_day = [None,None,"Day","日",False]
dict_author = [None,None,"Writer","作者",True]
dict_cbs = [None,None,"Publisher","出版社",False]
dict_genre = [None,None,"Genre","流派",True]
dict_tags = [None,None,"Tags","标签",True]
dict_homepage = [None,None,"Web","主页",False]
dict_page_count = [None,None,"PageCount","总页数",True]
dict_language = [None,None,"LanguageISO","语言",True]
dict_agerating = [None,None,"AgeRating","年龄分级",False]
dict_pages = [None,None,"Pages","页码",True]
CURRENT_DOWN_LINK = None
# ComicInfo.xml and ComicChapter.json end
dict_icon = [None,None,"Icon","图标",True]
dict_chapter_imgs = [None,None,"ChapterImgs","图像",True]
#主页
dict_list_chapter = [None,None,"ListChapter","全部章节名",True]
(update_at,current_chapter_img,file_chapter_imgs) = [None,None,None]
#繁体中文转简体中文
@classmethod
def ChineseConvert(cls, text,convert='t2s'): return OpenCC(convert).convert(str(text))
#处理成符合规定的文件名
@classmethod
def fixFileName(cls,filename,replace=None):
if not isinstance(filename,str): return filename
intab = r'[?*/\|.:><]'
str_replace = ""
if replace != None: str_replace = replace
filename = re.sub(intab, str_replace, filename)
count = 1
while True:
str_file = filename[0-count]
if str_file == " ": count += 1
else:
filename = filename[0:len(filename)+1-count]
break
return filename
@classmethod
def setValue(cls,value):
if value != None: value = cls.ChineseConvert(value)
return value
@classmethod
def setField(cls,field,value,origin=True,convert=True):
if value != None:
if origin:
field[1] = value
if convert: value = cls.ChineseConvert(value)
field[0] = value
return field
@classmethod
def getFieldValue(cls,field):
if field == None: return None
return field[0]
@classmethod
def setFieldOrigin(cls,filed,origin):
filed[1] = origin
return filed
@classmethod
def getFieldOrigin(cls,filed): return filed[1]
@classmethod
def getFieldNode(cls,filed): return filed[2]
@classmethod
def getValue(cls,field,exec=None):
if exec != None: return cls.parseExec(field,exec=exec)
return field
#章节名
@classmethod
def setChapterName(cls,value,exec=None):
value = cls.fixFileName(cls.parseExec(value,exec=exec))
OldUtils.setOldChapter(value)
cls.dict_chapter = cls.setField(cls.dict_chapter,value)
@classmethod
def getChapterName(cls): return cls.getFieldValue(cls.dict_chapter)
@classmethod
def getOriginChapterName(cls): return cls.getFieldOrigin(cls.dict_chapter)
#漫画名
@classmethod
def setComicName(cls,value,exec=None):
value = cls.fixFileName(cls.parseExec(value,exec=exec))
OldUtils.setOldComicName(value)
cls.dict_comic_name = cls.setField(cls.dict_comic_name,value)
@classmethod
def getComicName(cls): return cls.getFieldValue(cls.dict_comic_name)
@classmethod
def getOriginComicName(cls): return cls.getFieldOrigin(cls.dict_comic_name)
#编号
@classmethod
def setNumber(cls,value): cls.dict_number = cls.setField(cls.dict_number,value)
@classmethod
def getNumber(cls): return cls.getFieldValue(cls.dict_number)
#概述
@classmethod
def setDep(cls,value,exec=None):
cls.dict_dep = cls.setField(cls.dict_dep,cls.parseExec(value,exec=exec))
@classmethod
def getDep(cls): return cls.getFieldValue(cls.dict_dep)
#作者
@classmethod
def setAuthor(cls,value): cls.dict_author = cls.setField(cls.dict_author,value)
@classmethod
def getAuthor(cls): return cls.getFieldValue(cls.dict_author)
#流派
@classmethod
def setGenre(cls,value): cls.dict_genre = cls.setField(cls.dict_genre,value)
@classmethod
def getGenre(cls): return cls.getFieldValue(cls.dict_genre)
#语言
@classmethod
def setLanguage(cls,value): cls.dict_language = cls.setField(cls.dict_language,value)
@classmethod
def getLanguage(cls): return cls.getFieldValue(cls.dict_language)
#年龄分级
@classmethod
def setAgeRating(cls,value): cls.dict_agerating = cls.setField(cls.dict_agerating,value)
@classmethod
def getAgeRating(cls): return cls.getFieldValue(cls.dict_agerating)
#标签
@classmethod
def setTags(cls,value): cls.dict_tags = cls.setField(cls.dict_tags,value)
@classmethod
def getTags(cls): return cls.getFieldValue(cls.dict_tags)
#总页数
@classmethod
def setPageCount(cls,value): cls.dict_page_count = cls.setField(cls.dict_page_count,value)
@classmethod
def getPageCount(cls): return cls.getFieldValue(cls.dict_page_count)
#------------------------------------------------------------------------
@classmethod
def parseExec(cls,data,exec,item=True):
if data !=None and exec != None:
dots = str(exec).split(".")
if not isinstance(data,dict): data = json.loads(data)
for dot in dots:
data = data.get(dot)
return data
@classmethod
def setHomePage(cls,value): cls.dict_homepage = cls.setField(cls.dict_homepage,value)
@classmethod
def getHomePage(cls): return cls.getFieldValue(cls.dict_homepage)
@classmethod
def setIcon(cls,value): cls.dict_icon = cls.setField(cls.dict_icon,value,convert=False)
@classmethod
def getIcon(cls): return cls.getFieldValue(cls.dict_icon)
@classmethod
def setListChapter(cls,value): cls.dict_list_chapter = cls.setField(cls.dict_list_chapter,value,convert=False)
@classmethod
def getListChapter(cls): return cls.getFieldValue(cls.dict_list_chapter)
@classmethod
def getLenChapters(cls): return len(cls.getListChapter())
@classmethod
def setChapterImgs(cls,value,exec=None,item=None):
cls.dict_chapter_imgs = cls.setField(cls.dict_chapter_imgs,cls.parseExec(value,exec=exec,item=item),convert=False)
@classmethod
def getChapterImgs(cls): return cls.getFieldValue(cls.dict_chapter_imgs)
@classmethod
def setUpdateAt(cls,value): cls.update_at = value
@classmethod
def getUpdateAt(cls): return cls.update_at
@classmethod
def setCurrentChapterImg(cls,value): cls.current_chapter_img = value
@classmethod
def getCurrentChapterImg(cls): return cls.current_chapter_img
@classmethod
def setChapterFilesName(cls,value): cls.file_chapter_imgs= value
@classmethod
def getChapterFilesName(cls): return cls.file_chapter_imgs
@classmethod
def setCurrentDownLink(cls,value): cls.CURRENT_DOWN_LINK = value
@classmethod
def getCurrentDownLink(cls): return cls.CURRENT_DOWN_LINK
class ListComic:
LIST_COMIC_QUEUE = Queue()
(LIST_COMIC_NAME,LIST_COMIC_LINK,LIST_COMIC_UPDATEAT) = [None,None,None]
@classmethod
def setListComicsLinksUpdateAt(cls,names,links,update_at):
if isinstance(names,list) and isinstance(links,list) and isinstance(update_at,list):
for x in range(0,len(names)):
cls.LIST_COMIC_QUEUE.put([names[x],links[x],update_at[x]])
@classmethod
def getListComicsLinksUpdateAt(cls):
result = None
if cls.LIST_COMIC_NAME != None and cls.LIST_COMIC_LINK != None:
cls.setListComicsLinksUpdateAt(cls.LIST_COMIC_NAME,cls.LIST_COMIC_LINK,cls.LIST_COMIC_UPDATEAT)
(cls.LIST_COMIC_NAME,cls.LIST_COMIC_LINK,cls.LIST_COMIC_UPDATEAT) = [None,None,None]
if not cls.LIST_COMIC_QUEUE.empty(): result = cls.LIST_COMIC_QUEUE.get(False)
return result
@classmethod
def addListComicChapterLink(cls,name,link,update_at):
if name != None and link != None:
cls.LIST_COMIC_QUEUE.put(name,link,update_at)
@classmethod
def getListValue(cls,result,type,start_add=None,result_type="list"):
if result == None: return None
if type == None: return result
if result_type == "list" and type != None:
data = []
for x in range(0, len(result)):
if start_add != None:
data.append(start_add+result[x].get(type))
else:
data.append(result[x].get(type))
return data
return result
@classmethod
def setListComicName(cls,value,type=None): cls.LIST_COMIC_NAME = cls.getListValue(value,type)
@classmethod
def getListComicName(cls): return cls.LIST_COMIC_NAME
@classmethod
def setListComicChapterLink(cls,value,type=None,start_add=None): cls.LIST_COMIC_LINK = cls.getListValue(value,type,start_add)
@classmethod
def getListComicChapterLink(cls): return cls.LIST_COMIC_LINK
@classmethod
def setListComicUpdateAt(cls,value,type=None): cls.LIST_COMIC_UPDATEAT = cls.getListValue(value,type)
@classmethod
def getListComicUpdateAt(cls): return cls.LIST_COMIC_UPDATEAT
@classmethod
def getListComicChapterLink(cls): return cls.LIST_COMIC_QUEUE.get(False)
#domain end....
+156
View File
@@ -0,0 +1,156 @@
import json,os
import logging
from xml.dom.minidom import Document
from Comics.utils.Comic import Comic
from Comics.utils.Constant import ComicPath
class ComicInfoEntity:
@classmethod
def getNodes(cls):
return [Comic.dict_chapter,Comic.dict_comic_name,Comic.dict_number,Comic.dict_comic_names,
Comic.dict_dep,Comic.dict_year,Comic.dict_month,Comic.dict_day,Comic.dict_author,
Comic.dict_cbs,Comic.dict_genre,Comic.dict_tags,Comic.dict_page_count,
Comic.dict_language,Comic.dict_agerating,Comic.dict_pages]
@classmethod
def getJsonNodes(cls):
return [Comic.dict_chapter,Comic.dict_comic_name,Comic.dict_icon,Comic.dict_number,
Comic.dict_comic_names,
Comic.dict_dep,Comic.dict_year,Comic.dict_month,Comic.dict_day,Comic.dict_author,
Comic.dict_cbs,Comic.dict_genre,Comic.dict_tags,Comic.dict_page_count,
Comic.dict_language,Comic.dict_agerating,Comic.dict_pages,
Comic.dict_list_chapter,Comic.dict_chapter_imgs]
class ComicInfo:
IS_NEW_ICON = False
document = Document()
path_comic_info = None
@classmethod
def parseExec(cls,data,exec,start_add=None,item=True):
if data !=None and exec != None:
dots = str(exec).split(".")
if not isinstance(data,dict): data = json.loads(data)
for dot in dots:
data = data.get(dot)
if start_add != None and data != None:
data = start_add+data
return data
@classmethod
def setNodeAndValue(cls,node,value):
if value != None:
if isinstance(value,str):
c_node = cls.document.createElement(node)
child_node = cls.document.createTextNode(value)
c_node.appendChild(child_node)
return c_node
else: return value
return None
#页数
@classmethod
def setPages(cls,values=None):
if values == None: values = Comic.getChapterFilesName()
if values != None and isinstance(values,list):
suffix = "."+str(values[0]).split(".")[-1]
join_list=",".join(values).replace(suffix,"")
values = join_list.split(",")
Comic.setPageCount(len(values)+1 if cls.IS_NEW_ICON else len(values))
root_node = cls.document.createElement("Pages")
if cls.IS_NEW_ICON:
#添加封面
icon_node = cls.document.createElement("Page")
icon_node.setAttribute("Image",ComicPath.COMIC_ICON_NAME)
icon_node.setAttribute("Type","FrontCover")
root_node.appendChild(icon_node)
for page in values:
c_node = cls.document.createElement("Page")
page = page.split("_")[-1]
c_node.setAttribute("Image",page)
root_node.appendChild(c_node)
Comic.dict_pages = Comic.setField(Comic.dict_pages,root_node,convert=False)
@classmethod
def getBaseUrl(cls,url=None):
if url == None:
url = Comic.getHomePage()
(num,index) = [3,0]
for x in range(0, num):
index = str(url).find("/",index)+1
return url[0:index-1]
#XML根文档
@classmethod
def root_node(cls,root_value): return cls.document.createElement(root_value)
@classmethod
def add_nodes(cls,root,list_value):
if len(list_value) == 0: return list_value
for value in list_value:
#Comic.chapter
if value[0] == None and value[4]:
#数据为空 value[0], 但不允许为空value[4] = False
msg = f"#数据为空 key={value[3]} value[0]={value[0]}, 但不允许为空value[4]={value[4]}"
logger.error(msg)
exit()
if value[0] != None: root.appendChild(cls.setNodeAndValue(value[2],value[0]))
@classmethod
def initComicInfoXML(cls):
cls.setPages()
@classmethod
def writeComicInfoXML(cls,overlay=False):
save_path = ComicPath.getPathComicInfoXML()
if os.path.exists(save_path):
if overlay:
os.remove(save_path)
logging.info(f"已存在文件,已删除... {save_path}")
else:
logging.info(f"已存在文件,跳过中... {save_path}")
return True
cls.initComicInfoXML()
root = cls.root_node("ComicInfo")
new_document = Document()
new_document.appendChild(root)
cls.add_nodes(root,ComicInfoEntity.getNodes())
with open(save_path, "w", encoding="utf-8") as fo:
new_document.writexml(fo, indent='', addindent='\t', newl='\n', encoding="utf-8")
fo.close()
logging.info(f"已生成文件... {save_path}")
@classmethod
def setComicInfo(cls,comicname=None,homepage=None,alias=None,author=None,icon=None,tags=None,
dep=None,genre=None,lang=None,age_rating=None,chapters=None,current_chapter_img=None):
author = ",".join(set(str(str(author).replace("&",",").replace(" ",",")).split(",")))
Comic.setHomePage(homepage)
Comic.setIcon(icon)
Comic.setListChapter(chapters)
#Comic.setUpdateAt(update_at)
Comic.setComicName(str(comicname))
#if alias != None: comicInfo.setComicNames(alias)
Comic.setAuthor(author)
Comic.setTags(tags)
Comic.setDep(dep)
#comicInfo.setCBS("韩漫")
if genre != None: Comic.setGenre(genre)
Comic.setLanguage(lang)
Comic.setAgeRating(age_rating)
Comic.setCurrentChapterImg(current_chapter_img)
@classmethod
def writeJson(cls):
dict_data = {}
nodes = ComicInfoEntity.getJsonNodes()
for node in nodes:
key = Comic.getFieldNode(node)
value = Comic.getFieldOrigin(node)
if isinstance(value,list):
value = ",".join(value)
if key != None and isinstance(value,str):
child_dict = { key : value}
dict_data.update(child_dict)
s = json.dumps(dict_data,ensure_ascii=True)
logging.debug(f"json={s}")
with open(ComicPath.getPathConfComicChapterJson(mkdir=True),"w") as fs:
fs.write(s)
+16
View File
@@ -0,0 +1,16 @@
from opencc import OpenCC
class ComicPath:
@classmethod
def getDirComicChapter(cls):
return None
@classmethod
def getFileScrambleImageName(cls,count,block,suffix=".jpg"): return "scramble="+str(block)+"_"+str(count)+suffix
@classmethod
def getFileScrambleImageSave(cls,file): return str(file).split("_")[-1]
#繁体中文转简体中文
@classmethod
def ChineseConvert(cls, text,convert='t2s'): return OpenCC(convert).convert(str(text))
+226
View File
@@ -0,0 +1,226 @@
import base64,hashlib,os,shutil
import math,time,json,datetime,logging
from PIL import Image
from tinydb import TinyDB, Query
from Comics.spiders.utils.Constant import ComicPath
class CommonUtils:
@classmethod
def parseExec(cls,data,exec):
if data !=None and exec != None:
dots = str(exec).split(".")
if not isinstance(data,dict): data = json.loads(data)
for dot in dots:
data = data.get(dot)
return data
class imageUtils:
@classmethod
def deScrambleImagesByDir(cls,chapter_dir):
scramble_count = 0
if os.path.exists(chapter_dir): #获取章节图片路径
dirs = os.listdir(chapter_dir)
for img in dirs:
if img.startswith("scramble="):
imageUtils.encode_scramble_image(os.path.join(chapter_dir,img))
scramble_count += 1
logging.debug(f"scramble= {scramble_count}")
return scramble_count
@classmethod
def deScrambleImagesByPath(cls,img_path,img_save=None):
if os.path.basename(img_path).startswith("scramble="):
imageUtils.encode_scramble_image(img_path,img_save)
return True
else:
return False
@classmethod
def encodeImage(cls,str_en):
#print("en",str_en)
enc = base64.b64decode(str_en)
#print("解密:",enc)
m = hashlib.md5()
m.update(enc)
md5 = m.digest()
d = md5[-1]
#print(md5)
try:
blocks = d % 10 + 5
except:
blocks = 0 %10 + 5
#print("blocks=",blocks)
return blocks
@classmethod
def scrambleImage(cls,file_path):
#检测到未下载完的图像 直接返回None
if str(file_path).endswith(".downloads"):
os.remove(file_path)
return None
file_str = str(file_path).split("=")
#10_29.jpg
base_dir = file_str[0].replace("scramble","")
base_name = file_str[-1]
base_fn = base_name.split("_")
save_name = base_fn[1]
save_name_delesu = save_name.split(".")[0]
blocks = int(base_fn[0])
save_file_path = os.path.join(base_dir,save_name)
print("sva",save_file_path)
if os.path.exists(save_file_path):
print("图片已解密,已跳过:", save_file_path)
return None
image_su = str(file_path).split(".")[-1]
try:
img = Image.open(file_path)
except:
print(f"error Image: {file_path}")
width = img.width
height = img.height
#blocks = cls.encodeImage(enStr)
print("blocks=",blocks)
block_height = int(height / blocks)
block_width = int(width / blocks)
print("blockHeight=",block_height)
suffix = str(file_path).split(".")[-1]
split_path = os.path.join(base_dir,save_name_delesu+"split")
if image_su == "downloads":
return None
is_split = cls.splitimage(file_path,blocks,1,split_path)
if is_split != None:
cls.image_compose(split_path,blocks,1,save_file_path,block_height,width)
else:
if os.path.exists(split_path):
shutil.rmtree(split_path)
if os.path.exists(file_path):
shutil.move(file_path, save_file_path)
#完成后清空
return file_path
@classmethod
def splitimage(cls,src,rownum,colnum,dstpath):
img=Image.open(src)
w,h=img.size
if rownum<= h and colnum<=w:
s=os.path.split(src)
if dstpath=='':
dstpath = s[0]
if not os.path.exists(dstpath):
os.makedirs(dstpath)
fn=s[1].split('.')
basename=fn[0]
ext=fn[-1]
num=0
rowheight=h//rownum
colwidth=w//colnum
for r in range(rownum):
for c in range(colnum):
box=(c*colwidth,r*rowheight,(c+1)*colwidth,(r+1)*rowheight)
count_image = "{:0>3d}".format(num)
file_path = os.path.join(dstpath,str(count_image)+'.'+ext)
print("file_path=",file_path)
img.crop(box).save(file_path)
num=num+1
return "成功"
else:
print('不数!')
return None
@classmethod
def image_compose(cls,src,row,column,save_path,image_height,image_width):
image_size = image_height
#image_height = 376
#image_width = 720
images_format = ['.png','.jpg']
#image_names = [name for name in os.listdir(src) for item in images_format if
# os.path.splitext(name)[1] == item][::-1]
img_list=os.listdir(src)
img_list.sort()
img_list.sort(key=lambda x: int(x[:-4]))
##文件名按数字排序
img_nums=len(img_list)
image_names = []
for i in range(img_nums):
img_name=os.path.join(src,img_list[i])
image_names.append(img_name)
#使用倒序
image_names = image_names[::-1]
# 简单的对于参数的设定和实际图片集的大小进行数量判断
if len(image_names) < row * column:
raise ValueError("合成图片的参数和要求的数量不能匹配!")
to_image = Image.new('RGB', (column * image_width, row * image_height)) #创建一个新图
# 循环遍历,把每张图片按顺序粘贴到对应位置上
for y in range(1, row + 1):
for x in range(1, column + 1):
#1 * (row=1 -1) col=1 -1
image_path = image_names[column * (y - 1) + x - 1]
print("split_image=",image_path)
from_image = Image.open(image_path)
#保持原图片大小
#.resize(
# (image_size, image_size),Image.ANTIALIAS)
to_image.paste(from_image, ((x - 1) * image_size, (y - 1) * image_size))
from_image.close()
to_image.save(save_path)
print("图片合并完成:", save_path)
shutil.rmtree(src)
# 保存新图
@classmethod
def getScrambleImage(cls,path):
scramble_file_cache = cls.scrambleImage(path)
if scramble_file_cache != None and os.path.exists(scramble_file_cache): os.remove(scramble_file_cache)
@classmethod
def encode_scramble_image(cls,imgpath,img_save=None):
image = Image.open(imgpath)
w, h = image.size
#image.show()
file_str = str(imgpath).split("=")
#10_29.jpg
base_fn = file_str[-1].split("_")
blocks = int(base_fn[0])
if img_save == None:
save_path = os.path.join(os.path.dirname(imgpath),ComicPath.getFileScrambleImageSave(imgpath))
else: save_path = img_save
# print(type(aid),type(img_name))
if blocks:
s = blocks # 随机值
# print(s)
l = h % s # 切割最后多余的值
box_list = []
hz = 0
for i in range(s):
c = math.floor(h / s)
g = i * c
hz += c
h2 = h - c * (i + 1) - l
if i == 0:
c += l;hz += l
else:
g += l
box_list.append((0, h2, w, h - g))
# print(box_list,len(box_list))
item_width = w
# box_list.reverse() #还原切图可以倒序列表
# print(box_list, len(box_list))
newh = 0
image_list = [image.crop(box) for box in box_list]
# print(box_list)
newimage = Image.new("RGB", (w, h))
for image in image_list:
# image.show()
b_w, b_h = image.size
newimage.paste(image, (0, newh))
newh += b_h
newimage.save(save_path)
print("解密成功=",save_path)
if os.path.exists(imgpath):
os.remove(imgpath)
print("remove=",imgpath)
+16
View File
@@ -0,0 +1,16 @@
class OldUtils:
old_comic_name=None
old_chapter = None
@classmethod
def setOldComicName(cls,value): cls.old_comic_name = value
@classmethod
def setOldChapter(cls,value): cls.old_chapter=value
@classmethod
def getOldComicName(cls): return cls.old_comic_name
@classmethod
def getOldChapter(cls): return cls.old_chapter