This commit is contained in:
cwx committed 2023-05-21 03:10:51 +08:00
1 parent 5884f1e92c
commit 8f55a51140
16 files changed
+489 -492

No files matched your search

+105
View File
@@ -0,0 +1,105 @@
import os, shutil, time, logging
from datetime import datetime
from pathlib import Path
from zipfile import ZipFile
from Comics.settings import COMIC_INFO_XML_FILE,CBZ_EXPORT_PATH,IMAGES_STORE
class CBZUtils:
@classmethod
def readDirsOrFiles(cls,dir,type):
data = []
files = os.listdir(dir)
for file in files:
path = os.path.join(dir,file)
if type == "files" and os.path.isfile(path):
data.append(path)
if type == "dirs" and os.path.isdir(path):
data.append(path)
return data
@classmethod
def zip_compression(cls, source_dir=None, target_file=None, remove=True):
target_dir = os.path.dirname(target_file)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
if not os.path.exists(target_file) and source_dir != None:
with ZipFile(target_file, mode='w') as zf:
for path, dir_names, filenames in os.walk(source_dir):
path = Path(path)
arc_dir = path.relative_to(source_dir)
y = 0
for filename in filenames:
y = y + 1
print("打包中:" + str(y) + "/" + str(len(filenames)), os.path.join(source_dir, filename))
zf.write(path.joinpath(filename), arc_dir.joinpath(filename))
zf.close()
logging.info(f"打包完成:{target_file}")
@classmethod
def packComicChapterCBZ(cls, comic, chapter, remove=True):
images_chapter_path = os.path.join(IMAGES_STORE, comic, chapter)
cbz_chapter_path = os.path.join(CBZ_EXPORT_PATH, comic, chapter)+".CBZ"
if os.path.exists(images_chapter_path):
dirs = os.listdir(images_chapter_path)
for file in dirs:
if file.startswith("scramble="):
try:
os.remove(file)
except:
print(f"删除 {file} 发生错误,已跳过")
return False
cls.zip_compression(images_chapter_path, cbz_chapter_path)
time.sleep(0.1)
if remove: shutil.rmtree(images_chapter_path)
return True
@classmethod
def replaceZip(cls,filepath,unpack_dir=None):
if not cls.compareFileDate(filepath): return None
if unpack_dir == None:
unpack_dir = str(filepath).split(".")[0]
fz = ZipFile(filepath, 'r')
for file in fz.namelist():
if file.endswith(".jpg"):
data = fz.read(file)
if len(data) < 500 and os.path.exists(filepath):
os.remove(filepath)
print(f"数据不完整,已删除:{filepath}")
if cls.compareFileDate(filepath):
os.utime(filepath)
print(f"已更新文件时间 {filepath}")
if os.path.exists(unpack_dir):
shutil.rmtree(unpack_dir)
# 删除删除main.ftl文件
#delete_filename = ''
#if os.path.exists(delete_filename):
# os.remove(delete_filename)
# time.sleep(60)
# shutil.copy(文件的路径,另一个目录);拷贝main.ftl到准备压缩的目录下
#cls.zip_compression()
#小于则运行
@classmethod
def compareFileDate(cls,filepath):
if os.path.exists(filepath):
ctime = os.path.getmtime(filepath)
str_ctime = datetime.fromtimestamp(int(ctime))
file_ctime = str(str_ctime.year)+"{:0>2d}".format(str_ctime.month)+"{:0>2d}".format(str_ctime.day)+"{:0>2d}".format(str_ctime.hour)
c_ctime = 2023011603
else:
return False
if int(file_ctime) < c_ctime:
return True
return False
@classmethod
def zip_info(cls, path, filter=True):
result = None
try:
with ZipFile(path, "r") as zip_file:
result = zip_file.namelist()
if filter:
result.remove(COMIC_INFO_XML_FILE)
except Exception as e:
print(e)
return result
-248
View File
@@ -1,248 +0,0 @@
import json,re
from opencc import OpenCC
from queue import Queue
from utils.OldUtils import OldUtils
class Comic:
# ComicInfo.xml and ComicChapter.json bengin
# value origin node dep required
dict_chapter = [None,None,"Title","章节名",True]
dict_comic_name = [None,None,"Series","漫画名",True]
dict_number = [None,None,"Number","编号",True]
dict_comic_names = [None,None,"SeriesGroup","别名",False]
dict_dep = [None,None,"Summary","概述",True]
dict_year = [None,None,"Year","年",False]
dict_month = [None,None,"Month","月",False]
dict_day = [None,None,"Day","日",False]
dict_author = [None,None,"Writer","作者",True]
dict_cbs = [None,None,"Publisher","出版社",False]
dict_genre = [None,None,"Genre","流派",True]
dict_tags = [None,None,"Tags","标签",True]
dict_homepage = [None,None,"Web","主页",False]
dict_page_count = [None,None,"PageCount","总页数",True]
dict_language = [None,None,"LanguageISO","语言",True]
dict_agerating = [None,None,"AgeRating","年龄分级",False]
dict_pages = [None,None,"Pages","页码",True]
CURRENT_DOWN_LINK = None
# ComicInfo.xml and ComicChapter.json end
dict_icon = [None,None,"Icon","图标",True]
dict_chapter_imgs = [None,None,"ChapterImgs","图像",True]
#主页
dict_list_chapter = [None,None,"ListChapter","全部章节名",True]
(update_at,current_chapter_img,file_chapter_imgs) = [None,None,None]
#繁体中文转简体中文
@classmethod
def ChineseConvert(cls, text,convert='t2s'): return OpenCC(convert).convert(str(text))
#处理成符合规定的文件名
@classmethod
def fixFileName(cls,filename,replace=None):
if not isinstance(filename,str): return filename
intab = r'[?*/\|.:><]'
str_replace = ""
if replace != None: str_replace = replace
filename = re.sub(intab, str_replace, filename)
count = 1
while True:
str_file = filename[0-count]
if str_file == " ": count += 1
else:
filename = filename[0:len(filename)+1-count]
break
return filename
@classmethod
def setValue(cls,value):
if value != None: value = cls.ChineseConvert(value)
return value
@classmethod
def setField(cls,field,value,origin=True,convert=True):
if value != None:
if origin:
field[1] = value
if convert: value = cls.ChineseConvert(value)
field[0] = value
return field
@classmethod
def getFieldValue(cls,field):
if field == None: return None
return field[0]
@classmethod
def setFieldOrigin(cls,filed,origin):
filed[1] = origin
return filed
@classmethod
def getFieldOrigin(cls,filed): return filed[1]
@classmethod
def getFieldNode(cls,filed): return filed[2]
@classmethod
def getValue(cls,field,exec=None):
if exec != None: return cls.parseExec(field,exec=exec)
return field
#章节名
@classmethod
def setChapterName(cls,value,exec=None):
value = cls.fixFileName(cls.parseExec(value,exec=exec))
OldUtils.setOldChapter(value)
cls.dict_chapter = cls.setField(cls.dict_chapter,value)
@classmethod
def getChapterName(cls): return cls.getFieldValue(cls.dict_chapter)
@classmethod
def getOriginChapterName(cls): return cls.getFieldOrigin(cls.dict_chapter)
#漫画名
@classmethod
def setComicName(cls,value,exec=None):
value = cls.fixFileName(cls.parseExec(value,exec=exec))
OldUtils.setOldComicName(value)
cls.dict_comic_name = cls.setField(cls.dict_comic_name,value)
@classmethod
def getComicName(cls): return cls.getFieldValue(cls.dict_comic_name)
@classmethod
def getOriginComicName(cls): return cls.getFieldOrigin(cls.dict_comic_name)
#编号
@classmethod
def setNumber(cls,value): cls.dict_number = cls.setField(cls.dict_number,value)
@classmethod
def getNumber(cls): return cls.getFieldValue(cls.dict_number)
#概述
@classmethod
def setDep(cls,value,exec=None):
cls.dict_dep = cls.setField(cls.dict_dep,cls.parseExec(value,exec=exec))
@classmethod
def getDep(cls): return cls.getFieldValue(cls.dict_dep)
#作者
@classmethod
def setAuthor(cls,value): cls.dict_author = cls.setField(cls.dict_author,value)
@classmethod
def getAuthor(cls): return cls.getFieldValue(cls.dict_author)
#流派
@classmethod
def setGenre(cls,value): cls.dict_genre = cls.setField(cls.dict_genre,value)
@classmethod
def getGenre(cls): return cls.getFieldValue(cls.dict_genre)
#语言
@classmethod
def setLanguage(cls,value): cls.dict_language = cls.setField(cls.dict_language,value)
@classmethod
def getLanguage(cls): return cls.getFieldValue(cls.dict_language)
#年龄分级
@classmethod
def setAgeRating(cls,value): cls.dict_agerating = cls.setField(cls.dict_agerating,value)
@classmethod
def getAgeRating(cls): return cls.getFieldValue(cls.dict_agerating)
#标签
@classmethod
def setTags(cls,value): cls.dict_tags = cls.setField(cls.dict_tags,value)
@classmethod
def getTags(cls): return cls.getFieldValue(cls.dict_tags)
#总页数
@classmethod
def setPageCount(cls,value): cls.dict_page_count = cls.setField(cls.dict_page_count,value)
@classmethod
def getPageCount(cls): return cls.getFieldValue(cls.dict_page_count)
#------------------------------------------------------------------------
@classmethod
def parseExec(cls,data,exec,item=True):
if data !=None and exec != None:
dots = str(exec).split(".")
if not isinstance(data,dict): data = json.loads(data)
for dot in dots:
data = data.get(dot)
return data
@classmethod
def setHomePage(cls,value): cls.dict_homepage = cls.setField(cls.dict_homepage,value)
@classmethod
def getHomePage(cls): return cls.getFieldValue(cls.dict_homepage)
@classmethod
def setIcon(cls,value): cls.dict_icon = cls.setField(cls.dict_icon,value,convert=False)
@classmethod
def getIcon(cls): return cls.getFieldValue(cls.dict_icon)
@classmethod
def setListChapter(cls,value): cls.dict_list_chapter = cls.setField(cls.dict_list_chapter,value,convert=False)
@classmethod
def getListChapter(cls): return cls.getFieldValue(cls.dict_list_chapter)
@classmethod
def getLenChapters(cls): return len(cls.getListChapter())
@classmethod
def setChapterImgs(cls,value,exec=None,item=None):
cls.dict_chapter_imgs = cls.setField(cls.dict_chapter_imgs,cls.parseExec(value,exec=exec,item=item),convert=False)
@classmethod
def getChapterImgs(cls): return cls.getFieldValue(cls.dict_chapter_imgs)
@classmethod
def setUpdateAt(cls,value): cls.update_at = value
@classmethod
def getUpdateAt(cls): return cls.update_at
@classmethod
def setCurrentChapterImg(cls,value): cls.current_chapter_img = value
@classmethod
def getCurrentChapterImg(cls): return cls.current_chapter_img
@classmethod
def setChapterFilesName(cls,value): cls.file_chapter_imgs= value
@classmethod
def getChapterFilesName(cls): return cls.file_chapter_imgs
@classmethod
def setCurrentDownLink(cls,value): cls.CURRENT_DOWN_LINK = value
@classmethod
def getCurrentDownLink(cls): return cls.CURRENT_DOWN_LINK
class ListComic:
LIST_COMIC_QUEUE = Queue()
(LIST_COMIC_NAME,LIST_COMIC_LINK,LIST_COMIC_UPDATEAT) = [None,None,None]
@classmethod
def setListComicsLinksUpdateAt(cls,names,links,update_at):
if isinstance(names,list) and isinstance(links,list) and isinstance(update_at,list):
for x in range(0,len(names)):
cls.LIST_COMIC_QUEUE.put([names[x],links[x],update_at[x]])
@classmethod
def getListComicsLinksUpdateAt(cls):
result = None
if cls.LIST_COMIC_NAME != None and cls.LIST_COMIC_LINK != None:
cls.setListComicsLinksUpdateAt(cls.LIST_COMIC_NAME,cls.LIST_COMIC_LINK,cls.LIST_COMIC_UPDATEAT)
(cls.LIST_COMIC_NAME,cls.LIST_COMIC_LINK,cls.LIST_COMIC_UPDATEAT) = [None,None,None]
if not cls.LIST_COMIC_QUEUE.empty(): result = cls.LIST_COMIC_QUEUE.get(False)
return result
@classmethod
def addListComicChapterLink(cls,name,link,update_at):
if name != None and link != None:
cls.LIST_COMIC_QUEUE.put(name,link,update_at)
@classmethod
def getListValue(cls,result,type,start_add=None,result_type="list"):
if result == None: return None
if type == None: return result
if result_type == "list" and type != None:
data = []
for x in range(0, len(result)):
if start_add != None:
data.append(start_add+result[x].get(type))
else:
data.append(result[x].get(type))
return data
return result
@classmethod
def setListComicName(cls,value,type=None): cls.LIST_COMIC_NAME = cls.getListValue(value,type)
@classmethod
def getListComicName(cls): return cls.LIST_COMIC_NAME
@classmethod
def setListComicChapterLink(cls,value,type=None,start_add=None): cls.LIST_COMIC_LINK = cls.getListValue(value,type,start_add)
@classmethod
def getListComicChapterLink(cls): return cls.LIST_COMIC_LINK
@classmethod
def setListComicUpdateAt(cls,value,type=None): cls.LIST_COMIC_UPDATEAT = cls.getListValue(value,type)
@classmethod
def getListComicUpdateAt(cls): return cls.LIST_COMIC_UPDATEAT
@classmethod
def getListComicChapterLink(cls): return cls.LIST_COMIC_QUEUE.get(False)
#domain end....
+26 -83
View File
@@ -1,41 +1,14 @@
import json,os
import logging
from xml.dom.minidom import Document
from Comics.utils.Comic import Comic
from Comics.utils.Constant import ComicPath
from itemadapter import is_item, ItemAdapter
class ComicInfoEntity:
@classmethod
def getNodes(cls):
return [Comic.dict_chapter,Comic.dict_comic_name,Comic.dict_number,Comic.dict_comic_names,
Comic.dict_dep,Comic.dict_year,Comic.dict_month,Comic.dict_day,Comic.dict_author,
Comic.dict_cbs,Comic.dict_genre,Comic.dict_tags,Comic.dict_page_count,
Comic.dict_language,Comic.dict_agerating,Comic.dict_pages]
@classmethod
def getJsonNodes(cls):
return [Comic.dict_chapter,Comic.dict_comic_name,Comic.dict_icon,Comic.dict_number,
Comic.dict_comic_names,
Comic.dict_dep,Comic.dict_year,Comic.dict_month,Comic.dict_day,Comic.dict_author,
Comic.dict_cbs,Comic.dict_genre,Comic.dict_tags,Comic.dict_page_count,
Comic.dict_language,Comic.dict_agerating,Comic.dict_pages,
Comic.dict_list_chapter,Comic.dict_chapter_imgs]
class ComicInfo:
IS_NEW_ICON = False
document = Document()
path_comic_info = None
@classmethod
def parseExec(cls,data,exec,start_add=None,item=True):
if data !=None and exec != None:
dots = str(exec).split(".")
if not isinstance(data,dict): data = json.loads(data)
for dot in dots:
data = data.get(dot)
if start_add != None and data != None:
data = start_add+data
return data
@classmethod
def setNodeAndValue(cls,node,value):
if value != None:
@@ -50,12 +23,12 @@ class ComicInfo:
#页数
@classmethod
def setPages(cls,values=None):
if values == None: values = Comic.getChapterFilesName()
#if values == None: values = Comic.getChapterFilesName()
if values != None and isinstance(values,list):
suffix = "."+str(values[0]).split(".")[-1]
join_list=",".join(values).replace(suffix,"")
values = join_list.split(",")
Comic.setPageCount(len(values)+1 if cls.IS_NEW_ICON else len(values))
#Comic.setPageCount(len(values)+1 if cls.IS_NEW_ICON else len(values))
root_node = cls.document.createElement("Pages")
if cls.IS_NEW_ICON:
#添加封面
@@ -68,12 +41,12 @@ class ComicInfo:
page = page.split("_")[-1]
c_node.setAttribute("Image",page)
root_node.appendChild(c_node)
Comic.dict_pages = Comic.setField(Comic.dict_pages,root_node,convert=False)
#Comic.dict_pages = Comic.setField(Comic.dict_pages,root_node,convert=False)
@classmethod
def getBaseUrl(cls,url=None):
if url == None:
url = Comic.getHomePage()
#if url == None:
# url = Comic.getHomePage()
(num,index) = [3,0]
for x in range(0, num):
index = str(url).find("/",index)+1
@@ -84,24 +57,30 @@ class ComicInfo:
def root_node(cls,root_value): return cls.document.createElement(root_value)
@classmethod
def add_nodes(cls,root,list_value):
if len(list_value) == 0: return list_value
for value in list_value:
#Comic.chapter
if value[0] == None and value[4]:
#数据为空 value[0], 但不允许为空value[4] = False
msg = f"#数据为空 key={value[3]} value[0]={value[0]}, 但不允许为空value[4]={value[4]}"
logger.error(msg)
exit()
if value[0] != None: root.appendChild(cls.setNodeAndValue(value[2],value[0]))
def add_nodes(cls,root,item):
item = ItemAdapter(item)
keys = item.keys()
files = item.field_names()
values = item.values()
print("test")
#if len(list_value) == 0: return list_value
#for value in list_value:
# #Comic.chapter
# if value[0] == None and value[4]:
# #数据为空 value[0], 但不允许为空value[4] = False
# msg = f"#数据为空 key={value[3]} value[0]={value[0]}, 但不允许为空value[4]={value[4]}"
# logging.error(msg)
# exit()
# if value[0] != None: root.appendChild(cls.setNodeAndValue(value[2],value[0]))
@classmethod
def initComicInfoXML(cls):
cls.setPages()
@classmethod
def writeComicInfoXML(cls,overlay=False):
save_path = ComicPath.getPathComicInfoXML()
def writeComicInfoXML(cls,item,overlay=False):
#save_path = ComicPath.getPathComicInfoXML()
save_path = "ComicInfo.xml"
if os.path.exists(save_path):
if overlay:
os.remove(save_path)
@@ -113,44 +92,8 @@ class ComicInfo:
root = cls.root_node("ComicInfo")
new_document = Document()
new_document.appendChild(root)
cls.add_nodes(root,ComicInfoEntity.getNodes())
cls.add_nodes(root, item)
with open(save_path, "w", encoding="utf-8") as fo:
new_document.writexml(fo, indent='', addindent='\t', newl='\n', encoding="utf-8")
fo.close()
logging.info(f"已生成文件... {save_path}")
@classmethod
def setComicInfo(cls,comicname=None,homepage=None,alias=None,author=None,icon=None,tags=None,
dep=None,genre=None,lang=None,age_rating=None,chapters=None,current_chapter_img=None):
author = ",".join(set(str(str(author).replace("&",",").replace(" ",",")).split(",")))
Comic.setHomePage(homepage)
Comic.setIcon(icon)
Comic.setListChapter(chapters)
#Comic.setUpdateAt(update_at)
Comic.setComicName(str(comicname))
#if alias != None: comicInfo.setComicNames(alias)
Comic.setAuthor(author)
Comic.setTags(tags)
Comic.setDep(dep)
#comicInfo.setCBS("韩漫")
if genre != None: Comic.setGenre(genre)
Comic.setLanguage(lang)
Comic.setAgeRating(age_rating)
Comic.setCurrentChapterImg(current_chapter_img)
@classmethod
def writeJson(cls):
dict_data = {}
nodes = ComicInfoEntity.getJsonNodes()
for node in nodes:
key = Comic.getFieldNode(node)
value = Comic.getFieldOrigin(node)
if isinstance(value,list):
value = ",".join(value)
if key != None and isinstance(value,str):
child_dict = { key : value}
dict_data.update(child_dict)
s = json.dumps(dict_data,ensure_ascii=True)
logging.debug(f"json={s}")
with open(ComicPath.getPathConfComicChapterJson(mkdir=True),"w") as fs:
fs.write(s)
logging.info(f"已生成文件... {save_path}")
+24 -2
View File
@@ -1,5 +1,7 @@
import os.path
import re
from opencc import OpenCC
from Comics.settings import IMAGES_STORE
class ComicPath:
@classmethod
def getDirComicChapter(cls):
@@ -13,4 +15,24 @@ class ComicPath:
#繁体中文转简体中文
@classmethod
def ChineseConvert(cls, text,convert='t2s'): return OpenCC(convert).convert(str(text))
def chinese_convert(cls, text,convert='t2s'): return OpenCC(convert).convert(str(text))
#处理成符合规定的文件名
@classmethod
def fix_file_name(cls, filename, replace=None):
if not isinstance(filename, str):
return filename
in_tab = r'[?*/\|.:><]'
str_replace = ""
if replace is not None:
str_replace = replace
filename = re.sub(in_tab, str_replace, filename)
count = 1
while True:
str_file = filename[0-count]
if str_file == " ":
count += 1
else:
filename = filename[0:len(filename)+1-count]
break
return filename
+16 -7
View File
@@ -2,7 +2,17 @@ import base64,hashlib,os,shutil
import math,time,json,datetime,logging
from PIL import Image
from tinydb import TinyDB, Query
from Comics.spiders.utils.Constant import ComicPath
from Comics.utils.Constant import ComicPath
class fileUtils:
@classmethod
def save_file(cls,path,data):
dir = os.path.dirname(path)
if not os.path.exists(dir):
os.makedirs(dir)
with open(path,'w',encoding='utf-8') as fs:
fs.write(str(data))
fs.close()
class CommonUtils:
@classmethod
@@ -31,11 +41,9 @@ class imageUtils:
@classmethod
def deScrambleImagesByPath(cls,img_path,img_save=None):
if os.path.basename(img_path).startswith("scramble="):
imageUtils.encode_scramble_image(img_path,img_save)
return True
else:
return False
img_path = imageUtils.encode_scramble_image(img_path,img_save)
return img_path
@classmethod
def encodeImage(cls,str_en):
#print("en",str_en)
@@ -223,4 +231,5 @@ class imageUtils:
print("解密成功=",save_path)
if os.path.exists(imgpath):
os.remove(imgpath)
print("remove=",imgpath)
print("remove=",imgpath)
return save_path