fix
This commit is contained in:
parent
a8f44ec334
commit
6ca7ce1992
2
.gitignore
vendored
2
.gitignore
vendored
@ -1,4 +1,4 @@
|
|||||||
COMICOUT/
|
COMICOUT/
|
||||||
.conf/
|
.conf/
|
||||||
**/__pycache__/**
|
**/__pycache__/**
|
||||||
.vscode
|
logconf.log
|
||||||
@ -91,7 +91,13 @@ class baseComic:
|
|||||||
cls.Onechapter(chapter_url,scramble)
|
cls.Onechapter(chapter_url,scramble)
|
||||||
#进入下个阶段
|
#进入下个阶段
|
||||||
#章节图片全部下载后,调用下载封面
|
#章节图片全部下载后,调用下载封面
|
||||||
if ciUtils.isProgressDown: downloadUtils.download_comic_icon()
|
comic_icon_path = ComicPath.getPathConfComicIcon()
|
||||||
|
chapter_icon_path = ComicPath.getPathCBZComicChapterIcon()
|
||||||
|
if ciUtils.isProgressDown and os.path.exists(comic_icon_path) and not os.path.exists(chapter_icon_path):
|
||||||
|
shutil.copy(comic_icon_path,chapter_icon_path)
|
||||||
|
ntfy.sendMsg(f"复制成功:{comic_icon_path} -> {chapter_icon_path}")
|
||||||
|
ciUtils.iconDB()
|
||||||
|
ciUtils.setProgressCBZ()
|
||||||
#下个阶段
|
#下个阶段
|
||||||
if ciUtils.isProgressCBZ: is_next = CBZUtils.packAutoComicChapterCBZ()
|
if ciUtils.isProgressCBZ: is_next = CBZUtils.packAutoComicChapterCBZ()
|
||||||
#except Exception as e: is_next = ntfy.sendMsg(f"{ci.getComicName()} 下载出错了",error=e)
|
#except Exception as e: is_next = ntfy.sendMsg(f"{ci.getComicName()} 下载出错了",error=e)
|
||||||
@ -126,22 +132,19 @@ class baseComic:
|
|||||||
except:
|
except:
|
||||||
htmlUtils.remove_HtmlCache(url)
|
htmlUtils.remove_HtmlCache(url)
|
||||||
cls.comicChapterDownload(url)
|
cls.comicChapterDownload(url)
|
||||||
|
|
||||||
(list_img,files_name) = [Comic.getChapterImgs(),Comic.file_chapter_imgs]
|
|
||||||
#保存信息
|
#保存信息
|
||||||
ci.writeJson()
|
ci.writeJson()
|
||||||
#验证数据是已存在且是否完整
|
#验证数据是已存在且是否完整
|
||||||
is_next = CBZUtils.nextCBZ()
|
is_next = CBZUtils.nextCBZ()
|
||||||
is_old=CBZUtils.updateOldCBZ(files_name)
|
is_old=CBZUtils.updateOldCBZ()
|
||||||
#不存在ComicInfo.xml则生成
|
#不存在ComicInfo.xml则生成
|
||||||
if is_next and fu.notExists(ComicPath.getPathComicInfoXML()): ci.writeComicInfoXML()
|
if is_next and fu.notExists(ComicPath.getPathComicInfoXML()): ci.writeComicInfoXML()
|
||||||
if is_next and not is_old:
|
if not is_next and is_old:
|
||||||
ComicPath.TIME_SLEEP = random.randint(8,15)
|
|
||||||
downloadUtils.download_images(list_img,ComicPath.getDirComicChapter(),files_name=files_name,concurrency=5,timeout=10)
|
|
||||||
# ntfy.sendMsg("等待数据检验中...",sleep=0.5)
|
|
||||||
is_next = fu.equImages(ComicPath.getDirComicChapter(),list_img)
|
|
||||||
# if not is_next: ntfy.sendMsg(msg=f"下载数据(不完整,{int(repeat*2)}秒钟后尝试第{repeat}次",sleep=int(repeat*2))
|
|
||||||
# repeat += 1
|
|
||||||
else:
|
|
||||||
ComicPath.TIME_SLEEP = 0.5
|
ComicPath.TIME_SLEEP = 0.5
|
||||||
|
downloadUtils.queueDownClear()
|
||||||
|
else:
|
||||||
|
ComicPath.TIME_SLEEP = random.randint(8,15)
|
||||||
|
#图标
|
||||||
|
downloadUtils.putDownImageUrlDirFile(Comic.getIcon(),ComicPath.getDirConfComic(),ComicPath.COMIC_ICON_FILE_NAME)
|
||||||
|
downloadUtils.start_downloads()
|
||||||
return is_next
|
return is_next
|
||||||
@ -5,30 +5,30 @@ from utils.OldUtils import OldUtils
|
|||||||
|
|
||||||
class Comic:
|
class Comic:
|
||||||
# ComicInfo.xml and ComicChapter.json bengin
|
# ComicInfo.xml and ComicChapter.json bengin
|
||||||
# value origin node dep
|
# value origin node dep required
|
||||||
dict_chapter = [None,None,"Title","章节名"]
|
dict_chapter = [None,None,"Title","章节名",True]
|
||||||
dict_comic_name = [None,None,"Series","漫画名"]
|
dict_comic_name = [None,None,"Series","漫画名",True]
|
||||||
dict_number = [None,None,"Number","编号"]
|
dict_number = [None,None,"Number","编号",True]
|
||||||
dict_comic_names = [None,None,"SeriesGroup","别名"]
|
dict_comic_names = [None,None,"SeriesGroup","别名",False]
|
||||||
dict_dep = [None,None,"Summary","概述"]
|
dict_dep = [None,None,"Summary","概述",True]
|
||||||
dict_year = [None,None,"Year","年"]
|
dict_year = [None,None,"Year","年",False]
|
||||||
dict_month = [None,None,"Month","月"]
|
dict_month = [None,None,"Month","月",False]
|
||||||
dict_day = [None,None,"Day","日"]
|
dict_day = [None,None,"Day","日",False]
|
||||||
dict_author = [None,None,"Writer","作者"]
|
dict_author = [None,None,"Writer","作者",True]
|
||||||
dict_cbs = [None,None,"Publisher","出版社"]
|
dict_cbs = [None,None,"Publisher","出版社",False]
|
||||||
dict_genre = [None,None,"Genre","流派"]
|
dict_genre = [None,None,"Genre","流派",True]
|
||||||
dict_tags = [None,None,"Tags","标签"]
|
dict_tags = [None,None,"Tags","标签",True]
|
||||||
dict_homepage = [None,None,"Web","主页"]
|
dict_homepage = [None,None,"Web","主页",False]
|
||||||
dict_page_count = [None,None,"PageCount","总页数"]
|
dict_page_count = [None,None,"PageCount","总页数",True]
|
||||||
dict_language = [None,None,"LanguageISO","语言"]
|
dict_language = [None,None,"LanguageISO","语言",True]
|
||||||
dict_agerating = [None,None,"AgeRating","年龄分级"]
|
dict_agerating = [None,None,"AgeRating","年龄分级",False]
|
||||||
dict_pages = [None,None,"Pages","页码"]
|
dict_pages = [None,None,"Pages","页码",True]
|
||||||
CURRENT_DOWN_LINK = None
|
CURRENT_DOWN_LINK = None
|
||||||
# ComicInfo.xml and ComicChapter.json end
|
# ComicInfo.xml and ComicChapter.json end
|
||||||
dict_icon = [None,None,"Icon","图标"]
|
dict_icon = [None,None,"Icon","图标",True]
|
||||||
dict_chapter_imgs = [None,None,"ChapterImgs","图像"]
|
dict_chapter_imgs = [None,None,"ChapterImgs","图像",True]
|
||||||
#主页
|
#主页
|
||||||
dict_list_chapter = [None,None,"ListChapter","全部章节名"]
|
dict_list_chapter = [None,None,"ListChapter","全部章节名",True]
|
||||||
(update_at,current_chapter_img,file_chapter_imgs) = [None,None,None]
|
(update_at,current_chapter_img,file_chapter_imgs) = [None,None,None]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -3,6 +3,7 @@ from xml.dom.minidom import Document
|
|||||||
from utils.FileUtils import dbUtils as db
|
from utils.FileUtils import dbUtils as db
|
||||||
from common.Comic import Comic
|
from common.Comic import Comic
|
||||||
from common.Constant import ComicPath
|
from common.Constant import ComicPath
|
||||||
|
from utils.Logger import logger
|
||||||
|
|
||||||
class ComicInfoEntity:
|
class ComicInfoEntity:
|
||||||
@classmethod
|
@classmethod
|
||||||
@ -48,7 +49,8 @@ class ComicInfo:
|
|||||||
|
|
||||||
#页数
|
#页数
|
||||||
@classmethod
|
@classmethod
|
||||||
def setPages(cls,values):
|
def setPages(cls,values=None):
|
||||||
|
if values == None: values = Comic.getChapterFilesName()
|
||||||
if values != None and isinstance(values,list):
|
if values != None and isinstance(values,list):
|
||||||
suffix = "."+str(values[0]).split(".")[-1]
|
suffix = "."+str(values[0]).split(".")[-1]
|
||||||
join_list=",".join(values).replace(suffix,"")
|
join_list=",".join(values).replace(suffix,"")
|
||||||
@ -86,10 +88,20 @@ class ComicInfo:
|
|||||||
if len(list_value) == 0: return list_value
|
if len(list_value) == 0: return list_value
|
||||||
for value in list_value:
|
for value in list_value:
|
||||||
#Comic.chapter
|
#Comic.chapter
|
||||||
|
if value[0] == None and value[4]:
|
||||||
|
#数据为空 value[0], 但不允许为空value[4] = False
|
||||||
|
msg = f"#数据为空 key={value[3]} value[0]={value[0]}, 但不允许为空value[4]={value[4]}"
|
||||||
|
logger.error(msg)
|
||||||
|
exit()
|
||||||
if value[0] != None: root.appendChild(cls.setNodeAndValue(value[2],value[0]))
|
if value[0] != None: root.appendChild(cls.setNodeAndValue(value[2],value[0]))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def initComicInfoXML(cls):
|
||||||
|
cls.setPages()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def writeComicInfoXML(cls,overlay=False):
|
def writeComicInfoXML(cls,overlay=False):
|
||||||
|
cls.initComicInfoXML()
|
||||||
root = cls.root_node("ComicInfo")
|
root = cls.root_node("ComicInfo")
|
||||||
new_document = Document()
|
new_document = Document()
|
||||||
new_document.appendChild(root)
|
new_document.appendChild(root)
|
||||||
@ -98,7 +110,7 @@ class ComicInfo:
|
|||||||
with open(save_path, "w", encoding="utf-8") as fo:
|
with open(save_path, "w", encoding="utf-8") as fo:
|
||||||
new_document.writexml(fo, indent='', addindent='\t', newl='\n', encoding="utf-8")
|
new_document.writexml(fo, indent='', addindent='\t', newl='\n', encoding="utf-8")
|
||||||
fo.close()
|
fo.close()
|
||||||
print(f"已生成文件... {save_path}")
|
logger.info(f"已生成文件... {save_path}")
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def setComicInfo(cls,comicname=None,homepage=None,alias=None,author=None,icon=None,tags=None,
|
def setComicInfo(cls,comicname=None,homepage=None,alias=None,author=None,icon=None,tags=None,
|
||||||
@ -131,7 +143,8 @@ class ComicInfo:
|
|||||||
if key != None and isinstance(value,str):
|
if key != None and isinstance(value,str):
|
||||||
child_dict = { key : value}
|
child_dict = { key : value}
|
||||||
dict_data.update(child_dict)
|
dict_data.update(child_dict)
|
||||||
s = json.dumps(dict_data,ensure_ascii=False)
|
s = json.dumps(dict_data,ensure_ascii=True)
|
||||||
|
logger.info(f"json={s}")
|
||||||
with open(ComicPath.getPathConfComicChapterJson(mkdir=True),"w") as fs:
|
with open(ComicPath.getPathConfComicChapterJson(mkdir=True),"w") as fs:
|
||||||
fs.write(s)
|
fs.write(s)
|
||||||
|
|
||||||
|
|||||||
@ -76,6 +76,8 @@ class pathStr:
|
|||||||
|
|
||||||
class ComicPath:
|
class ComicPath:
|
||||||
COMIC_ICON_NAME = "000"
|
COMIC_ICON_NAME = "000"
|
||||||
|
COMIC_ICON_SUFFIX = "jpg"
|
||||||
|
COMIC_ICON_FILE_NAME = COMIC_ICON_NAME+"."+COMIC_ICON_SUFFIX
|
||||||
DEFAULT_PATH = "default"
|
DEFAULT_PATH = "default"
|
||||||
PROGRESS_PATH = "progress"
|
PROGRESS_PATH = "progress"
|
||||||
ICONS_PATH = "icons"
|
ICONS_PATH = "icons"
|
||||||
@ -115,7 +117,7 @@ class ComicPath:
|
|||||||
def getPathConfComicChapterJson(cls,mkdir=True): return cls.setDirConf([Comic.getOriginComicName(),Comic.getOriginChapterName()],suffix="json",mkdir=mkdir)
|
def getPathConfComicChapterJson(cls,mkdir=True): return cls.setDirConf([Comic.getOriginComicName(),Comic.getOriginChapterName()],suffix="json",mkdir=mkdir)
|
||||||
#漫画Icon路径
|
#漫画Icon路径
|
||||||
@classmethod
|
@classmethod
|
||||||
def getPathConfComicIcon(cls,mkdir=True,suffix="jpg"): return cls.setDirConf([Comic.getOriginComicName(),cls.COMIC_ICON_NAME],suffix=suffix,mkdir=mkdir)
|
def getPathConfComicIcon(cls,mkdir=True): return cls.setDirConf([Comic.getOriginComicName(),cls.COMIC_ICON_NAME],suffix=cls.COMIC_ICON_SUFFIX,mkdir=mkdir)
|
||||||
#Icons json文件路径
|
#Icons json文件路径
|
||||||
@classmethod
|
@classmethod
|
||||||
def getDirConfDefault(cls,path=None,suffix=None,mkdir=True): return cls.setDirConf([cls.DEFAULT_PATH,path],suffix=suffix,mkdir=mkdir)
|
def getDirConfDefault(cls,path=None,suffix=None,mkdir=True): return cls.setDirConf([cls.DEFAULT_PATH,path],suffix=suffix,mkdir=mkdir)
|
||||||
@ -130,7 +132,7 @@ class ComicPath:
|
|||||||
def getDirCBZComicChapter(cls,suffix=None): return cls.setDirCBZ([Comic.getComicName(),Comic.getChapterName()],suffix=suffix)
|
def getDirCBZComicChapter(cls,suffix=None): return cls.setDirCBZ([Comic.getComicName(),Comic.getChapterName()],suffix=suffix)
|
||||||
#漫画章节CBZ同根下封面
|
#漫画章节CBZ同根下封面
|
||||||
@classmethod
|
@classmethod
|
||||||
def getPathCBZComicChapterIcon(cls,suffix="jpg"): return cls.setDirCBZ([Comic.getComicName(),Comic.getChapterName()],suffix=suffix)
|
def getPathCBZComicChapterIcon(cls,suffix="jpg",mkdir=True): return cls.setDirCBZ([Comic.getComicName(),Comic.getChapterName()],suffix=suffix,mkdir=mkdir)
|
||||||
#旧版本漫画章节CBZ路径
|
#旧版本漫画章节CBZ路径
|
||||||
@classmethod
|
@classmethod
|
||||||
def getPathOldCBZComicChapter(cls): return cls.setDirOld([Comic.getOriginComicName(),Comic.getOriginChapterName()],suffix="CBZ")
|
def getPathOldCBZComicChapter(cls): return cls.setDirOld([Comic.getOriginComicName(),Comic.getOriginChapterName()],suffix="CBZ")
|
||||||
|
|||||||
@ -6,6 +6,7 @@ from common.Constant import ComicPath
|
|||||||
from utils.FileUtils import imageUtils
|
from utils.FileUtils import imageUtils
|
||||||
from utils.NetUtils import htmlUtils
|
from utils.NetUtils import htmlUtils
|
||||||
from utils.ComicUtils import ntfy
|
from utils.ComicUtils import ntfy
|
||||||
|
from utils.NetUtils import downloadUtils as downUtils
|
||||||
|
|
||||||
class DomainDown:
|
class DomainDown:
|
||||||
@classmethod
|
@classmethod
|
||||||
@ -30,16 +31,19 @@ class DomainDown:
|
|||||||
ntfy.sendMsg(f"未获取到章节图像 comic_name={Comic.getComicName()} chapter={Comic.getChapterName()}")
|
ntfy.sendMsg(f"未获取到章节图像 comic_name={Comic.getComicName()} chapter={Comic.getChapterName()}")
|
||||||
|
|
||||||
count = 1
|
count = 1
|
||||||
list_img,list_file_name = [[],[]]
|
(files_name,images_url) = [[],[]]
|
||||||
for image in images:
|
for image in images:
|
||||||
(image_src,scramble) = [image.get("src"),image.get("scramble")]
|
(image_src,scramble) = [image.get("src"),image.get("scramble")]
|
||||||
count_image = "{:0>3d}".format(count)
|
count_image = "{:0>3d}".format(count)
|
||||||
list_img.append(image_src)
|
|
||||||
image_src_suffix = "."+str(image_src).split(".")[-1]
|
image_src_suffix = "."+str(image_src).split(".")[-1]
|
||||||
if scramble:
|
if scramble:
|
||||||
de_str = str(image_src).split("/")[-1].replace(image_src_suffix,"==")
|
de_str = str(image_src).split("/")[-1].replace(image_src_suffix,"==")
|
||||||
blocks_num = imageUtils.encodeImage(de_str)
|
blocks_num = imageUtils.encodeImage(de_str)
|
||||||
count_image = "scramble="+str(blocks_num)+"_"+count_image
|
count_image = "scramble="+str(blocks_num)+"_"+count_image
|
||||||
list_file_name.append(count_image+image_src_suffix)
|
image_file_name = count_image+image_src_suffix
|
||||||
|
files_name.append(image_file_name)
|
||||||
|
images_url.append(image_src)
|
||||||
|
downUtils.putDownImageUrlDirFile(image_src,ComicPath.getDirComicChapter(),image_file_name)
|
||||||
count+=1
|
count+=1
|
||||||
ciUtils.comicChapterDownload(list_img,list_file_name)
|
Comic.setChapterImgs(images_url)
|
||||||
|
Comic.setChapterFilesName(files_name)
|
||||||
45
logconf.yml
Normal file
45
logconf.yml
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
version: 1
|
||||||
|
#是否覆盖掉已经存在的loggers
|
||||||
|
disable_existing_loggers: True
|
||||||
|
|
||||||
|
formatters:
|
||||||
|
|
||||||
|
tostrout:
|
||||||
|
format: "%(asctime)s - %(name)s - %(levelname)-9s - %(filename)-8s : %(lineno)s line - %(message)s"
|
||||||
|
datefmt: "%Y/%m/%d %H:%M:%S"
|
||||||
|
|
||||||
|
tofile:
|
||||||
|
format: "%(asctime)s - %(name)s - %(levelname)-9s - %(filename)-8s : %(lineno)s line - %(message)s"
|
||||||
|
|
||||||
|
handlers:
|
||||||
|
sh:
|
||||||
|
class: logging.StreamHandler
|
||||||
|
level: DEBUG
|
||||||
|
formatter: tostrout
|
||||||
|
stream: ext://sys.stdout
|
||||||
|
|
||||||
|
fh:
|
||||||
|
class: logging.handlers.TimedRotatingFileHandler
|
||||||
|
filename: logconf.log
|
||||||
|
interval: 1
|
||||||
|
backupCount: 2
|
||||||
|
when: D
|
||||||
|
level: INFO
|
||||||
|
formatter: tofile
|
||||||
|
|
||||||
|
loggers:
|
||||||
|
logger1:
|
||||||
|
level: DEBUG
|
||||||
|
handlers: [sh]
|
||||||
|
#是否往上级Logger传递,如果为yes的话,root选择了两个logger,这里的日志也会在两个logger的配置中输出,会重复。所以选No,自己记录自己的日志。
|
||||||
|
propagate: no
|
||||||
|
|
||||||
|
logger2:
|
||||||
|
level: INFO
|
||||||
|
handlers: [fh]
|
||||||
|
propagate: no
|
||||||
|
|
||||||
|
root:
|
||||||
|
level: DEBUG
|
||||||
|
handlers: [sh,fh]
|
||||||
|
propagate: no
|
||||||
@ -2,13 +2,12 @@ import os,shutil,time,requests
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from zipfile import ZipFile
|
from zipfile import ZipFile
|
||||||
|
from queue import Queue
|
||||||
from common.ComicInfo import ComicInfoUtils as ciUtils
|
from common.ComicInfo import ComicInfoUtils as ciUtils
|
||||||
from common.ComicInfo import ComicInfo as ci
|
from common.ComicInfo import ComicInfo as ci
|
||||||
from common.ComicInfo import Comic
|
from common.ComicInfo import Comic
|
||||||
from utils.FileUtils import fileUtils as fu
|
from utils.FileUtils import fileUtils as fu
|
||||||
from common.Constant import pathStr
|
|
||||||
from common.Constant import ComicPath
|
from common.Constant import ComicPath
|
||||||
from utils.OldUtils import OldUtils
|
|
||||||
|
|
||||||
class ntfy:
|
class ntfy:
|
||||||
@classmethod
|
@classmethod
|
||||||
@ -121,7 +120,7 @@ class CBZUtils:
|
|||||||
with ZipFile(path, "r") as zip_file:
|
with ZipFile(path, "r") as zip_file:
|
||||||
result = zip_file.namelist()
|
result = zip_file.namelist()
|
||||||
if filter:
|
if filter:
|
||||||
filter_icon = ComicPath.COMIC_ICON_NAME+".jpg"
|
filter_icon = ComicPath.COMIC_ICON_FILE_NAME
|
||||||
filter_info_xml = ComicPath.COMIC_INFO_XML
|
filter_info_xml = ComicPath.COMIC_INFO_XML
|
||||||
if filter_icon in result: result.remove(filter_icon)
|
if filter_icon in result: result.remove(filter_icon)
|
||||||
if filter_info_xml in result: result.remove(filter_info_xml)
|
if filter_info_xml in result: result.remove(filter_info_xml)
|
||||||
@ -155,7 +154,8 @@ class CBZUtils:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def updateOldCBZ(cls,filesname,result=False):
|
def updateOldCBZ(cls,filesname=None,result=False):
|
||||||
|
if filesname == None: filesname = Comic.getChapterFilesName()
|
||||||
old_zipfile_path = ComicPath.getPathOldCBZComicChapter()
|
old_zipfile_path = ComicPath.getPathOldCBZComicChapter()
|
||||||
|
|
||||||
#判断是否存在已下载CBZ文件
|
#判断是否存在已下载CBZ文件
|
||||||
|
|||||||
@ -3,7 +3,7 @@ import math,time,json
|
|||||||
from PIL import Image
|
from PIL import Image
|
||||||
from tinydb import TinyDB, Query
|
from tinydb import TinyDB, Query
|
||||||
from common.Constant import ComicPath
|
from common.Constant import ComicPath
|
||||||
from common.Comic import Comic
|
from utils.Logger import logger
|
||||||
|
|
||||||
class imageUtils:
|
class imageUtils:
|
||||||
|
|
||||||
@ -226,15 +226,15 @@ class fileUtils:
|
|||||||
img = Image.open(file_path)
|
img = Image.open(file_path)
|
||||||
img.verify()
|
img.verify()
|
||||||
img.close()
|
img.close()
|
||||||
print(f"{file_path} 类型为type:{type} size: {file_size_unit}")
|
logger.debug(f"{file_path} 类型为type:{type} size: {file_size_unit}")
|
||||||
return True
|
return True
|
||||||
except:
|
except:
|
||||||
if os.path.exists(file_path):
|
if os.path.exists(file_path):
|
||||||
os.remove(file_path)
|
os.remove(file_path)
|
||||||
print(f"{file_path}已损坏 type:{type} size: {file_size_unit},删除重试中")
|
logger.warning(f"{file_path}已损坏 type:{type} size: {file_size_unit},删除重试中")
|
||||||
return False
|
return False
|
||||||
else:
|
else:
|
||||||
print(f"文件小于100b删除中,{file_path} size: {file_size_unit}")
|
logger.warning(f"文件小于100b删除中,{file_path} size: {file_size_unit}")
|
||||||
try:
|
try:
|
||||||
os.remove(file_path)
|
os.remove(file_path)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@ -263,15 +263,15 @@ class fileUtils:
|
|||||||
if not os.path.exists(path):
|
if not os.path.exists(path):
|
||||||
return None
|
return None
|
||||||
try:
|
try:
|
||||||
print(f"{path} 删除中...")
|
logger.debug(f"{path} 删除中...")
|
||||||
if os.path.isfile(path):
|
if os.path.isfile(path):
|
||||||
os.remove(path)
|
os.remove(path)
|
||||||
if os.path.isdir(path):
|
if os.path.isdir(path):
|
||||||
shutil.rmtree(path)
|
shutil.rmtree(path)
|
||||||
print(f"已删除:{path}")
|
logger.debug(f"已删除:{path}")
|
||||||
return True
|
return True
|
||||||
except:
|
except:
|
||||||
print(f"删除错误:{path}")
|
logger.warning(f"删除错误:{path}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
#文件保存
|
#文件保存
|
||||||
@ -298,7 +298,7 @@ class fileUtils:
|
|||||||
result = path + "文件写入成功"
|
result = path + "文件写入成功"
|
||||||
except:
|
except:
|
||||||
result = path + "文件写入失败"
|
result = path + "文件写入失败"
|
||||||
print(result)
|
logger.debug(result)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
class dbUtils:
|
class dbUtils:
|
||||||
@ -317,7 +317,7 @@ class dbUtils:
|
|||||||
else: db.update({"progress":progress},comic.name== name)
|
else: db.update({"progress":progress},comic.name== name)
|
||||||
msg = "失败"
|
msg = "失败"
|
||||||
if cls.query(name,progress,db_name): msg = "成功"
|
if cls.query(name,progress,db_name): msg = "成功"
|
||||||
print(f"设置{msg}, name={name} value={progress} db={db_name}")
|
logger.info(f"设置{msg}, name={name} value={progress} db={db_name}")
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def query(cls,name,progress=None,db_name=None):
|
def query(cls,name,progress=None,db_name=None):
|
||||||
|
|||||||
18
utils/Logger.py
Normal file
18
utils/Logger.py
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
# -*- coding:utf-8 -*-
|
||||||
|
import logging
|
||||||
|
import logging.config
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
class logger:
|
||||||
|
with open("logconf.yml", "r",encoding="utf-8") as f:
|
||||||
|
dict_conf = yaml.safe_load(f)
|
||||||
|
logging.config.dictConfig(dict_conf)
|
||||||
|
root = logging.getLogger()
|
||||||
|
@classmethod
|
||||||
|
def debug(cls,value): cls.root.debug(value)
|
||||||
|
@classmethod
|
||||||
|
def info(cls,value): cls.root.info(value)
|
||||||
|
@classmethod
|
||||||
|
def warning(cls,value): cls.root.warning(value)
|
||||||
|
@classmethod
|
||||||
|
def error(cls,value): cls.root.error(value)
|
||||||
@ -12,7 +12,7 @@ from common.ComicInfo import ComicInfo as ci
|
|||||||
from common.ComicInfo import Comic
|
from common.ComicInfo import Comic
|
||||||
from common.Constant import ComicPath
|
from common.Constant import ComicPath
|
||||||
from utils.FileUtils import fileUtils as fu
|
from utils.FileUtils import fileUtils as fu
|
||||||
|
from utils.Logger import logger
|
||||||
|
|
||||||
class htmlUtils:
|
class htmlUtils:
|
||||||
|
|
||||||
@ -134,9 +134,9 @@ class htmlUtils:
|
|||||||
if os.path.exists(html_cache_path):
|
if os.path.exists(html_cache_path):
|
||||||
try:
|
try:
|
||||||
os.remove(html_cache_path)
|
os.remove(html_cache_path)
|
||||||
print(f"html_cache更新成功 {html_cache_path}")
|
logger.info(f"html_cache更新成功 {html_cache_path}")
|
||||||
except:
|
except:
|
||||||
print(f"html_cache更新失败 {html_cache_path}")
|
logger.info(f"html_cache更新失败 {html_cache_path}")
|
||||||
#获取html实体数据
|
#获取html实体数据
|
||||||
et = cls.getHTML(url)
|
et = cls.getHTML(url)
|
||||||
if et == None:
|
if et == None:
|
||||||
@ -156,6 +156,9 @@ class htmlUtils:
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
class downloadUtils:
|
class downloadUtils:
|
||||||
|
QUEUE_DOWN = Queue()
|
||||||
|
TYPE_IMG = "image"
|
||||||
|
TYPE_ICON = "icon"
|
||||||
headers = {
|
headers = {
|
||||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||||||
"Proxy-Connection": "keep-alive",
|
"Proxy-Connection": "keep-alive",
|
||||||
@ -164,56 +167,66 @@ class downloadUtils:
|
|||||||
"Accept-Encoding": "gzip, deflate, sdch",
|
"Accept-Encoding": "gzip, deflate, sdch",
|
||||||
# 'Connection': 'close',
|
# 'Connection': 'close',
|
||||||
}
|
}
|
||||||
down_queue = Queue()
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def common_download(cls,file_name,image_url,dst_dir,timeout=10,proxy=None,proxy_type=None):
|
def queueDownClear(cls): cls.QUEUE_DOWN = Queue()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def putDownUrlDirFileType(cls,url,dir,file,type):
|
||||||
|
cls.QUEUE_DOWN.put([url,dir,file,type])
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def getDownUrlDirFileType(cls):
|
||||||
|
if not cls.QUEUE_DOWN.empty(): return cls.QUEUE_DOWN.get(False)
|
||||||
|
else: return None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def putDownImageUrlDirFile(cls,url,dir,file): cls.putDownUrlDirFileType(url,dir,file,cls.TYPE_IMG)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def common_download(cls,repair_max=10,timeout=10,proxy=None,proxy_type=None):
|
||||||
|
result = cls.getDownUrlDirFileType()
|
||||||
|
if result == None: return None
|
||||||
|
(file_url,dir,file,file_type) = [result[0],result[1],result[2],result[3]]
|
||||||
|
if file_url == None:
|
||||||
|
logger.error("common_down file_url 为空")
|
||||||
|
raise NameError("common_down file_url为空")
|
||||||
proxies = None
|
proxies = None
|
||||||
if proxy_type is not None:
|
if proxy_type is not None:
|
||||||
proxies = {
|
proxies = {
|
||||||
"http": proxy_type + "://" + proxy,
|
"http": proxy_type + "://" + proxy,
|
||||||
"https": proxy_type + "://" + proxy }
|
"https": proxy_type + "://" + proxy }
|
||||||
response = None
|
response = None
|
||||||
file_path = os.path.join(dst_dir, file_name)
|
save_path = os.path.join(dir,file)
|
||||||
if os.path.exists(file_path):
|
logger.debug(f"save_path {save_path}")
|
||||||
print("download_image 文件已存在,已跳过=",file_path)
|
if not os.path.exists(dir): os.makedirs(dir)
|
||||||
return None
|
temp_path = save_path+".downloads"
|
||||||
temp_path = os.path.join(dst_dir, file_name+".downloads")
|
|
||||||
repair_count = 1
|
repair_count = 1
|
||||||
|
while not os.path.exists(save_path):
|
||||||
try:
|
try:
|
||||||
response = requests.get(
|
response = requests.get(
|
||||||
image_url, headers=cls.headers, timeout=timeout, proxies=proxies)
|
file_url, headers=cls.headers, timeout=timeout, proxies=proxies)
|
||||||
if response.status_code != 200 and repair_count <= 5:
|
if response.status_code != 200 and repair_count <= repair_max:
|
||||||
|
logger.warning("下载异常")
|
||||||
raise NameError("下载异常")
|
raise NameError("下载异常")
|
||||||
with open(temp_path, 'wb') as f:
|
with open(temp_path, 'wb') as f:
|
||||||
f.write(response.content)
|
f.write(response.content)
|
||||||
time.sleep(0.7)
|
time.sleep(0.7)
|
||||||
response.close()
|
response.close()
|
||||||
#验证是否是图像
|
#验证是否是图像
|
||||||
if fu.ver_file(temp_path,type="image"):
|
if fu.ver_file(temp_path,type=file_type):
|
||||||
shutil.move(temp_path, file_path)
|
shutil.move(temp_path, save_path)
|
||||||
print("## OK: {} {}".format(file_path, image_url))
|
logger.info("## OK: {} {}".format(save_path, file_url))
|
||||||
else:
|
else:
|
||||||
raise NameError("## Fail: {} {}".format(image_url, "图像损坏"))
|
logger.warning("## Fail: {} {}".format(file_url, "图像损坏"))
|
||||||
|
raise NameError("## Fail: {} {}".format(file_url, "图像损坏"))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"common_down() 出错了 {e}")
|
logger.warning(f'重试:第{repair_count}次 异常:{e} {file_url}')
|
||||||
print(f'重试:第{repair_count}次 {image_url}')
|
cls.putDownUrlDirFileType(file_url,dir,file,file_type)
|
||||||
cls.down_queue.put([file_name,image_url,dst_dir])
|
|
||||||
repair_count += 1
|
repair_count += 1
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def download_image(cls,timeout=20, proxy_type=None, proxy=None,type="image"):
|
def start_downloads(cls,repair_max=20,concurrency=None,timeout=20,proxy_type=None, proxy=None):
|
||||||
repeat = 1
|
|
||||||
while not cls.down_queue.empty() and repeat <= 10:
|
|
||||||
data = cls.down_queue.get(False)
|
|
||||||
(file_name,image_url,dst_dir) = [data[0],data[1],data[2]]
|
|
||||||
cls.common_download(file_name,image_url,dst_dir,timeout=timeout,
|
|
||||||
proxy=proxy,proxy_type=proxy_type)
|
|
||||||
repeat += 1
|
|
||||||
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def download_images(cls,image_urls, dst_dir,concurrency=None,timeout=20,proxy_type=None, proxy=None,files_name=None):
|
|
||||||
"""
|
"""
|
||||||
Download image according to given urls and automatically rename them in order.
|
Download image according to given urls and automatically rename them in order.
|
||||||
:param timeout:
|
:param timeout:
|
||||||
@ -226,46 +239,12 @@ class downloadUtils:
|
|||||||
:return: none
|
:return: none
|
||||||
"""
|
"""
|
||||||
if concurrency == None:
|
if concurrency == None:
|
||||||
concurrency = len(image_urls)
|
concurrency = cls.QUEUE_DOWN.qsize()
|
||||||
|
logger.debug(f"concurrency= {concurrency}")
|
||||||
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor:
|
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor:
|
||||||
future_list = list()
|
future_list = list()
|
||||||
count = 0
|
while not cls.QUEUE_DOWN.empty():
|
||||||
if not os.path.exists(dst_dir):
|
logger.debug("QUEUE_DOWN 不为空 准备下载中...")
|
||||||
os.makedirs(dst_dir)
|
|
||||||
for image_url in image_urls:
|
|
||||||
file_name = files_name[count]
|
|
||||||
cls.down_queue.put([file_name,image_url,dst_dir])
|
|
||||||
future_list.append(executor.submit(
|
future_list.append(executor.submit(
|
||||||
cls.download_image,timeout, proxy_type, proxy))
|
cls.common_download,timeout, proxy_type, proxy))
|
||||||
count += 1
|
|
||||||
concurrent.futures.wait(future_list, timeout)
|
concurrent.futures.wait(future_list, timeout)
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def download_comic_icon(cls,is_new=ciUtils.IS_NEW_ICON):
|
|
||||||
icon_url = Comic.getIcon()
|
|
||||||
if icon_url == None:
|
|
||||||
print("icon 不存在,已跳过")
|
|
||||||
return None
|
|
||||||
icon_suffix = str(icon_url).split(".")[-1]
|
|
||||||
#判断漫画名路径是否已存在comicname/cover.jpg, 存在跳过
|
|
||||||
path_comic_icon = ComicPath.getPathConfComicIcon(suffix=icon_suffix)
|
|
||||||
if not ciUtils.equIcon() and fu.exists(path_comic_icon): os.remove(path_comic_icon)
|
|
||||||
while fu.notExists(path_comic_icon):
|
|
||||||
cls.down_queue.put([ComicPath.COMIC_ICON_NAME+"."+icon_suffix,icon_url,ComicPath.getDirConfComic()])
|
|
||||||
cls.download_image()
|
|
||||||
save_path = ComicPath.getPathCBZComicChapterIcon(icon_suffix)
|
|
||||||
if is_new:
|
|
||||||
#历史版本ICON
|
|
||||||
if os.path.exists(save_path): os.remove(save_path)
|
|
||||||
if os.path.exists(path_comic_icon):
|
|
||||||
base_dir = ComicPath.getDirComicChapter()
|
|
||||||
if not os.path.exists(base_dir): os.makedirs(base_dir)
|
|
||||||
shutil.copy(path_comic_icon,os.path.join(base_dir,ComicPath.COMIC_ICON_NAME+icon_suffix))
|
|
||||||
else:
|
|
||||||
if fu.notExists(ComicPath.getDirCBZComic()): os.makedirs(ComicPath.getDirCBZComic())
|
|
||||||
if fu.notExists(save_path):
|
|
||||||
shutil.copy(path_comic_icon,save_path)
|
|
||||||
print(f"{path_comic_icon} 已复制至: {save_path}")
|
|
||||||
#保存icon信息
|
|
||||||
ciUtils.iconDB()
|
|
||||||
ciUtils.setProgressCBZ()
|
|
||||||
Loading…
Reference in New Issue
Block a user