cwx/PyComicPackRouMan
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
cwx
2023-04-06 15:34:14 +08:00
parent a8f44ec334
commit 6ca7ce1992
11 changed files with 205 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils/ComicUtils.py
+5 -5
View File
@@ -2,14 +2,13 @@ import os,shutil,time,requests
from datetime import datetime
from pathlib import Path
from zipfile import ZipFile
from queue import Queue
from common.ComicInfo import ComicInfoUtils as ciUtils
from common.ComicInfo import ComicInfo as ci
from common.ComicInfo import Comic
from utils.FileUtils import fileUtils as fu
from common.Constant import pathStr
from common.Constant import ComicPath
from utils.OldUtils import OldUtils
class ntfy:
@classmethod
def sendMsg(cls, msg,alert=False,sleep=None,error=None):
@@ -121,7 +120,7 @@ class CBZUtils:
with ZipFile(path, "r") as zip_file:
result = zip_file.namelist()
if filter:
filter_icon = ComicPath.COMIC_ICON_NAME+".jpg"
filter_icon = ComicPath.COMIC_ICON_FILE_NAME
filter_info_xml = ComicPath.COMIC_INFO_XML
if filter_icon in result: result.remove(filter_icon)
if filter_info_xml in result: result.remove(filter_info_xml)
@@ -155,7 +154,8 @@ class CBZUtils:
return False
@classmethod
def updateOldCBZ(cls,filesname,result=False):
def updateOldCBZ(cls,filesname=None,result=False):
if filesname == None: filesname = Comic.getChapterFilesName()
old_zipfile_path = ComicPath.getPathOldCBZComicChapter()
#判断是否存在已下载CBZ文件
utils/FileUtils.py
+9 -9
View File
@@ -3,7 +3,7 @@ import math,time,json
from PIL import Image
from tinydb import TinyDB, Query
from common.Constant import ComicPath
from common.Comic import Comic
from utils.Logger import logger
class imageUtils:
@@ -226,15 +226,15 @@ class fileUtils:
img = Image.open(file_path)
img.verify()
img.close()
print(f"{file_path} 类型为type:{type} size: {file_size_unit}")
logger.debug(f"{file_path} 类型为type:{type} size: {file_size_unit}")
return True
except:
if os.path.exists(file_path):
os.remove(file_path)
print(f"{file_path}已损坏 type:{type} size: {file_size_unit}，删除重试中")
logger.warning(f"{file_path}已损坏 type:{type} size: {file_size_unit}，删除重试中")
return False
else:
print(f"文件小于100b删除中,{file_path} size: {file_size_unit}")
logger.warning(f"文件小于100b删除中,{file_path} size: {file_size_unit}")
try:
os.remove(file_path)
except Exception as e:
@@ -263,15 +263,15 @@ class fileUtils:
if not os.path.exists(path):
return None
try:
print(f"{path} 删除中...")
logger.debug(f"{path} 删除中...")
if os.path.isfile(path):
os.remove(path)
if os.path.isdir(path):
shutil.rmtree(path)
print(f"已删除：{path}")
logger.debug(f"已删除：{path}")
return True
except:
print(f"删除错误：{path}")
logger.warning(f"删除错误：{path}")
return False
#文件保存
@@ -298,7 +298,7 @@ class fileUtils:
result = path + "文件写入成功"
except:
result = path + "文件写入失败"
print(result)
logger.debug(result)
return result
class dbUtils:
@@ -317,7 +317,7 @@ class dbUtils:
else: db.update({"progress":progress},comic.name== name)
msg = "失败"
if cls.query(name,progress,db_name): msg = "成功"
print(f"设置{msg}, name={name} value={progress} db={db_name}")
logger.info(f"设置{msg}, name={name} value={progress} db={db_name}")
@classmethod
def query(cls,name,progress=None,db_name=None):
utils/Logger.py
+18
View File
@@ -0,0 +1,18 @@
# -*- coding:utf-8 -*-
import logging
import logging.config
import yaml
class logger:
with open("logconf.yml", "r",encoding="utf-8") as f:
dict_conf = yaml.safe_load(f)
logging.config.dictConfig(dict_conf)
root = logging.getLogger()
@classmethod
def debug(cls,value): cls.root.debug(value)
@classmethod
def info(cls,value): cls.root.info(value)
@classmethod
def warning(cls,value): cls.root.warning(value)
@classmethod
def error(cls,value): cls.root.error(value)
utils/NetUtils.py
+61 -82
View File
@@ -12,7 +12,7 @@ from common.ComicInfo import ComicInfo as ci
from common.ComicInfo import Comic
from common.Constant import ComicPath
from utils.FileUtils import fileUtils as fu
from utils.Logger import logger
class htmlUtils:
@@ -134,9 +134,9 @@ class htmlUtils:
if os.path.exists(html_cache_path):
try:
os.remove(html_cache_path)
print(f"html_cache更新成功 {html_cache_path}")
logger.info(f"html_cache更新成功 {html_cache_path}")
except:
print(f"html_cache更新失败 {html_cache_path}")
logger.info(f"html_cache更新失败 {html_cache_path}")
#获取html实体数据
et = cls.getHTML(url)
if et == None:
@@ -156,6 +156,9 @@ class htmlUtils:
return result
class downloadUtils:
QUEUE_DOWN = Queue()
TYPE_IMG = "image"
TYPE_ICON = "icon"
headers = {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Proxy-Connection": "keep-alive",
@@ -164,56 +167,66 @@ class downloadUtils:
"Accept-Encoding": "gzip, deflate, sdch",
# 'Connection': 'close',
}
down_queue = Queue()
@classmethod
def queueDownClear(cls): cls.QUEUE_DOWN = Queue()
@classmethod
def putDownUrlDirFileType(cls,url,dir,file,type):
cls.QUEUE_DOWN.put([url,dir,file,type])
@classmethod
def getDownUrlDirFileType(cls):
if not cls.QUEUE_DOWN.empty(): return cls.QUEUE_DOWN.get(False)
else: return None
@classmethod
def putDownImageUrlDirFile(cls,url,dir,file): cls.putDownUrlDirFileType(url,dir,file,cls.TYPE_IMG)
@classmethod
def common_download(cls,file_name,image_url,dst_dir,timeout=10,proxy=None,proxy_type=None):
def common_download(cls,repair_max=10,timeout=10,proxy=None,proxy_type=None):
result = cls.getDownUrlDirFileType()
if result == None: return None
(file_url,dir,file,file_type) = [result[0],result[1],result[2],result[3]]
if file_url == None:
logger.error("common_down file_url 为空")
raise NameError("common_down file_url为空")
proxies = None
if proxy_type is not None:
proxies = {
"http": proxy_type + "://" + proxy,
"https": proxy_type + "://" + proxy }
response = None
file_path = os.path.join(dst_dir, file_name)
if os.path.exists(file_path):
print("download_image 文件已存在，已跳过=",file_path)
return None
temp_path = os.path.join(dst_dir, file_name+".downloads")
save_path = os.path.join(dir,file)
logger.debug(f"save_path {save_path}")
if not os.path.exists(dir): os.makedirs(dir)
temp_path = save_path+".downloads"
repair_count = 1
try:
response = requests.get(
image_url, headers=cls.headers, timeout=timeout, proxies=proxies)
if response.status_code != 200 and repair_count <= 5:
raise NameError("下载异常")
with open(temp_path, 'wb') as f:
f.write(response.content)
time.sleep(0.7)
response.close()
#验证是否是图像
if fu.ver_file(temp_path,type="image"):
shutil.move(temp_path, file_path)
print("## OK: {} {}".format(file_path, image_url))
else:
raise NameError("## Fail: {} {}".format(image_url, "图像损坏"))
except Exception as e:
print(f"common_down() 出错了 {e}")
print(f'重试：第{repair_count}次 {image_url}')
cls.down_queue.put([file_name,image_url,dst_dir])
repair_count += 1
while not os.path.exists(save_path):
try:
response = requests.get(
file_url, headers=cls.headers, timeout=timeout, proxies=proxies)
if response.status_code != 200 and repair_count <= repair_max:
logger.warning("下载异常")
raise NameError("下载异常")
with open(temp_path, 'wb') as f:
f.write(response.content)
time.sleep(0.7)
response.close()
#验证是否是图像
if fu.ver_file(temp_path,type=file_type):
shutil.move(temp_path, save_path)
logger.info("## OK: {} {}".format(save_path, file_url))
else:
logger.warning("## Fail: {} {}".format(file_url, "图像损坏"))
raise NameError("## Fail: {} {}".format(file_url, "图像损坏"))
except Exception as e:
logger.warning(f'重试：第{repair_count}次 异常:{e} {file_url}')
cls.putDownUrlDirFileType(file_url,dir,file,file_type)
repair_count += 1
@classmethod
def download_image(cls,timeout=20, proxy_type=None, proxy=None,type="image"):
repeat = 1
while not cls.down_queue.empty() and repeat <= 10:
data = cls.down_queue.get(False)
(file_name,image_url,dst_dir) = [data[0],data[1],data[2]]
cls.common_download(file_name,image_url,dst_dir,timeout=timeout,
proxy=proxy,proxy_type=proxy_type)
repeat += 1
@classmethod
def download_images(cls,image_urls, dst_dir,concurrency=None,timeout=20,proxy_type=None, proxy=None,files_name=None):
def start_downloads(cls,repair_max=20,concurrency=None,timeout=20,proxy_type=None, proxy=None):
"""
Download image according to given urls and automatically rename them in order.
:param timeout:
@@ -226,46 +239,12 @@ class downloadUtils:
:return: none
"""
if concurrency == None:
concurrency = len(image_urls)
concurrency = cls.QUEUE_DOWN.qsize()
logger.debug(f"concurrency= {concurrency}")
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor:
future_list = list()
count = 0
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
for image_url in image_urls:
file_name = files_name[count]
cls.down_queue.put([file_name,image_url,dst_dir])
while not cls.QUEUE_DOWN.empty():
logger.debug("QUEUE_DOWN 不为空 准备下载中...")
future_list.append(executor.submit(
cls.download_image,timeout, proxy_type, proxy))
count += 1
concurrent.futures.wait(future_list, timeout)
@classmethod
def download_comic_icon(cls,is_new=ciUtils.IS_NEW_ICON):
icon_url = Comic.getIcon()
if icon_url == None:
print("icon 不存在，已跳过")
return None
icon_suffix = str(icon_url).split(".")[-1]
#判断漫画名路径是否已存在comicname/cover.jpg, 存在跳过
path_comic_icon = ComicPath.getPathConfComicIcon(suffix=icon_suffix)
if not ciUtils.equIcon() and fu.exists(path_comic_icon): os.remove(path_comic_icon)
while fu.notExists(path_comic_icon):
cls.down_queue.put([ComicPath.COMIC_ICON_NAME+"."+icon_suffix,icon_url,ComicPath.getDirConfComic()])
cls.download_image()
save_path = ComicPath.getPathCBZComicChapterIcon(icon_suffix)
if is_new:
#历史版本ICON
if os.path.exists(save_path): os.remove(save_path)
if os.path.exists(path_comic_icon):
base_dir = ComicPath.getDirComicChapter()
if not os.path.exists(base_dir): os.makedirs(base_dir)
shutil.copy(path_comic_icon,os.path.join(base_dir,ComicPath.COMIC_ICON_NAME+icon_suffix))
else:
if fu.notExists(ComicPath.getDirCBZComic()): os.makedirs(ComicPath.getDirCBZComic())
if fu.notExists(save_path):
shutil.copy(path_comic_icon,save_path)
print(f"{path_comic_icon} 已复制至: {save_path}")
#保存icon信息
ciUtils.iconDB()
ciUtils.setProgressCBZ()
cls.common_download,timeout, proxy_type, proxy))
concurrent.futures.wait(future_list, timeout)