update version
This commit is contained in:
cwx committed
1 parent
57a0ae6841
commit
c62151bc71
25 files changed
+1271
-1408
No files matched your search
@@ -1,162 +0,0 @@
|
||||
import json
|
||||
import os,shutil,time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from zipfile import ZipFile
|
||||
from utils.comic.ComicInfo import comicInfo
|
||||
from utils.Ntfy import ntfy
|
||||
from utils.FileUtils import fileUtils as fu
|
||||
from utils.comic.PathStr import pathStr
|
||||
from utils.OldUtils import OldUtils
|
||||
|
||||
class CBZUtils:
|
||||
|
||||
@classmethod
|
||||
def getCBZ_Dir(cls): return comicInfo.getNewCBZComicChapter()
|
||||
|
||||
@classmethod
|
||||
def getCBZ_Path(cls): return comicInfo.getNewFileCBZComicChapter()
|
||||
|
||||
@classmethod
|
||||
def readDirsOrFiles(cls,dir,type):
|
||||
data = []
|
||||
files = os.listdir(dir)
|
||||
for file in files:
|
||||
path = os.path.join(dir,file)
|
||||
if type == "files" and os.path.isfile(path):
|
||||
data.append(path)
|
||||
if type == "dirs" and os.path.isdir(path):
|
||||
data.append(path)
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
def zip_compression(cls,source_dir=None, target_file=None, remove=True):
|
||||
target_dir = os.path.dirname(target_file)
|
||||
if not os.path.exists(target_dir):
|
||||
os.makedirs(target_dir)
|
||||
if not os.path.exists(target_file) and source_dir != None:
|
||||
with ZipFile(target_file, mode='w') as zf:
|
||||
for path, dir_names, filenames in os.walk(source_dir):
|
||||
path = Path(path)
|
||||
arc_dir = path.relative_to(source_dir)
|
||||
y = 0
|
||||
for filename in filenames:
|
||||
y = y + 1
|
||||
print("打包中：" + str(y) + "/" + str(len(filenames)), os.path.join(source_dir, filename))
|
||||
zf.write(path.joinpath(filename), arc_dir.joinpath(filename))
|
||||
zf.close()
|
||||
ntfy.sendMsg(f"打包完成：{target_file}")
|
||||
cls.verCBZComic(target_file)
|
||||
|
||||
@classmethod
|
||||
def packAutoComicChapterCBZ(cls):
|
||||
chapter_path = comicInfo.getDirComicChapter()
|
||||
if os.path.exists(chapter_path):
|
||||
dirs = os.listdir(chapter_path)
|
||||
for file in dirs:
|
||||
if file.startswith("scramble="):
|
||||
try:
|
||||
os.remove(file)
|
||||
except:
|
||||
print(f"删除 {file} 发生错误，已跳过")
|
||||
return False
|
||||
cls.zip_compression(comicInfo.getDirComicChapter(), cls.getCBZ_Path())
|
||||
time.sleep(0.1)
|
||||
fu.remove(comicInfo.getDirComicChapter())
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def replaceZip(cls,filepath,unpack_dir=None):
|
||||
if not cls.compareFileDate(filepath): return None
|
||||
if unpack_dir == None:
|
||||
unpack_dir = str(filepath).split(".")[0]
|
||||
fz = ZipFile(filepath, 'r')
|
||||
for file in fz.namelist():
|
||||
if file.endswith(".jpg"):
|
||||
data = fz.read(file)
|
||||
if len(data) < 500 and os.path.exists(filepath):
|
||||
os.remove(filepath)
|
||||
print(f"数据不完整,已删除:{filepath}")
|
||||
if cls.compareFileDate(filepath):
|
||||
os.utime(filepath)
|
||||
print(f"已更新文件时间 {filepath}")
|
||||
if os.path.exists(unpack_dir):
|
||||
shutil.rmtree(unpack_dir)
|
||||
|
||||
@classmethod
|
||||
def compareFileDate(cls,filepath):
|
||||
if os.path.exists(filepath):
|
||||
ctime = os.path.getmtime(filepath)
|
||||
str_ctime = datetime.fromtimestamp(int(ctime))
|
||||
file_ctime = str(str_ctime.year)+"{:0>2d}".format(str_ctime.month)+"{:0>2d}".format(str_ctime.day)+"{:0>2d}".format(str_ctime.hour)
|
||||
c_ctime = 2023011603
|
||||
else:
|
||||
return False
|
||||
if int(file_ctime) < c_ctime:
|
||||
return True
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def zip_info(cls,path,filter=True):
|
||||
result = None
|
||||
try:
|
||||
with ZipFile(path, "r") as zip_file:
|
||||
result = zip_file.namelist()
|
||||
if filter:
|
||||
filter_icon = comicInfo.COMIC_ICON_NAME+".jpg"
|
||||
filter_info_xml = comicInfo.COMIC_INFO_XML
|
||||
if filter_icon in result: result.remove(filter_icon)
|
||||
if filter_info_xml in result: result.remove(filter_info_xml)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return result
|
||||
|
||||
#CBZ检验是否完整
|
||||
@classmethod
|
||||
def verCBZComic(cls,path=None,list_img=None,min_size=300000):
|
||||
#数据检验
|
||||
if path == None: path = cls.getCBZ_Path()
|
||||
#文件不存在 则返回
|
||||
if fu.notExists(path): return False
|
||||
if list_img == None: list_img = comicInfo.getChapterImgs()
|
||||
|
||||
if fu.exists(path) and len(cls.zip_info(path)) == len(list_img):
|
||||
print(f"文件校验成功：{path}")
|
||||
comicInfo.setProgress(comicInfo.PROGRESS_DONE)
|
||||
return True
|
||||
else:
|
||||
try:
|
||||
if len(cls.zip_info(path)) < len(list_img) or os.path.getsize(path) < min_size:
|
||||
fu.remove(path)
|
||||
comicInfo.setProgress(comicInfo.PROGRESS_NONE)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def updateOldCBZ(cls,filesname,result=False):
|
||||
old_zipfile_path = os.path.join(pathStr.old_cbz_path,OldUtils.getOldComicName(),OldUtils.getOldChapter()+".CBZ")
|
||||
#判断是否存在已下载CBZ文件
|
||||
if fu.exists(old_zipfile_path) and fu.notExists(CBZUtils.getCBZ_Path()):
|
||||
print(f"存在CBZ文件{old_zipfile_path},解压中...")
|
||||
zip_file = ZipFile(old_zipfile_path)
|
||||
#CBZ中文件数量，剔除ComicInfo.xml
|
||||
if len(filesname) == len(zip_file.namelist())-1:
|
||||
unzip_path = comicInfo.getDirComicChapter()
|
||||
zip_file.extractall(unzip_path)
|
||||
zip_file.close()
|
||||
print(f"解压完成: CBZ文件{old_zipfile_path}")
|
||||
print("文件校验中...")
|
||||
for file in os.listdir(unzip_path):
|
||||
#检验图片损坏则删除
|
||||
if file.endswith(".jpg") and not fu.ver_file(os.path.join(unzip_path,file),type="image"):
|
||||
fu.remove(unzip_path)
|
||||
return False
|
||||
comicInfo.writeComicInfoXML(overlay=True)
|
||||
result = True
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def nextCBZ(cls,list_img=None):
|
||||
if list_img == None: list_img = comicInfo.getChapterImgs()
|
||||
return not cls.verCBZComic(list_img=list_img)
|
||||
+180
-5
@@ -1,6 +1,181 @@
|
||||
from opencc import OpenCC
|
||||
|
||||
class fontUtils:
|
||||
import os,shutil,time,requests
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from zipfile import ZipFile
|
||||
from common.ComicInfo import ComicInfoUtils as ciUtils
|
||||
from common.ComicInfo import ComicInfo as ci
|
||||
from common.ComicInfo import Comic
|
||||
from utils.FileUtils import fileUtils as fu
|
||||
from common.Constant import pathStr
|
||||
from common.Constant import ComicPath
|
||||
from utils.OldUtils import OldUtils
|
||||
|
||||
class ntfy:
|
||||
@classmethod
|
||||
def ChineseConvert(cls, text,convert='t2s'):
|
||||
return OpenCC(convert).convert(text) # convert from Simplified Chinese to Traditional Chinese
|
||||
def sendMsg(cls, msg,alert=True,sleep=None,error=None):
|
||||
try:
|
||||
print(f"#ntfy: {msg}")
|
||||
if alert:
|
||||
requests.post("https://ntfy.caiwenxiu.cn/PyComic",
|
||||
data=msg.encode(encoding='utf-8'))
|
||||
except:
|
||||
print(f"#ntfy error: {msg}")
|
||||
if sleep != None:
|
||||
time.sleep(int(sleep))
|
||||
if error != None:
|
||||
print(f"#ntfy Error: {error}")
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
class CBZUtils:
|
||||
|
||||
@classmethod
|
||||
def getCBZ_Dir(cls): return ComicPath.getNewCBZComicChapter()
|
||||
|
||||
@classmethod
|
||||
def getCBZ_Path(cls): return ComicPath.getNewFileCBZComicChapter()
|
||||
|
||||
@classmethod
|
||||
def readDirsOrFiles(cls,dir,type):
|
||||
data = []
|
||||
files = os.listdir(dir)
|
||||
for file in files:
|
||||
path = os.path.join(dir,file)
|
||||
if type == "files" and os.path.isfile(path):
|
||||
data.append(path)
|
||||
if type == "dirs" and os.path.isdir(path):
|
||||
data.append(path)
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
def zip_compression(cls,source_dir=None, target_file=None, remove=True):
|
||||
target_dir = os.path.dirname(target_file)
|
||||
if not os.path.exists(target_dir):
|
||||
os.makedirs(target_dir)
|
||||
if not os.path.exists(target_file) and source_dir != None:
|
||||
with ZipFile(target_file, mode='w') as zf:
|
||||
for path, dir_names, filenames in os.walk(source_dir):
|
||||
path = Path(path)
|
||||
arc_dir = path.relative_to(source_dir)
|
||||
y = 0
|
||||
for filename in filenames:
|
||||
y = y + 1
|
||||
print("打包中：" + str(y) + "/" + str(len(filenames)), os.path.join(source_dir, filename))
|
||||
zf.write(path.joinpath(filename), arc_dir.joinpath(filename))
|
||||
zf.close()
|
||||
ntfy.sendMsg(f"打包完成：{target_file}")
|
||||
cls.verCBZComic(target_file)
|
||||
|
||||
@classmethod
|
||||
def packAutoComicChapterCBZ(cls):
|
||||
chapter_path = ComicPath.getDirComicChapter()
|
||||
if os.path.exists(chapter_path):
|
||||
dirs = os.listdir(chapter_path)
|
||||
for file in dirs:
|
||||
if file.startswith("scramble="):
|
||||
try:
|
||||
os.remove(file)
|
||||
except:
|
||||
print(f"删除 {file} 发生错误，已跳过")
|
||||
return False
|
||||
cls.zip_compression(ComicPath.getDirComicChapter(), cls.getCBZ_Path())
|
||||
time.sleep(0.1)
|
||||
fu.remove(ComicPath.getDirComicChapter())
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def replaceZip(cls,filepath,unpack_dir=None):
|
||||
if not cls.compareFileDate(filepath): return None
|
||||
if unpack_dir == None:
|
||||
unpack_dir = str(filepath).split(".")[0]
|
||||
fz = ZipFile(filepath, 'r')
|
||||
for file in fz.namelist():
|
||||
if file.endswith(".jpg"):
|
||||
data = fz.read(file)
|
||||
if len(data) < 500 and os.path.exists(filepath):
|
||||
os.remove(filepath)
|
||||
print(f"数据不完整,已删除:{filepath}")
|
||||
if cls.compareFileDate(filepath):
|
||||
os.utime(filepath)
|
||||
print(f"已更新文件时间 {filepath}")
|
||||
if os.path.exists(unpack_dir):
|
||||
shutil.rmtree(unpack_dir)
|
||||
|
||||
@classmethod
|
||||
def compareFileDate(cls,filepath):
|
||||
if os.path.exists(filepath):
|
||||
ctime = os.path.getmtime(filepath)
|
||||
str_ctime = datetime.fromtimestamp(int(ctime))
|
||||
file_ctime = str(str_ctime.year)+"{:0>2d}".format(str_ctime.month)+"{:0>2d}".format(str_ctime.day)+"{:0>2d}".format(str_ctime.hour)
|
||||
c_ctime = 2023011603
|
||||
else:
|
||||
return False
|
||||
if int(file_ctime) < c_ctime:
|
||||
return True
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def zip_info(cls,path,filter=True):
|
||||
result = None
|
||||
try:
|
||||
with ZipFile(path, "r") as zip_file:
|
||||
result = zip_file.namelist()
|
||||
if filter:
|
||||
filter_icon = ci.COMIC_ICON_NAME+".jpg"
|
||||
filter_info_xml = ci.COMIC_INFO_XML
|
||||
if filter_icon in result: result.remove(filter_icon)
|
||||
if filter_info_xml in result: result.remove(filter_info_xml)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return result
|
||||
|
||||
#CBZ检验是否完整
|
||||
@classmethod
|
||||
def verCBZComic(cls,path=None,list_img=None,min_size=300000):
|
||||
#数据检验
|
||||
if path == None: path = cls.getCBZ_Path()
|
||||
#文件不存在 则返回
|
||||
if fu.notExists(path): return False
|
||||
if list_img == None: list_img = Comic.getChapterImgs()
|
||||
|
||||
if fu.exists(path) and len(cls.zip_info(path)) == len(list_img):
|
||||
print(f"文件校验成功：{path}")
|
||||
ciUtils.setProgress(ciUtils.PROGRESS_DONE)
|
||||
return True
|
||||
else:
|
||||
try:
|
||||
if len(cls.zip_info(path)) < len(list_img) or os.path.getsize(path) < min_size:
|
||||
fu.remove(path)
|
||||
ciUtils.setProgress(ciUtils.PROGRESS_NONE)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def updateOldCBZ(cls,filesname,result=False):
|
||||
old_zipfile_path = ComicPath.setJoinPathDir([OldUtils.getOldComicName(),OldUtils.getOldChapter()],
|
||||
pathStr.old_cbz_path,prefix="CBZ")
|
||||
#判断是否存在已下载CBZ文件
|
||||
if fu.exists(old_zipfile_path) and fu.notExists(CBZUtils.getCBZ_Path()):
|
||||
print(f"存在CBZ文件{old_zipfile_path},解压中...")
|
||||
zip_file = ZipFile(old_zipfile_path)
|
||||
#CBZ中文件数量，剔除ComicInfo.xml
|
||||
if len(filesname) == len(zip_file.namelist())-1:
|
||||
unzip_path = ComicPath.getDirComicChapter()
|
||||
zip_file.extractall(unzip_path)
|
||||
zip_file.close()
|
||||
print(f"解压完成: CBZ文件{old_zipfile_path}")
|
||||
print("文件校验中...")
|
||||
for file in os.listdir(unzip_path):
|
||||
#检验图片损坏则删除
|
||||
if file.endswith(".jpg") and not fu.ver_file(os.path.join(unzip_path,file),type="image"):
|
||||
fu.remove(unzip_path)
|
||||
return False
|
||||
ci.writeComicInfoXML(overlay=True)
|
||||
result = True
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def nextCBZ(cls,list_img=None):
|
||||
if list_img == None: list_img = Comic.getChapterImgs()
|
||||
return not cls.verCBZComic(list_img=list_img)
|
||||
+29
-3
@@ -1,9 +1,8 @@
|
||||
import base64,hashlib,os,shutil
|
||||
import math,time
|
||||
import numpy as np
|
||||
import math,time,json
|
||||
from PIL import Image
|
||||
from tinydb import TinyDB, Query
|
||||
from utils.comic.PathStr import pathStr
|
||||
from common.Constant import pathStr
|
||||
|
||||
class imageUtils:
|
||||
|
||||
@@ -273,7 +272,34 @@ class fileUtils:
|
||||
except:
|
||||
print(f"删除错误：{path}")
|
||||
return False
|
||||
|
||||
#文件保存
|
||||
@classmethod
|
||||
def file_save(cls,path,data,mode=None,print_msg=False):
|
||||
result = {}
|
||||
f = {}
|
||||
dir_name = os.path.dirname(path)
|
||||
if not os.path.exists(dir_name):
|
||||
os.makedirs(dir_name)
|
||||
|
||||
save_path = os.path.join(path)
|
||||
if os.path.exists(save_path):
|
||||
os.remove(save_path)
|
||||
data = json.dumps(data)
|
||||
if mode == None:
|
||||
mode = "w+"
|
||||
try:
|
||||
f = open(save_path, mode, encoding="utf-8")
|
||||
f.write(data)
|
||||
f.close()
|
||||
if print_msg:
|
||||
print("data=",data)
|
||||
result = path + "文件写入成功"
|
||||
except:
|
||||
result = path + "文件写入失败"
|
||||
print(result)
|
||||
return result
|
||||
|
||||
class dbUtils:
|
||||
@classmethod
|
||||
def base_path(cls,path):
|
||||
|
||||
@@ -1,123 +0,0 @@
|
||||
from fake_useragent import UserAgent
|
||||
import requests,os,json
|
||||
from lxml import html
|
||||
import traceback
|
||||
import time,re
|
||||
from urllib3.util.retry import Retry
|
||||
from requests.adapters import HTTPAdapter
|
||||
from utils.Ntfy import ntfy
|
||||
from utils.comic.PathStr import pathStr
|
||||
|
||||
class htmlUtils:
|
||||
|
||||
headers = {'User-Agent': UserAgent().random}
|
||||
url_data = {}
|
||||
|
||||
@classmethod
|
||||
def getPathSaveHtml(cls,url,type=None):
|
||||
rstr = r"[\/\\\:\*\?\"\<\>\|\.]" # '/ \ : * ? " < > |'
|
||||
try:
|
||||
file_url = re.sub(rstr, "", url)
|
||||
except:
|
||||
file_url = "error_cache"
|
||||
file_path = os.path.join(pathStr.base_html_cache(),file_url)
|
||||
if type == "new":
|
||||
return file_path
|
||||
if os.path.exists(file_path):
|
||||
if type == "read":
|
||||
with open(file_path,"r",encoding="utf-8") as fs: return fs.read()
|
||||
return file_path
|
||||
else:
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def saveHtml(cls,url,data,type=None):
|
||||
file_path = cls.getPathSaveHtml(url,type="new")
|
||||
dir_name = os.path.dirname(file_path)
|
||||
if not os.path.exists(dir_name):
|
||||
os.makedirs(dir_name)
|
||||
with open(file_path,"w",encoding="utf-8") as fs:
|
||||
if type== "json": data = json.dumps(data)
|
||||
fs.write(str(data))
|
||||
|
||||
@classmethod
|
||||
def remove_HtmlCache(cls,url):
|
||||
file_path = cls.getPathSaveHtml(url,type="new")
|
||||
if os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
print("已删除")
|
||||
except:
|
||||
print()
|
||||
|
||||
@classmethod
|
||||
def getHTML(cls, curl,type=None,update=False):
|
||||
url_text = None
|
||||
if update: cls.remove_HtmlCache(curl)
|
||||
retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[ 500, 502, 503, 504 ])
|
||||
s = requests.Session()
|
||||
s.mount('http://', HTTPAdapter(max_retries=retries))
|
||||
s.mount('https://', HTTPAdapter(max_retries=retries))
|
||||
#数据为空则获取数据
|
||||
try: url_text = cls.getPathSaveHtml(curl,"read")
|
||||
except: url_text = None
|
||||
if url_text != None and update == False: return html.fromstring(url_text)
|
||||
else: url_text = None
|
||||
repeat = 0
|
||||
while url_text == None and repeat <=5:
|
||||
try:
|
||||
print(f"请求地址：{curl}")
|
||||
res = s.get(curl,stream=True, headers=cls.headers, timeout=5,allow_redirects=True)
|
||||
if type == "bytes":
|
||||
url_text = res
|
||||
if type == "json":
|
||||
cls.saveHtml(curl,res.text,type="json")
|
||||
return json.loads(res.text)
|
||||
if type == None:
|
||||
url_text = html.fromstring(res.text)
|
||||
cls.saveHtml(curl,res.text)
|
||||
except:
|
||||
repeat += 1
|
||||
ntfy.sendMsg(f"请求失败：{curl}",sleep=1)
|
||||
return url_text
|
||||
|
||||
@classmethod
|
||||
def getBytes(cls, url):
|
||||
return cls.getHTML(url,type="bytes")
|
||||
|
||||
@classmethod
|
||||
def getJSON(cls,url,update=False):
|
||||
return cls.getHTML(url,type="json",update=update)
|
||||
|
||||
@classmethod
|
||||
def xpathData(cls,c_xpath,url=None,num=None,not_eq=None,update=False):
|
||||
if url == None:
|
||||
url = cls.temp_url
|
||||
else:
|
||||
cls.temp_url = url
|
||||
result = []
|
||||
if update:
|
||||
html_cache_path = cls.getPathSaveHtml(url,"new")
|
||||
if os.path.exists(html_cache_path):
|
||||
try:
|
||||
os.remove(html_cache_path)
|
||||
ntfy.sendMsg(f"html_cache更新成功 {html_cache_path}")
|
||||
except:
|
||||
ntfy.sendMsg(f"html_cache更新失败 {html_cache_path}")
|
||||
#获取html实体数据
|
||||
et = cls.getHTML(url)
|
||||
if et == None:
|
||||
return None
|
||||
#比对数据
|
||||
count = 1
|
||||
xpaths = et.xpath(c_xpath)
|
||||
for x in xpaths:
|
||||
if x != not_eq:
|
||||
result.append(x)
|
||||
count +=1
|
||||
if num != None:
|
||||
try:
|
||||
result = result[num]
|
||||
except:
|
||||
result = None
|
||||
return result
|
||||
@@ -0,0 +1,278 @@
|
||||
from __future__ import print_function
|
||||
from queue import Queue
|
||||
from fake_useragent import UserAgent
|
||||
import shutil,imghdr,concurrent.futures
|
||||
import requests,os,json,time,re
|
||||
from lxml import html
|
||||
from urllib3.util.retry import Retry
|
||||
from requests.adapters import HTTPAdapter
|
||||
from common.Constant import pathStr
|
||||
from common.ComicInfo import ComicInfoUtils as ciUtils
|
||||
from common.ComicInfo import ComicInfo as ci
|
||||
from common.ComicInfo import Comic
|
||||
from common.Constant import ComicPath
|
||||
from utils.FileUtils import fileUtils as fu
|
||||
|
||||
|
||||
class htmlUtils:
|
||||
|
||||
headers = {'User-Agent': UserAgent().random}
|
||||
url_data = {}
|
||||
#domain
|
||||
@classmethod
|
||||
def parseExec(cls,data,exec):
|
||||
if data !=None and exec != None:
|
||||
dots = str(exec).split(".")
|
||||
if not isinstance(data,dict): data = json.loads(data)
|
||||
for dot in dots:
|
||||
data = data.get(dot)
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
def getXpathData(cls,c_xpath,url=None,num=None,not_eq=None,update=False):
|
||||
return htmlUtils.xpathData(c_xpath=c_xpath,url=url,num=num,not_eq=not_eq,update=update)
|
||||
|
||||
@classmethod
|
||||
def setXpathData(cls,url,xpath,exec,num=None,result_type=None,type=None,start_add=None):
|
||||
result = cls.parseExec(htmlUtils.xpathData(xpath,url=url,num=num),exec)
|
||||
if result == None: return None
|
||||
if result_type == "list" and type != None:
|
||||
data = []
|
||||
for x in range(0, len(result)):
|
||||
if start_add != None:
|
||||
data.append(start_add+result[x].get(type))
|
||||
else:
|
||||
data.append(result[x].get(type))
|
||||
return data
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def getPathSaveHtml(cls,url,type=None):
|
||||
rstr = r"[\/\\\:\*\?\"\<\>\|\.]" # '/ \ : * ? " < > |'
|
||||
try:
|
||||
file_url = re.sub(rstr, "", url)
|
||||
except:
|
||||
file_url = "error_cache"
|
||||
file_path = os.path.join(pathStr.base_html_cache(),file_url)
|
||||
if type == "new":
|
||||
return file_path
|
||||
if os.path.exists(file_path):
|
||||
if type == "read":
|
||||
with open(file_path,"r",encoding="utf-8") as fs: return fs.read()
|
||||
return file_path
|
||||
else:
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def saveHtml(cls,url,data,type=None):
|
||||
file_path = cls.getPathSaveHtml(url,type="new")
|
||||
dir_name = os.path.dirname(file_path)
|
||||
if not os.path.exists(dir_name):
|
||||
os.makedirs(dir_name)
|
||||
with open(file_path,"w",encoding="utf-8") as fs:
|
||||
if type== "json": data = json.dumps(data)
|
||||
fs.write(str(data))
|
||||
|
||||
@classmethod
|
||||
def remove_HtmlCache(cls,url):
|
||||
file_path = cls.getPathSaveHtml(url,type="new")
|
||||
if os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
print("已删除")
|
||||
except:
|
||||
print()
|
||||
|
||||
@classmethod
|
||||
def getHTML(cls, curl,type=None,update=False):
|
||||
url_text = None
|
||||
if update: cls.remove_HtmlCache(curl)
|
||||
retries = Retry(total=1, backoff_factor=0.5, status_forcelist=[ 500, 502, 503, 504 ])
|
||||
s = requests.Session()
|
||||
s.keep_alive = False
|
||||
s.mount('http://', HTTPAdapter(max_retries=retries))
|
||||
s.mount('https://', HTTPAdapter(max_retries=retries))
|
||||
#数据为空则获取数据
|
||||
try: url_text = cls.getPathSaveHtml(curl,"read")
|
||||
except: url_text = None
|
||||
if url_text != None and update == False: return html.fromstring(url_text)
|
||||
else: url_text = None
|
||||
repeat = 0
|
||||
while url_text == None and repeat <=5:
|
||||
try:
|
||||
print(f"请求地址：{curl}")
|
||||
res = s.get(curl,stream=True, headers=cls.headers, timeout=10,allow_redirects=True)
|
||||
if type == "bytes":
|
||||
url_text = res
|
||||
if type == "json":
|
||||
cls.saveHtml(curl,res.text,type="json")
|
||||
return json.loads(res.text)
|
||||
if type == None:
|
||||
url_text = html.fromstring(res.text)
|
||||
cls.saveHtml(curl,res.text)
|
||||
res.close()
|
||||
except Exception as e:
|
||||
repeat += 1
|
||||
print(f"请求失败：Exception: {e} {curl}")
|
||||
return url_text
|
||||
|
||||
@classmethod
|
||||
def getBytes(cls, url):
|
||||
return cls.getHTML(url,type="bytes")
|
||||
|
||||
@classmethod
|
||||
def getJSON(cls,url,update=False):
|
||||
return cls.getHTML(url,type="json",update=update)
|
||||
|
||||
@classmethod
|
||||
def xpathData(cls,c_xpath,url=None,num=None,not_eq=None,update=False):
|
||||
if url == None: url = cls.temp_url
|
||||
else: cls.temp_url = url
|
||||
result = []
|
||||
if update:
|
||||
html_cache_path = cls.getPathSaveHtml(url,"new")
|
||||
if os.path.exists(html_cache_path):
|
||||
try:
|
||||
os.remove(html_cache_path)
|
||||
print(f"html_cache更新成功 {html_cache_path}")
|
||||
except:
|
||||
print(f"html_cache更新失败 {html_cache_path}")
|
||||
#获取html实体数据
|
||||
et = cls.getHTML(url)
|
||||
if et == None:
|
||||
return None
|
||||
#比对数据
|
||||
count = 1
|
||||
xpaths = et.xpath(c_xpath)
|
||||
for x in xpaths:
|
||||
if x != not_eq:
|
||||
result.append(x)
|
||||
count +=1
|
||||
if num != None:
|
||||
try:
|
||||
result = result[num]
|
||||
except:
|
||||
result = None
|
||||
return result
|
||||
|
||||
class downloadUtils:
|
||||
headers = {
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||||
"Proxy-Connection": "keep-alive",
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36",
|
||||
"Accept-Encoding": "gzip, deflate, sdch",
|
||||
# 'Connection': 'close',
|
||||
}
|
||||
down_queue = Queue()
|
||||
|
||||
@classmethod
|
||||
def common_download(cls,file_name,image_url,dst_dir,timeout=10,proxy=None,proxy_type=None):
|
||||
proxies = None
|
||||
if proxy_type is not None:
|
||||
proxies = {
|
||||
"http": proxy_type + "://" + proxy,
|
||||
"https": proxy_type + "://" + proxy }
|
||||
response = None
|
||||
file_path = os.path.join(dst_dir, file_name)
|
||||
if os.path.exists(file_path):
|
||||
print("download_image 文件已存在，已跳过=",file_path)
|
||||
return None
|
||||
temp_path = os.path.join(dst_dir, file_name+".downloads")
|
||||
repair_count = 1
|
||||
response = requests.get(
|
||||
image_url, headers=cls.headers, timeout=timeout, proxies=proxies)
|
||||
while response.status_code != 200 and repair_count <= 5:
|
||||
time.sleep(0.7)
|
||||
cls.download_image(image_url,dst_dir,file_name)
|
||||
print(f'重试：第{repair_count}次 {image_url}')
|
||||
repair_count += 1
|
||||
with open(temp_path, 'wb') as f:
|
||||
f.write(response.content)
|
||||
response.close()
|
||||
#验证是否是图像
|
||||
if fu.ver_file(temp_path,type="image"):
|
||||
shutil.move(temp_path, file_path)
|
||||
print("## OK: {} {}".format(file_path, image_url))
|
||||
else:
|
||||
print("## Fail: {} {}".format(image_url, "图像损坏"))
|
||||
cls.down_queue.put([file_name,image_url,dst_dir])
|
||||
|
||||
@classmethod
|
||||
def download_image(cls,timeout=20, proxy_type=None, proxy=None,type="image"):
|
||||
repeat = 0
|
||||
while not cls.down_queue.empty() and repeat <= 10:
|
||||
repeat += 1
|
||||
data = cls.down_queue.get(False)
|
||||
(file_name,image_url,dst_dir) = [data[0],data[1],data[2]]
|
||||
if repeat > 1:
|
||||
print(f"第{repeat}次下载数据中... file_name={file_name}")
|
||||
try:
|
||||
cls.common_download(file_name,image_url,dst_dir)
|
||||
except:
|
||||
print(f"下载重试中 {file_name}={image_url}")
|
||||
cls.down_queue.put([file_name,image_url,dst_dir])
|
||||
|
||||
|
||||
@classmethod
|
||||
def download_images(cls,image_urls, dst_dir,concurrency=None,timeout=20,proxy_type=None, proxy=None,files_name=None):
|
||||
"""
|
||||
Download image according to given urls and automatically rename them in order.
|
||||
:param timeout:
|
||||
:param proxy:
|
||||
:param proxy_type:
|
||||
:param image_urls: list of image urls
|
||||
:param dst_dir: output the downloaded images to dst_dir
|
||||
:param file_prefix: if set to "img", files will be in format "img_xxx.jpg"
|
||||
:param concurrency: number of requests process simultaneously
|
||||
:return: none
|
||||
"""
|
||||
if concurrency == None:
|
||||
concurrency = len(image_urls)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor:
|
||||
future_list = list()
|
||||
count = 0
|
||||
if not os.path.exists(dst_dir):
|
||||
os.makedirs(dst_dir)
|
||||
for image_url in image_urls:
|
||||
file_name = files_name[count]
|
||||
cls.down_queue.put([file_name,image_url,dst_dir])
|
||||
future_list.append(executor.submit(
|
||||
cls.download_image,timeout, proxy_type, proxy))
|
||||
count += 1
|
||||
concurrent.futures.wait(future_list, timeout)
|
||||
|
||||
@classmethod
|
||||
def download_comic_icon(cls,is_new=ciUtils.IS_NEW_ICON):
|
||||
icon_url = Comic.getIcon()
|
||||
if icon_url == None:
|
||||
print("icon 不存在，已跳过")
|
||||
return None
|
||||
save_name = ci.COMIC_ICON_NAME
|
||||
icon_prefix = "."+str(icon_url).split(".")[-1]
|
||||
icon_prefix = icon_prefix.split("?")[0]
|
||||
#判断漫画名路径是否已存在comicname/cover.jpg, 存在跳过
|
||||
path_comic_icon = os.path.join(ComicPath.getDirConfComic(),save_name+icon_prefix)
|
||||
if not ciUtils.equIcon() and fu.exists(path_comic_icon):
|
||||
os.remove(path_comic_icon)
|
||||
if fu.notExists(path_comic_icon):
|
||||
cls.download_images([icon_url],ComicPath.getDirConfComic(),files_name=[save_name+icon_prefix])
|
||||
#if not os.path.exists(path_cbz_comic):
|
||||
# os.makedirs(path_cbz_comic)
|
||||
save_path = os.path.join(ComicPath.getDirCBZComic(),Comic.getChapterName()+icon_prefix)
|
||||
if is_new:
|
||||
#历史版本ICON
|
||||
if os.path.exists(save_path):
|
||||
os.remove(save_path)
|
||||
if os.path.exists(path_comic_icon):
|
||||
base_dir = ComicPath.getDirComicChapter()
|
||||
if not os.path.exists(base_dir): os.makedirs(base_dir)
|
||||
shutil.copy(path_comic_icon,os.path.join(base_dir,save_name+icon_prefix))
|
||||
else:
|
||||
if fu.notExists(ComicPath.getDirCBZComic()): os.makedirs(ComicPath.getDirCBZComic())
|
||||
shutil.copy(path_comic_icon,save_path)
|
||||
print(f"{path_comic_icon} 已复制至: {save_path}")
|
||||
#保存icon信息
|
||||
ciUtils.iconDB()
|
||||
ciUtils.nextDownloadToCBZChapter()
|
||||
ciUtils.setProgress(ciUtils.PROGRESS_CBZ)
|
||||
@@ -1,19 +0,0 @@
|
||||
import requests,time
|
||||
|
||||
class ntfy:
|
||||
@classmethod
|
||||
def sendMsg(cls, msg,alert=True,sleep=None,error=None):
|
||||
try:
|
||||
print(f"#ntfy: {msg}")
|
||||
if alert:
|
||||
requests.post("https://ntfy.caiwenxiu.cn/PyComic",
|
||||
data=msg.encode(encoding='utf-8'))
|
||||
except:
|
||||
print(f"#ntfy error: {msg}")
|
||||
if sleep != None:
|
||||
time.sleep(int(sleep))
|
||||
if error != None:
|
||||
print(f"#ntfy Error: {error}")
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
@@ -1,133 +0,0 @@
|
||||
import json,os,time,random,shutil
|
||||
from utils.HtmlUtils import htmlUtils
|
||||
from utils.FileUtils import imageUtils
|
||||
from utils.comic.ComicInfo import comicInfo as ci
|
||||
from utils.CBZUtils import CBZUtils
|
||||
from utils.downloader import download_images
|
||||
from utils.downloader import download_comic_icon
|
||||
from utils.Ntfy import ntfy
|
||||
from entity.down.RouMan import comicCommon as RouManComicCommon
|
||||
from entity.down.JM import comicCommon as JMComicCommon
|
||||
from entity.down.BaoZi import comicCommon as BaoZiComicCommon
|
||||
from utils.comic.PathStr import pathStr
|
||||
from utils.FileUtils import fileUtils as fu
|
||||
|
||||
class baseComic:
|
||||
count_chapter = 0
|
||||
|
||||
#校验该漫画是否为最新
|
||||
# Y/跳过 N/下载 返回下载链接
|
||||
@classmethod
|
||||
def updateComics(cls):
|
||||
(book_name,comic_href,updated) = [ci.getComicName(),ci.getCurrentChapterImg(),ci.getUpdateAt()]
|
||||
#白名单跳过
|
||||
if ci.getIsComicNameSkips(book_name): return None
|
||||
if not ci.isUpdateComic():
|
||||
ntfy.sendMsg(f"开始下载 漫画：{book_name}")
|
||||
return comic_href
|
||||
else:
|
||||
ntfy.sendMsg(f"{book_name} 已是最新")
|
||||
chapters = htmlUtils.xpathData('//div[contains(@class,"bookid_chapterBox")]//div[contains(@class,"bookid_chapter")]/a/text()',url=comic_href,update=False)
|
||||
chapter_index = 1
|
||||
for chapter in chapters:
|
||||
ci.setChapterIndex(chapter_index)
|
||||
ci.setChapterName(chapter)
|
||||
cbz_path = ci.getNewCBZComicChapter("file")
|
||||
icon_path = ci.getNewIconComicChapter("file")
|
||||
CBZUtils.replaceZip(cbz_path)
|
||||
#判断漫画是否完成
|
||||
if ci.isProgress(ci.PROGRESS_DONE) and not os.path.exists(cbz_path): ci.isProgress(ci.PROGRESS_DONE,remove=True)
|
||||
if not os.path.exists(cbz_path):
|
||||
ci.updateComicDate("0")
|
||||
return comic_href
|
||||
chapter_index = chapter_index + 1
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def oneComic(cls,url,title,author,icon,tags,dep,chapters,chapter_href,alias=None,genre=None,lang="zh",age_rating=None,sleep=None):
|
||||
ci.setComicInfo(homepage=url,comicname=title,author=author,icon=icon,tags=tags,dep=dep,genre=genre,lang=lang,age_rating=age_rating,chapters=chapters)
|
||||
cls.count_chapter = 0
|
||||
for href in chapter_href:
|
||||
ci.setChapterName(chapters[cls.count_chapter])
|
||||
ci.setChapterIndex(cls.count_chapter+1)
|
||||
#存在完成配置文件 但文件不存在 将清空完成配置文件
|
||||
if ci.isProgress(ci.PROGRESS_DONE) and not fu.exists(ci.getNewCBZComicChapter("file")): ci.isProgress(ci.PROGRESS_DONE,remove=True)
|
||||
#不存在完成配置文件 则允许下载
|
||||
if not ci.isProgress(ci.PROGRESS_DONE): cls.comicChapters(href,scramble=True,sleep=random.randint(1,5))
|
||||
cls.count_chapter += 1
|
||||
#一本漫画下载后等待
|
||||
#清空文件夹
|
||||
if os.path.exists(ci.getDirComic()): shutil.rmtree(ci.getDirComic())
|
||||
if sleep != None: time.sleep(sleep)
|
||||
|
||||
'''
|
||||
|
||||
读取某章节下所有图片
|
||||
'''
|
||||
@classmethod
|
||||
def comicChapters(cls,chapter_url,scramble=None,sleep=None):
|
||||
#try:
|
||||
cls.Onechapter(chapter_url,scramble)
|
||||
#进入下个阶段
|
||||
#章节图片全部下载后，调用下载封面
|
||||
if ci.isProgress(ci.PROGRESS_DOWN): download_comic_icon()
|
||||
#下个阶段
|
||||
if ci.isProgress(ci.PROGRESS_CBZ): is_next = CBZUtils.packAutoComicChapterCBZ()
|
||||
#except Exception as e: is_next = ntfy.sendMsg(f"{ci.getComicName()} 下载出错了",error=e)
|
||||
ntfy.sendMsg(f"预计总章节大小：{cls.count_chapter + 1} / "+ str(ci.getLenChapters()))
|
||||
ci.setChapterIndex(cls.count_chapter + 1)
|
||||
if sleep != None and is_next: ntfy.sendMsg(f"{sleep} 秒后开始下载下一个章节",sleep=sleep)
|
||||
|
||||
#根据章节地址下载全部图片，并将文件名scramble开头的图片进行解密
|
||||
@classmethod
|
||||
def Onechapter(cls,chapter_url,scramble=None):
|
||||
is_next = True
|
||||
if not str(chapter_url).startswith("http"): chapter_url = ci.getBaseUrl() + chapter_url
|
||||
#下载图片
|
||||
is_next = cls.comicChapterDownload(chapter_url)
|
||||
ci.nextInfoToImgChapter()
|
||||
#下载完成后, 开始解密图片
|
||||
chapter_dir = ci.getDirComicChapter()
|
||||
if scramble and os.path.exists(chapter_dir): #获取章节图片路径
|
||||
dirs = os.listdir(chapter_dir)
|
||||
for img in dirs:
|
||||
if img.startswith("scramble="):
|
||||
imageUtils.encode_scramble_image(os.path.join(chapter_dir,img))
|
||||
#进入下一阶段
|
||||
ci.nextImgToDownloadChapter()
|
||||
return is_next
|
||||
|
||||
@classmethod
|
||||
def comicChapterDownload(cls,url,is_next=True):
|
||||
#获取本次工程的HOME目录
|
||||
comic_main = pathStr.getComicMain()
|
||||
try:
|
||||
if comic_main == pathStr.comic_jm: JMComicCommon.comicChapterDownload(url)
|
||||
if comic_main == pathStr.comic_bz: BaoZiComicCommon.comicChapterDownload(url)
|
||||
if comic_main == pathStr.comic_rm: RouManComicCommon.comicChapterDownload(url)
|
||||
except:
|
||||
htmlUtils.remove_HtmlCache(url)
|
||||
cls.comicChapterDownload(url,is_next)
|
||||
if comic_main == None: print("comic_main为空，退出中...") & exit()
|
||||
|
||||
(list_img,files_name,chapter_name,book_name) = [ci.getChapterListImg(),ci.getChapterFilesName(),ci.getChapter(),ci.getComicName()]
|
||||
#保存信息
|
||||
ci.nextSaveInfoChapter(chapter_name,list_img)
|
||||
#验证数据是已存在且是否完整
|
||||
cbz_file = ci.getNewFileCBZComicChapter()
|
||||
#更新Icon
|
||||
ci.getNewIconComicChapter()
|
||||
#检验CBZ文件
|
||||
CBZUtils.verCBZComic(cbz_file)
|
||||
is_next = CBZUtils.nextCBZ()
|
||||
is_old=CBZUtils.updateOldCBZ(files_name)
|
||||
#不存在ComicInfo.xml则生成
|
||||
if is_next and fu.notExists(ci.getPathComicInfoXML()): ci.writeComicInfoXML(chapter_name)
|
||||
if is_next and not is_old:
|
||||
# ntfy.sendMsg(f"{book_name} {chapter_name} 下载中")
|
||||
download_images(list_img,ci.getDirComicChapter(), files_name=files_name,concurrency=None,timeout=8)
|
||||
# ntfy.sendMsg("等待数据检验中...",sleep=0.5)
|
||||
is_next = fu.equImages(ci.getDirComicChapter(),list_img)
|
||||
# if not is_next: ntfy.sendMsg(msg=f"下载数据(不完整，{int(repeat*2)}秒钟后尝试第{repeat}次",sleep=int(repeat*2))
|
||||
# repeat += 1
|
||||
return is_next
|
||||
@@ -1,494 +0,0 @@
|
||||
from xml.dom.minidom import Document
|
||||
import os,re
|
||||
from utils.comic.PathStr import pathStr
|
||||
import json,shutil
|
||||
from utils.FileUtils import dbUtils
|
||||
from utils.ComicUtils import fontUtils
|
||||
from utils.OldUtils import OldUtils
|
||||
|
||||
class comicInfo():
|
||||
COMIC_ICON_NAME = "000"
|
||||
COMIC_INFO_XML = "ComicInfo.xml"
|
||||
PROGRESS_INFO = "info"
|
||||
PROGRESS_DOWN = "download"
|
||||
PROGRESS_IMG = "download"
|
||||
PROGRESS_CBZ = "cbz"
|
||||
PROGRESS_DONE = "done"
|
||||
PROGRESS_NONE = "none"
|
||||
|
||||
IS_NEW_ICON = False
|
||||
|
||||
document = Document()
|
||||
path_comic_info = None
|
||||
|
||||
root = "ComicInfo"
|
||||
chapter = "Title"
|
||||
comic_name = "Series"
|
||||
number = "Number"
|
||||
dep = "Summary"
|
||||
author = "Writer"
|
||||
genre = "Genre"
|
||||
cbs = "Publisher"
|
||||
lang = "LanguageISO"
|
||||
comic_names = "SeriesGroup"
|
||||
tags = "Tags"
|
||||
date_year = "Year"
|
||||
date_month = "Month"
|
||||
date_day = "Day"
|
||||
page_count = "PageCount"
|
||||
pages = "Pages"
|
||||
web = "Web"
|
||||
age_rating = "AgeRating"
|
||||
|
||||
str_comic_name = None
|
||||
str_chapter = None
|
||||
str_number = None
|
||||
str_icon = None
|
||||
str_homepage = None
|
||||
str_listchapter = None
|
||||
str_chapter_imgs = None
|
||||
str_update_at = None
|
||||
str_date_year = None
|
||||
str_date_month = None
|
||||
str_date_day = None
|
||||
str_page_count = None
|
||||
str_web = None
|
||||
str_list_img = None
|
||||
str_files_img = None
|
||||
str_chapter_index= None
|
||||
str_value1 = None
|
||||
str_current_chapter_img = None
|
||||
list_skip = []
|
||||
|
||||
chapter_node = None
|
||||
comic_name_node = None
|
||||
number_node = None
|
||||
dep_node = None
|
||||
author_node = None
|
||||
genre_node = None
|
||||
cbs_node = None
|
||||
lang_node = None
|
||||
comic_names_node = None
|
||||
tags_node = None
|
||||
date_year_node = None
|
||||
date_month_node = None
|
||||
date_day_node = None
|
||||
page_count_node = None
|
||||
pages_node = None
|
||||
web_node = None
|
||||
age_rating_node = None
|
||||
|
||||
@classmethod
|
||||
def setNodeAndValue(cls,node,value):
|
||||
if value != None:
|
||||
c_node = cls.document.createElement(node)
|
||||
node_text = cls.document.createTextNode(fontUtils.ChineseConvert(str(value).replace("\n","")))
|
||||
c_node.appendChild(node_text)
|
||||
return c_node
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def root_node(cls): return cls.document.createElement(cls.root)
|
||||
|
||||
@classmethod
|
||||
def setChapterName(cls,value):
|
||||
cls.str_chapter = fontUtils.ChineseConvert(cls.fixFileName(value))
|
||||
OldUtils.setOldChapter(cls.fixFileName(value))
|
||||
cls.chapter_node = cls.setNodeAndValue(cls.chapter,value)
|
||||
|
||||
@classmethod
|
||||
def setComicValue(cls,value):
|
||||
result = None
|
||||
if value != None or value != "": result = value
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def setListChapter(cls, value): cls.str_list_chapter = cls.setComicValue(value)
|
||||
@classmethod
|
||||
def setChapterImgs(cls, value): cls.str_chapter_imgs = cls.setComicValue(value)
|
||||
@classmethod
|
||||
def setCurrentChapterImg(cls, value): cls.str_current_chapter_img = cls.setComicValue(value)
|
||||
@classmethod
|
||||
def getCurrentChapterImg(cls): return cls.str_current_chapter_img
|
||||
@classmethod
|
||||
def getChapterImgs(cls): return cls.str_chapter_imgs
|
||||
@classmethod
|
||||
def getLenChapters(cls): return len(cls.str_list_chapter)
|
||||
@classmethod
|
||||
def setComicName(cls,value):
|
||||
cls.str_comic_name = fontUtils.ChineseConvert(cls.fixFileName(value))
|
||||
OldUtils.setOldComicName(cls.fixFileName(value))
|
||||
cls.comic_name_node = cls.setNodeAndValue(cls.comic_name, value)
|
||||
|
||||
@classmethod
|
||||
def setComicNames(cls,value):
|
||||
#去重
|
||||
value = ",".join(set(str(fontUtils.ChineseConvert(value)).split(",")))
|
||||
cls.comic_names_node = cls.setNodeAndValue(cls.comic_names,value)
|
||||
@classmethod
|
||||
def setNumber(cls,value):
|
||||
cls.str_number = value
|
||||
cls.number_node = cls.setNodeAndValue(cls.number, value)
|
||||
@classmethod
|
||||
def getNumber(cls): return cls.str_number
|
||||
@classmethod
|
||||
def setDep(cls,value): cls.dep_node = cls.setNodeAndValue(cls.dep, value)
|
||||
@classmethod
|
||||
def setAuthor(cls,value): cls.author_node = cls.setNodeAndValue(cls.author,cls.getListToString(value))
|
||||
@classmethod
|
||||
def setLang(cls,value): cls.lang_node = cls.setNodeAndValue(cls.lang, value)
|
||||
@classmethod
|
||||
def setAgeRating(cls,value): cls.age_rating_node = cls.setNodeAndValue(cls.age_rating, value)
|
||||
@classmethod
|
||||
def setGenre(cls,value): cls.genre_node = cls.setNodeAndValue(cls.genre, cls.getListToString(value))
|
||||
@classmethod
|
||||
def setTags(cls,value): cls.tags_node = cls.setNodeAndValue(cls.tags,cls.getListToString(value))
|
||||
@classmethod
|
||||
def setCBS(cls,value): cls.cbs_node = cls.setNodeAndValue(cls.cbs,value)
|
||||
|
||||
@classmethod
|
||||
def setWeb(cls,value):
|
||||
cls.str_web = cls.setComicValue(value)
|
||||
cls.web_node = cls.setNodeAndValue(cls.web,cls.setComicValue(value))
|
||||
|
||||
@classmethod
|
||||
def setChapterListImg(cls,value): cls.str_list_img=cls.setComicValue(value)
|
||||
@classmethod
|
||||
def setValue1(cls,value): cls.str_value1 = value
|
||||
@classmethod
|
||||
def getValue1(cls): return cls.str_value1
|
||||
@classmethod
|
||||
def getChapterListImg(cls): return cls.str_list_img
|
||||
@classmethod
|
||||
def setChapterFilesName(cls,value): cls.str_files_img=cls.setComicValue(value)
|
||||
@classmethod
|
||||
def getChapterFilesName(cls): return cls.str_files_img
|
||||
@classmethod
|
||||
def getWeb(cls): return cls.str_web
|
||||
|
||||
@classmethod
|
||||
def setPageCount(cls,value):
|
||||
cls.str_page_count = cls.setComicValue(int(value))
|
||||
cls.page_count_node = cls.setNodeAndValue(cls.page_count,cls.str_page_count)
|
||||
|
||||
@classmethod
|
||||
def setPages(cls,value):
|
||||
if value != None:
|
||||
su = "."+str(value[0]).split(".")[-1]
|
||||
join_list=",".join(value).replace(su,"")
|
||||
value = join_list.split(",")
|
||||
cls.setPageCount(len(value)+1 if cls.IS_NEW_ICON else len(value))
|
||||
root_node = cls.document.createElement(cls.pages)
|
||||
if cls.IS_NEW_ICON:
|
||||
#添加封面
|
||||
icon_node = cls.document.createElement("Page")
|
||||
icon_node.setAttribute("Image","000")
|
||||
icon_node.setAttribute("Type","FrontCover")
|
||||
root_node.appendChild(icon_node)
|
||||
for page in value:
|
||||
c_node = cls.document.createElement("Page")
|
||||
page = page.split("_")[-1]
|
||||
c_node.setAttribute("Image",page)
|
||||
root_node.appendChild(c_node)
|
||||
cls.pages_node = root_node
|
||||
|
||||
@classmethod
|
||||
def setDate(cls,value,split):
|
||||
values = str(value).split(split)
|
||||
cls.str_date_year = values[0]
|
||||
cls.str_date_month = values[1]
|
||||
cls.str_date_day = values[2]
|
||||
cls.date_year_node = cls.setNodeAndValue(cls.date_year,values[0])
|
||||
cls.date_month_node = cls.setNodeAndValue(cls.date_month,values[1])
|
||||
cls.date_day_node = cls.setNodeAndValue(cls.date_day,values[2])
|
||||
|
||||
|
||||
@classmethod
|
||||
def setIcon(cls,value):
|
||||
cls.str_icon = cls.setComicValue(value)
|
||||
return cls.str_icon
|
||||
|
||||
@classmethod
|
||||
def setHomePage(cls, value): cls.str_homepage = cls.setComicValue(value)
|
||||
@classmethod
|
||||
def getHomePage(cls): return cls.str_homepage
|
||||
@classmethod
|
||||
def setUpdateAt(cls, value): cls.str_update_at = cls.setComicValue(value)
|
||||
@classmethod
|
||||
def getUpdateAt(cls): return cls.str_update_at
|
||||
@classmethod
|
||||
def getListToString(cls,to_list):
|
||||
value = to_list
|
||||
if isinstance(to_list,list):
|
||||
value = ",".join(to_list)
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def setChapterIndex(cls,value):
|
||||
cls.setNumber(value)
|
||||
cls.str_chapter_index = cls.setComicValue(value)
|
||||
@classmethod
|
||||
def getChapterIndex(cls): return cls.str_chapter_index
|
||||
@classmethod
|
||||
def setComicNameSkips(cls,value): return cls.list_skip.append(value)
|
||||
@classmethod
|
||||
def getIsComicNameSkips(cls,value): return value in ",".join(cls.list_skip)
|
||||
|
||||
@classmethod
|
||||
def getBaseUrl(cls,url=None):
|
||||
if url == None:
|
||||
url = cls.str_homepage
|
||||
num = 3
|
||||
index = 0
|
||||
for x in range(0, num):
|
||||
index = str(url).find("/",index)+1
|
||||
return url[0:index-1]
|
||||
|
||||
@classmethod
|
||||
def getIcon(cls): return cls.str_icon
|
||||
@classmethod
|
||||
def getComicName(cls): return cls.str_comic_name
|
||||
@classmethod
|
||||
def getChapter(cls): return cls.str_chapter
|
||||
|
||||
@classmethod
|
||||
def fixFileName(cls,filename,replace=None):
|
||||
intab = r'[?*/\|.:><]'
|
||||
str_replace = ""
|
||||
if replace != None:
|
||||
str_replace = replace
|
||||
filename = re.sub(intab, str_replace, filename)
|
||||
count = 1
|
||||
while True:
|
||||
str_file = filename[0-count]
|
||||
if str_file == " ":
|
||||
count += 1
|
||||
else:
|
||||
filename = filename[0:len(filename)+1-count]
|
||||
break
|
||||
return filename
|
||||
|
||||
@classmethod
|
||||
def getDirConfComic(cls):
|
||||
if cls.str_comic_name != None:
|
||||
return os.path.join(pathStr.base_conf_path(), cls.str_comic_name)
|
||||
else:
|
||||
print("comicName不存在，退出中")
|
||||
exit()
|
||||
|
||||
@classmethod
|
||||
def getDirCBZComic(cls):
|
||||
if cls.str_comic_name != None:
|
||||
return os.path.join(pathStr.base_cbz(), cls.str_comic_name)
|
||||
else:
|
||||
print("comicName不存在，退出中 getDirCBZComic")
|
||||
exit()
|
||||
|
||||
@classmethod
|
||||
def getDirCBZComicChapter(cls):
|
||||
if cls.str_comic_name != None and cls.str_chapter != None:
|
||||
return os.path.join(pathStr.base_cbz(),cls.str_comic_name,cls.str_chapter)
|
||||
else:
|
||||
print("comicName不存在，退出中 getDirCBZComicChapter")
|
||||
exit()
|
||||
|
||||
@classmethod
|
||||
def getSortDirCBZComicChapter(cls):
|
||||
if cls.str_comic_name != None and cls.str_chapter != None and cls.str_chapter_index != None:
|
||||
return os.path.join(pathStr.base_cbz(),cls.str_comic_name,str(cls.str_chapter_index)+" "+cls.str_chapter)
|
||||
else:
|
||||
print("comicName不存在，退出中 getSortDirCBZComicChapter")
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def getNewCBZComicChapter(cls,type="dir"): return cls.getNewToComicChapter(".CBZ", type)
|
||||
@classmethod
|
||||
def getNewIconComicChapter(cls,type="dir"): return cls.getNewToComicChapter(".jpg", type)
|
||||
@classmethod
|
||||
def getNewFileCBZComicChapter(cls,type="file"): return cls.getNewToComicChapter(".CBZ", type)
|
||||
@classmethod
|
||||
def getNewFileIconComicChapter(cls,type="file"): return cls.getNewToComicChapter(".jpg", type)
|
||||
|
||||
|
||||
@classmethod
|
||||
def getNewToComicChapter(cls,su,type="dir"):
|
||||
c_dir = cls.getDirCBZComicChapter()
|
||||
s_dir = cls.getSortDirCBZComicChapter()
|
||||
c_path = cls.getDirCBZComicChapter()+su
|
||||
s_path = cls.getSortDirCBZComicChapter()+su
|
||||
if os.path.exists(s_path) and s_path != None:
|
||||
shutil.move(s_path, c_path)
|
||||
print("文件已移动至：", c_path)
|
||||
if type == "file":
|
||||
return c_path
|
||||
return c_dir
|
||||
|
||||
@classmethod
|
||||
def getDirComic(cls):
|
||||
if cls.str_comic_name != None:
|
||||
return os.path.join(pathStr.base_comic_img(), cls.str_comic_name)
|
||||
else:
|
||||
print("comicName不存在，退出中")
|
||||
exit()
|
||||
|
||||
@classmethod
|
||||
def getDirComicChapter(cls):
|
||||
if cls.str_comic_name != None and cls.str_chapter != None:
|
||||
return os.path.join(pathStr.base_comic_img(),cls.str_comic_name,cls.str_chapter)
|
||||
else:
|
||||
print("comicName与chapter 不存在，退出中")
|
||||
exit()
|
||||
@classmethod
|
||||
def getPathComicInfoXML(cls):
|
||||
try:
|
||||
cls.path_comic_info = os.path.join(pathStr.base_comic_img(),cls.str_comic_name,cls.str_chapter, cls.COMIC_INFO_XML)
|
||||
except:
|
||||
return None
|
||||
return cls.path_comic_info
|
||||
|
||||
@classmethod
|
||||
def writeComicInfoXML(cls,chapter=None,path=None,overlay=False):
|
||||
root = cls.root_node()
|
||||
new_document = Document()
|
||||
new_document.appendChild(root)
|
||||
if cls.chapter_node != None: root.appendChild(cls.chapter_node)
|
||||
if cls.comic_name_node != None: root.appendChild(cls.comic_name_node)
|
||||
if cls.number_node != None: root.appendChild(cls.number_node)
|
||||
if cls.dep_node != None: root.appendChild(cls.dep_node)
|
||||
if cls.author_node != None: root.appendChild(cls.author_node)
|
||||
if cls.genre_node != None: root.appendChild(cls.genre_node)
|
||||
if cls.cbs_node != None: root.appendChild(cls.cbs_node)
|
||||
if cls.lang_node != None: root.appendChild(cls.lang_node)
|
||||
if cls.age_rating_node != None: root.appendChild(cls.age_rating_node)
|
||||
if cls.comic_names_node != None: root.appendChild(cls.comic_names_node)
|
||||
if cls.tags_node != None: root.appendChild(cls.tags_node)
|
||||
if cls.date_year_node != None: root.appendChild(cls.date_year_node)
|
||||
if cls.date_month_node != None: root.appendChild(cls.date_month_node)
|
||||
if cls.date_day_node != None: root.appendChild(cls.date_day_node)
|
||||
if cls.page_count_node != None: root.appendChild(cls.page_count_node)
|
||||
if cls.pages_node != None: root.appendChild(cls.pages_node)
|
||||
cls.getPathComicInfoXML()
|
||||
if path != None: cls.path_comic_info = os.path.join(path,cls.COMIC_INFO_XML)
|
||||
base_dir = os.path.dirname(cls.path_comic_info)
|
||||
if not os.path.exists(base_dir): os.makedirs(base_dir)
|
||||
if os.path.exists(cls.path_comic_info) and not overlay:
|
||||
print(f"{cls.COMIC_INFO_XML} 已存在")
|
||||
return None
|
||||
with open(cls.path_comic_info , "w", encoding="utf-8") as fo:
|
||||
new_document.writexml(fo, indent='', addindent='\t', newl='\n', encoding="utf-8")
|
||||
fo.close()
|
||||
print(f"{cls.COMIC_INFO_XML} 已生成 pathd=", cls.path_comic_info)
|
||||
|
||||
#文件保存
|
||||
@classmethod
|
||||
def file_save(cls,path,data,mode=None,print_msg=False):
|
||||
result = {}
|
||||
f = {}
|
||||
dir_name = os.path.dirname(path)
|
||||
if not os.path.exists(dir_name):
|
||||
os.makedirs(dir_name)
|
||||
|
||||
save_path = os.path.join(path)
|
||||
if os.path.exists(save_path):
|
||||
os.remove(save_path)
|
||||
data = json.dumps(data)
|
||||
if mode == None:
|
||||
mode = "w+"
|
||||
try:
|
||||
f = open(save_path, mode, encoding="utf-8")
|
||||
f.write(data)
|
||||
f.close()
|
||||
if print_msg:
|
||||
print("data=",data)
|
||||
result = path + "文件写入成功"
|
||||
except:
|
||||
result = path + "文件写入失败"
|
||||
print(result)
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def nextSavePath(cls,next,data=None):
|
||||
save_path = cls.getDirConfComic()+"/"+next+cls.str_chapter
|
||||
if data != None:
|
||||
cls.file_save(save_path, data)
|
||||
return save_path
|
||||
|
||||
@classmethod
|
||||
def nextSaveInfoChapter(cls,chapter,data=None):
|
||||
if data == None: data = cls.getChapterImgs()
|
||||
if cls.str_chapter != chapter:
|
||||
print(f"chapter {cls.str_chapter} 与 {chapter} 不一致，已自动跳过")
|
||||
cls.setProgress(cls.PROGRESS_INFO)
|
||||
cls.nextSavePath("info_",data)
|
||||
|
||||
@classmethod
|
||||
def nextInfoToImgChapter(cls): cls.setProgress(cls.PROGRESS_IMG)
|
||||
@classmethod
|
||||
def nextImgToDownloadChapter(cls): cls.setProgress(cls.PROGRESS_DOWN)
|
||||
@classmethod
|
||||
def nextDownloadToCBZChapter(cls): cls.setProgress(cls.PROGRESS_CBZ)
|
||||
@classmethod
|
||||
def nextCBZToDoneChapter(cls): cls.setProgress(cls.PROGRESS_DONE)
|
||||
@classmethod
|
||||
def nextDoneSave(cls,data): cls.nextSavePath("done_",data)
|
||||
@classmethod
|
||||
def setProgress(cls,progress):
|
||||
dbUtils.setComic(cls.str_chapter,progress,cls.str_comic_name)
|
||||
|
||||
@classmethod
|
||||
def isProgress(cls,progress,remove=None):
|
||||
if remove: cls.setProgress("None")
|
||||
return dbUtils.query(cls.str_chapter,progress,cls.str_comic_name)
|
||||
|
||||
@classmethod
|
||||
def iconDB(cls): dbUtils.setComic(cls.str_comic_name,cls.str_icon,"icons")
|
||||
|
||||
@classmethod
|
||||
def equIcon(cls): return dbUtils.query(cls.str_comic_name,cls.str_icon,"icons")
|
||||
|
||||
@classmethod
|
||||
def setConfDirComicPath(cls,file_name,comic_name=None):
|
||||
if comic_name != None: cls.setComicName(comic_name)
|
||||
return os.path.join(cls.getDirConfComic(),file_name)
|
||||
|
||||
@classmethod
|
||||
def saveConfComicData(cls,file_name,data,comic_name=None): cls.file_save(cls.setConfDirComicPath(file_name,comic_name), data)
|
||||
@classmethod
|
||||
def getPathInitConfComicData(cls,file_name,comic_name=None): return cls.setConfDirComicPath(file_name,comic_name)
|
||||
|
||||
@classmethod
|
||||
def updateComicDate(cls,date=None):
|
||||
update_at = cls.getUpdateAt()
|
||||
if date != None:
|
||||
update_at = date
|
||||
dbUtils.setComic(cls.str_comic_name, update_at, "update")
|
||||
|
||||
@classmethod
|
||||
def isUpdateComic(cls):
|
||||
return dbUtils.query(cls.str_comic_name, cls.str_update_at,"update")
|
||||
|
||||
@classmethod
|
||||
def comicChapterDownload(cls,imgs,names):
|
||||
cls.setChapterImgs(imgs)
|
||||
cls.setChapterListImg(imgs)
|
||||
cls.setPages(names)
|
||||
cls.setChapterFilesName(names)
|
||||
|
||||
@classmethod
|
||||
def setComicInfo(cls,comicname=None,homepage=None,alias=None,author=None,icon=None,tags=None,
|
||||
dep=None,genre=None,lang=None,age_rating=None,chapters=None,update_at=None,current_chapter_img=None):
|
||||
author = str(author).replace("&",",").replace(" ",",")
|
||||
cls.setHomePage(homepage)
|
||||
cls.setComicName(str(comicname))
|
||||
if alias != None: comicInfo.setComicNames(alias)
|
||||
cls.setAuthor(author)
|
||||
cls.setIcon(icon)
|
||||
cls.setTags(tags)
|
||||
cls.setDep(dep)
|
||||
#comicInfo.setCBS("韩漫")
|
||||
if genre != None: cls.setGenre(genre)
|
||||
cls.setLang(lang)
|
||||
cls.setAgeRating(age_rating)
|
||||
cls.setListChapter(chapters)
|
||||
cls.setUpdateAt(update_at)
|
||||
cls.setCurrentChapterImg(current_chapter_img)
|
||||
@@ -1,66 +0,0 @@
|
||||
import os,datetime
|
||||
from time import strftime
|
||||
class pathStr:
|
||||
|
||||
comic_name = None
|
||||
comic_jm="JM"
|
||||
comic_bz="BZ"
|
||||
comic_rm="RM"
|
||||
|
||||
comic_url_main = None
|
||||
base_comic_out = os.path.join("/mnt", "Comics")
|
||||
old_cbz_path = os.path.join("/mnt","OldComics")
|
||||
@classmethod
|
||||
def base_cbz(cls): return cls.getBaseComicPath("CBZ")
|
||||
@classmethod
|
||||
def base_comic_img(cls): return cls.getBaseComicPath("outputComic")
|
||||
@classmethod
|
||||
def base_conf_path(cls): return cls.getBaseComicPath(".conf")
|
||||
@classmethod
|
||||
def base_html_cache(cls): return cls.getBaseComicPath("html_cache")
|
||||
@classmethod
|
||||
def base_html_chapter(cls): return cls.getBaseComicPath("html_updated")
|
||||
@classmethod
|
||||
def base_comic_update(cls): return cls.getBaseComicPath("comic_update")
|
||||
@classmethod
|
||||
def base_db(cls): return cls.getBaseComicPath("db")
|
||||
|
||||
@classmethod
|
||||
def getBaseComicPath(cls,join_path): return os.path.join(cls.base_comic_out,join_path)
|
||||
|
||||
@classmethod
|
||||
def setComicMainAndPath(cls,value):
|
||||
cls.setComicMain(value)
|
||||
cls.setComicMainPath(value)
|
||||
|
||||
@classmethod
|
||||
def setComicMain(cls,value): cls.comic_name = value
|
||||
|
||||
@classmethod
|
||||
def getComicMain(cls): return cls.comic_name
|
||||
|
||||
@classmethod
|
||||
def setComicMainPath(cls,value):
|
||||
#if value != cls.comic_rm: cls.base_comic_out = os.path.join(cls.base_comic_out, value)
|
||||
cls.base_comic_out = os.path.join(cls.base_comic_out, value)
|
||||
|
||||
@classmethod
|
||||
def base_html_week(cls):
|
||||
date_path = cls.getDatePath()
|
||||
return os.path.join(cls.base_comic_out,"html_"+str(date_path))
|
||||
|
||||
@classmethod
|
||||
def getDatePath(cls):
|
||||
date = datetime.datetime.now()
|
||||
year = int(date.strftime("%Y"))
|
||||
month = int(date.strftime("%m"))
|
||||
day = int(date.strftime("%d"))
|
||||
week = cls.get_week_of_month(year, month, day)
|
||||
return f"{year}{month}{week}"
|
||||
|
||||
@classmethod
|
||||
def get_week_of_month(cls, year, month, day):
|
||||
begin = int(datetime.date(year, month, 1).strftime("%W"))
|
||||
end = int(datetime.date(year, month, day).strftime("%W"))
|
||||
week = "{:0>2d}".format(end - begin + 1)
|
||||
return week
|
||||
@@ -1,174 +0,0 @@
|
||||
""" Download image according to given urls and automatically rename them in order. """
|
||||
# -*- coding: utf-8 -*-
|
||||
# author: Yabin Zheng
|
||||
# Email: sczhengyabin@hotmail.com
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
from queue import Queue
|
||||
import shutil
|
||||
import imghdr
|
||||
import os
|
||||
import concurrent.futures
|
||||
import requests
|
||||
import time
|
||||
from utils.Ntfy import ntfy
|
||||
from utils.comic.ComicInfo import comicInfo
|
||||
from utils.HtmlUtils import htmlUtils
|
||||
from utils.FileUtils import fileUtils as fu
|
||||
|
||||
headers = {
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||||
"Proxy-Connection": "keep-alive",
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36",
|
||||
"Accept-Encoding": "gzip, deflate, sdch",
|
||||
# 'Connection': 'close',
|
||||
}
|
||||
|
||||
down_queue = Queue()
|
||||
|
||||
def common_download(file_name,image_url,dst_dir,timeout=10,proxy=None,proxy_type=None):
|
||||
proxies = None
|
||||
if proxy_type is not None:
|
||||
proxies = {
|
||||
"http": proxy_type + "://" + proxy,
|
||||
"https": proxy_type + "://" + proxy }
|
||||
response = None
|
||||
file_path = os.path.join(dst_dir, file_name)
|
||||
if os.path.exists(file_path):
|
||||
print("download_image 文件已存在，已跳过=",file_path)
|
||||
return None
|
||||
temp_path = os.path.join(dst_dir, file_name+".downloads")
|
||||
repair_count = 1
|
||||
response = requests.get(
|
||||
image_url, headers=headers, timeout=timeout, proxies=proxies)
|
||||
while response.status_code != 200 and repair_count <= 5:
|
||||
time.sleep(0.7)
|
||||
download_image(image_url,dst_dir,file_name)
|
||||
ntfy.sendMsg(f'重试：第{repair_count}次 {image_url}')
|
||||
repair_count += 1
|
||||
with open(temp_path, 'wb') as f:
|
||||
f.write(response.content)
|
||||
response.close()
|
||||
#验证是否是图像
|
||||
if fu.ver_file(temp_path,type="image"):
|
||||
shutil.move(temp_path, file_path)
|
||||
print("## OK: {} {}".format(file_path, image_url))
|
||||
else:
|
||||
print("## Fail: {} {}".format(image_url, "图像损坏"))
|
||||
down_queue.put([file_name,image_url,dst_dir])
|
||||
|
||||
def download_image(timeout=20, proxy_type=None, proxy=None,type="image"):
|
||||
repeat = 0
|
||||
while not down_queue.empty() and repeat <= 10:
|
||||
repeat += 1
|
||||
data = down_queue.get(False)
|
||||
(file_name,image_url,dst_dir) = [data[0],data[1],data[2]]
|
||||
if repeat > 1:
|
||||
ntfy.sendMsg(f"第{repeat}次下载数据中... file_name={file_name}")
|
||||
try:
|
||||
common_download(file_name,image_url,dst_dir)
|
||||
except:
|
||||
ntfy.sendMsg(f"下载重试中 {file_name}={image_url}")
|
||||
down_queue.put([file_name,image_url,dst_dir])
|
||||
|
||||
|
||||
|
||||
def download_images(image_urls, dst_dir,concurrency=None,timeout=20,proxy_type=None, proxy=None,files_name=None):
|
||||
"""
|
||||
Download image according to given urls and automatically rename them in order.
|
||||
:param timeout:
|
||||
:param proxy:
|
||||
:param proxy_type:
|
||||
:param image_urls: list of image urls
|
||||
:param dst_dir: output the downloaded images to dst_dir
|
||||
:param file_prefix: if set to "img", files will be in format "img_xxx.jpg"
|
||||
:param concurrency: number of requests process simultaneously
|
||||
:return: none
|
||||
"""
|
||||
if concurrency == None:
|
||||
concurrency = len(image_urls)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor:
|
||||
future_list = list()
|
||||
count = 0
|
||||
if not os.path.exists(dst_dir):
|
||||
os.makedirs(dst_dir)
|
||||
for image_url in image_urls:
|
||||
file_name = files_name[count]
|
||||
down_queue.put([file_name,image_url,dst_dir])
|
||||
future_list.append(executor.submit(
|
||||
download_image,timeout, proxy_type, proxy))
|
||||
count += 1
|
||||
concurrent.futures.wait(future_list, timeout)
|
||||
|
||||
def download_comic_icon(is_new=comicInfo.IS_NEW_ICON):
|
||||
icon_url = comicInfo.getIcon()
|
||||
if icon_url == None:
|
||||
print("icon 不存在，已跳过")
|
||||
return None
|
||||
save_name = comicInfo.COMIC_ICON_NAME
|
||||
icon_prefix = "."+str(icon_url).split(".")[-1]
|
||||
icon_prefix = icon_prefix.split("?")[0]
|
||||
#判断漫画名路径是否已存在comicname/cover.jpg, 存在跳过
|
||||
path_comic_icon = os.path.join(comicInfo.getDirConfComic(),save_name+icon_prefix)
|
||||
if not comicInfo.equIcon() and fu.exists(path_comic_icon):
|
||||
os.remove(path_comic_icon)
|
||||
if fu.notExists(path_comic_icon):
|
||||
download(icon_url, path_comic_icon)
|
||||
#if not os.path.exists(path_cbz_comic):
|
||||
# os.makedirs(path_cbz_comic)
|
||||
save_path = os.path.join(comicInfo.getDirCBZComic(),comicInfo.getChapter()+icon_prefix)
|
||||
if is_new:
|
||||
#历史版本ICON
|
||||
if os.path.exists(save_path):
|
||||
os.remove(save_path)
|
||||
if os.path.exists(path_comic_icon):
|
||||
base_dir = comicInfo.getDirComicChapter()
|
||||
if not os.path.exists(base_dir): os.makedirs(base_dir)
|
||||
shutil.copy(path_comic_icon,os.path.join(base_dir,save_name+icon_prefix))
|
||||
else:
|
||||
if fu.notExists(comicInfo.getDirCBZComic()): os.makedirs(comicInfo.getDirCBZComic())
|
||||
shutil.copy(path_comic_icon,save_path)
|
||||
print(f"{path_comic_icon} 已复制至: {save_path}")
|
||||
#保存icon信息
|
||||
comicInfo.iconDB()
|
||||
comicInfo.nextDownloadToCBZChapter()
|
||||
comicInfo.setProgress(comicInfo.PROGRESS_CBZ)
|
||||
|
||||
# 定义下载函数
|
||||
def download(url,path,file_type=None):
|
||||
if os.path.exists(path):
|
||||
if imghdr.what(path):
|
||||
msg = "已存在同路径文件,已跳过:"+path
|
||||
print(msg)
|
||||
return msg
|
||||
else:
|
||||
print("文件已损坏，已重试："+path)
|
||||
path = os.path.join(os.path.dirname(path),str(os.path.basename(path)).split("?")[0])
|
||||
tmp_file = path+".downloads"
|
||||
if os.path.exists(tmp_file):
|
||||
os.remove(tmp_file)
|
||||
print("存在缓存文件，已删除：",tmp_file)
|
||||
repair_count = 1
|
||||
res = htmlUtils.getBytes(url)
|
||||
while res.status_code != 200 and repair_count <= 5:
|
||||
res = htmlUtils.getBytes(url)
|
||||
print(f'重试：第{repair_count}次 {url}')
|
||||
repair_count += 1
|
||||
#判断是否为图片
|
||||
if file_type == "image":
|
||||
if 'image' not in res.headers.get("content-type",""):
|
||||
print(f"url= {url} Error: URL doesnot appear to be an image")
|
||||
basedir= os.path.dirname(path)
|
||||
if not os.path.exists(basedir):
|
||||
os.makedirs(basedir)
|
||||
#expected_length = res.headers.get('Content-Length')
|
||||
#actual_length = res.raw.tell()
|
||||
with open(tmp_file, 'wb') as f:
|
||||
for ch in res:
|
||||
f.write(ch)
|
||||
f.close()
|
||||
shutil.move(tmp_file, path)
|
||||
print(f"url={url} 保存至：{path}")
|
||||
return path
|
||||
Reference in new issue
Block a user