This commit is contained in:
cwx committed 2023-01-08 17:30:08 +08:00
1 parent 8739e0f4c8
commit f632ba738f
16 files changed
+1230 -45

No files matched your search

+1 -1
View File
@@ -60,6 +60,7 @@ class CBZUtils:
size_imgs = len(comicInfo.getChapterImgs())
if size_zip -1 == size_imgs:
ntfy.sendMsg(f"打包校验成功: {packCBZ_path}")
comicInfo.nextCBZToDoneChapter()
return True
else:
ntfy.sendMsg(f"打包检验不完整:{packCBZ_path}")
@@ -68,7 +69,6 @@ class CBZUtils:
except:
print("")
#ntfy.sendMsg("CBZ打包失败")
comicInfo.nextCBZToDoneChapter()
@classmethod
def zip_info(cls,zip_path=None):
+16 -3
View File
@@ -5,18 +5,29 @@ import traceback
import time
from urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
from myran import Myran
from utils.Ntfy import ntfy
import re
from utils.comic.PathStr import pathStr
class htmlUtils:
headers = {'User-Agent': UserAgent().random}
myran = Myran()
headers = {
#'cookie':'ipcountry=US; AVS=4eb0s4o5ho9hfmp704ge7jtium; ipm5=bb7f6ac39cebfa37e89bd07544c549fd; cover=1; guide=1; __atuvc=12|39,31|40,5|41,0|42,4|43; __atuvs=635cabf67eff0d49003; yuo1={"objName":"hT3l8Pyn15Uf","request_id":0,"zones":[{"idzone":"2967008","here":{}},{"idzone":"2967010","here":{}},{"idzone":"2967010","here":{}},{"idzone":"3597795","sub":"70","here":{}}]}',
#'referer': 'https://18comic.org/',
"User_Agent": myran.agents()
}
# headers = {'User-Agent': UserAgent().random}
url_data = {}
@classmethod
def getPathSaveHtml(cls,url,type=None):
rstr = r"[\/\\\:\*\?\"\<\>\|\.]" #  '/ \ : * ? " < > |'
file_url = re.sub(rstr, "", url)
try:
file_url = re.sub(rstr, "", url)
except:
file_url = "error_cache"
file_path = os.path.join(pathStr.base_html_cache,file_url)
if type == "new":
return file_path
@@ -64,7 +75,7 @@ class htmlUtils:
if count <= 3:
try:
print(f"请求地址:{curl}")
res = s.get(curl,stream=True, headers=cls.headers, timeout=180)
res = s.get(curl,stream=True, headers=cls.headers, timeout=1000)
if type == "bytes":
url_text = res
if type == "json":
@@ -74,6 +85,8 @@ class htmlUtils:
if type == None:
url_text = html.fromstring(res.text)
cls.saveHtml(curl,res.text)
# if res.status_code != 200:
# cls.remove_HtmlCache(curl)
except:
print(f'Retry! 第{count}次')
time.sleep(1)
+2
View File
@@ -20,6 +20,7 @@ class netUtils:
return msg
else:
print("文件已损坏,已重试:"+path)
path = os.path.join(os.path.dirname(path),str(os.path.basename(path)).split("?")[0])
tmp_file = path+".downloads"
if os.path.exists(tmp_file):
os.remove(tmp_file)
@@ -81,6 +82,7 @@ class netUtils:
return None
save_name = "cover"
icon_su = "."+str(icon_url).split(".")[-1]
icon_su = icon_su.split("?")[0]
#判断漫画名路径是否已存在comicname/cover.jpg, 存在跳过
pathComicIcon = os.path.join(comicInfo.getDirConfComic(),save_name+icon_su)
if not os.path.exists(pathComicIcon):
+27 -1
View File
@@ -1,3 +1,4 @@
from utils.HtmlUtils import htmlUtils
from utils.Ntfy import ntfy
from utils.CBZUtils import CBZUtils
from utils.comic.ComicInfo import comicInfo
@@ -31,4 +32,29 @@ class verUtils:
return False
else:
ntfy.sendMsg("info_data 为空")
return False
return False
@classmethod
def verNextCBZ(cls,list_img):
#验证数据是已存在且是否完整
cbz_path = comicInfo.getDirCBZComicChapter()+".CBZ"
is_next = False
if os.path.exists(cbz_path):
try:
cbz_size = len(CBZUtils.zip_info(cbz_path)) - 1
except:
cbz_size = 0
if len(list_img) == cbz_size:
ntfy.sendMsg(f"{comicInfo.getComicName()} {comicInfo.getChapter()} 数据完整,已跳过")
is_next = True
else:
ntfy.sendMsg(f"{comicInfo.getComicName()} {comicInfo.getChapter()} 数据不完整,尝试删除配置CBZ文件后重试")
try:
if cbz_size < len(list_img) or os.path.getsize(cbz_path) < 300000:
ntfy.sendMsg(f"删除 {cbz_path}")
os.remove(cbz_path)
else:
is_next = True
except:
ntfy(f"删除失败 {cbz_path}")
return is_next
+94 -30
View File
@@ -17,8 +17,14 @@ class comicInfo():
cbs = "Publisher"
lang = "LanguageISO"
comic_names = "SeriesGroup"
tags = "Tag"
tags = "Tags"
date_year = "Year"
date_month = "Month"
date_day = "Day"
page_count = "PageCount"
pages = "Pages"
web = "Web"
str_comicName = None
str_chapter = None
str_icon = None
@@ -26,7 +32,12 @@ class comicInfo():
str_listChapter = None
str_chapter_imgs = None
str_updateAt = None
str_date_year = None
str_date_month = None
str_date_day = None
str_page_count = None
str_web = None
chapter_node = None
comicName_node = None
dep_node = None
@@ -36,12 +47,18 @@ class comicInfo():
lang_node = None
comicNames_node = None
tags_node = None
date_year_node = None
date_month_node = None
date_day_node = None
page_count_node = None
pages_node = None
web_node = None
@classmethod
def setNodeAndValue(cls,node,value):
if not value == None:
if value != None:
c_node = cls.document.createElement(node)
node_text = cls.document.createTextNode(value)
node_text = cls.document.createTextNode(str(value).replace("\n",""))
c_node.appendChild(node_text)
return c_node
return None
@@ -86,7 +103,7 @@ class comicInfo():
@classmethod
def setAuthor(cls,value):
cls.author_node = cls.setNodeAndValue(cls.author,value)
cls.author_node = cls.setNodeAndValue(cls.author,cls.getListToString(value))
@classmethod
def setLang(cls,value):
@@ -94,16 +111,55 @@ class comicInfo():
@classmethod
def setTag(cls,value):
cls.tag_node = cls.setNodeAndValue(cls.tag, value)
cls.tag_node = cls.setNodeAndValue(cls.tag, cls.getListToString(value))
@classmethod
def setTags(cls,value):
cls.tags_node = cls.setNodeAndValue(cls.tags,value)
cls.tags_node = cls.setNodeAndValue(cls.tags,cls.getListToString(value))
@classmethod
def setCBS(cls,value):
cls.cbs_node = cls.setNodeAndValue(cls.cbs,value)
@classmethod
def setWeb(cls,value):
cls.str_web = value
cls.web_node = cls.setNodeAndValue(cls.web,value)
@classmethod
def getWeb(cls):
return cls.str_web
@classmethod
def setPageCount(cls,value):
cls.str_page_count = value
cls.page_count_node = cls.setNodeAndValue(cls.page_count,value)
@classmethod
def setPages(cls,value):
if value != None:
su = "."+str(value[0]).split(".")[-1]
join_list=",".join(value).replace(su,"")
value = join_list.split(",")
cls.setPageCount(str(len(value)))
root_node = cls.document.createElement(cls.pages)
for page in value:
c_node = cls.document.createElement("Page")
c_node.setAttribute("Image",page)
root_node.appendChild(c_node)
cls.pages_node = root_node
@classmethod
def setDate(cls,value,split):
values = str(value).split(split)
cls.str_date_year = values[0]
cls.str_date_month = values[1]
cls.str_date_day = values[2]
cls.date_year_node = cls.setNodeAndValue(cls.date_year,values[0])
cls.date_month_node = cls.setNodeAndValue(cls.date_month,values[1])
cls.date_day_node = cls.setNodeAndValue(cls.date_day,values[2])
@classmethod
def setIcon(cls,value):
cls.str_icon = value
@@ -124,7 +180,14 @@ class comicInfo():
@classmethod
def getUpdateAt(cls):
return cls.str_updateAt
@classmethod
def getListToString(cls,to_list):
value = to_list
if isinstance(to_list,list):
value = ",".join(to_list)
return value
'''
获取网站主页
'''
@@ -207,39 +270,40 @@ class comicInfo():
return cls.pathComicInfo
@classmethod
def writeComicInfoXML(cls,chapter):
def writeComicInfoXML(cls,chapter,path=None):
if cls.chapter == cls.fixFileName(chapter):
print(f"cls.chapter {cls.chapter} 与 chapter {chapter} 不相等,已自动跳过")
root = cls.Root()
newDocument = Document()
newDocument.appendChild(root)
if not cls.chapter_node == None:
root.appendChild(cls.chapter_node)
if not cls.comicName_node == None:
root.appendChild(cls.comicName_node)
if not cls.dep_node == None:
root.appendChild(cls.dep_node)
if not cls.author_node == None:
root.appendChild(cls.author_node)
if not cls.tag_node == None:
root.appendChild(cls.tag_node)
if not cls.cbs_node == None:
root.appendChild(cls.cbs_node)
if not cls.lang_node == None:
root.appendChild(cls.lang_node)
if not cls.comicNames_node == None:
root.appendChild(cls.comicNames_node)
if not cls.tags_node == None:
root.appendChild(cls.tags_node)
if cls.chapter_node != None: root.appendChild(cls.chapter_node)
if cls.comicName_node != None: root.appendChild(cls.comicName_node)
if cls.dep_node != None: root.appendChild(cls.dep_node)
if cls.author_node != None: root.appendChild(cls.author_node)
if cls.tag_node != None: root.appendChild(cls.tag_node)
if cls.cbs_node != None: root.appendChild(cls.cbs_node)
if cls.lang_node != None: root.appendChild(cls.lang_node)
if cls.comicNames_node != None: root.appendChild(cls.comicNames_node)
if cls.tags_node != None: root.appendChild(cls.tags_node)
if cls.date_year_node != None: root.appendChild(cls.date_year_node)
if cls.date_month_node != None: root.appendChild(cls.date_month_node)
if cls.date_day_node != None: root.appendChild(cls.date_day_node)
if cls.page_count_node != None: root.appendChild(cls.page_count_node)
if cls.pages_node != None: root.appendChild(cls.pages_node)
cls.getPathComicInfoXML()
if path != None:
cls.pathComicInfo = os.path.join(path,"ComicInfo.xml")
base_dir = os.path.dirname(cls.pathComicInfo)
if not os.path.exists(base_dir):
os.makedirs(base_dir)
if os.path.exists(cls.pathComicInfo):
print("ComicInfo.xml 已存在")
return None
with open(cls.pathComicInfo , "w", encoding="utf-8") as fo:
newDocument.writexml(fo, indent='', addindent='\t', newl='\n', encoding="utf-8")
fo.close()
#print("ComicInfo.xml 已生成 pathd=", cls.pathComicInfo)
print("ComicInfo.xml 已生成 pathd=", cls.pathComicInfo)
#文件保存
@classmethod
+1 -1
View File
@@ -2,7 +2,7 @@ import os,datetime
from time import strftime
class pathStr:
#base_comic_out = "COMICOUT"
base_comic_out = os.path.join("/mnt", "bigTComics")
base_comic_out = os.path.join("/mnt", "bigTComics","JM")
base_CBZ = os.path.join(base_comic_out,"CBZ")
base_comic_img = os.path.join(base_comic_out,"outputComic")
base_conf_path = os.path.join(base_comic_out,".conf")
+5 -2
View File
@@ -33,13 +33,16 @@ def download_image(image_url, dst_dir, file_name, timeout=20, proxy_type=None, p
response = None
file_path = os.path.join(dst_dir, file_name)
if os.path.exists(file_path):
print("文件已存在,已跳过=",file_path)
return None
temp_path = os.path.join(dst_dir, file_name+".downloads")
repair_count = 1
try:
response = requests.get(
image_url, headers=headers, timeout=timeout, proxies=proxies)
while response.status_code != 200 and repair_count <= 5:
time.sleep(3)
time.sleep(0.7)
download_image(image_url,dst_dir,file_name)
ntfy.sendMsg(f'重试:第{repair_count}次 {image_url}')
repair_count += 1
@@ -83,4 +86,4 @@ def download_images(image_urls, dst_dir, file_prefix="img", concurrency=50, time
future_list.append(executor.submit(
download_image, image_url, dst_dir, file_name, timeout, proxy_type, proxy))
count += 1
concurrent.futures.wait(future_list, timeout=360)
concurrent.futures.wait(future_list, timeout)
+277
View File
@@ -0,0 +1,277 @@
import hashlib
import json,os,time,random,shutil
import re,math
import execjs
from utils.NetUtils import netUtils
from utils.HtmlUtils import htmlUtils
from utils.ImageUtils import imageUtils
from utils.comic.ComicInfo import comicInfo
from utils.CBZUtils import CBZUtils
from utils.downloader import download_images
from utils.Ntfy import ntfy
from utils.VerUtils import verUtils
from PIL import Image
class comicEntity:
count_chapter = 0
aid = None
repeat = 0
@classmethod
def downladsComcis(cls,url):
#漫画名
comic_href_list = htmlUtils.xpathData("//div[@class='thumb-overlay-albums']/a/@href",url,update=True)
comics_name = htmlUtils.xpathData("//span[@class='video-title title-truncate m-t-5']/text()")
len_books = len(comic_href_list)
base_url = comicInfo.getBaseUrl(url)
for x in range(0, len_books):
book_name = comics_name[x]
comicInfo.setComicName(book_name)
comic_href = base_url+comic_href_list[x]
random_int = random.randint(5,20)
dir_conf_comic = comicInfo.getDirConfComic()
if not os.path.exists(dir_conf_comic):
ntfy.sendMsg(f"{random_int}秒后开始下载 漫画:{book_name}")
time.sleep(random_int)
else:
ntfy.sendMsg(f"已存在 漫画:{book_name}")
cls.oneComic(comic_href, random.uniform(0,10))
#print(books)
#for comicHref in comicsHref:
# cls.oneComic(comicHref,random.uniform(10,20))
@classmethod
def oneComic(cls,c_url,sleep=None):
nums = htmlUtils.xpathData("//div[@class='row']/div[6]/div[1]/div[1]/ul[contains(@class,'btn-toolbar')]/a",url=c_url,update=True)
book_name = htmlUtils.xpathData("//div[@itemprop='name']/h1[@id='book-name']/text()")
book_name = re.sub(r'[\\\/\|\(\)\~\?\.\:\:\-\*\<\>]', '', book_name)
tags = htmlUtils.xpathData("//div[@class='row']/div[@class='col-lg-7']/div[1]/div[@class='tag-block']/span[@data-type='tags']/a[@class='btn btn-sm btn-primary']/text()")
author = htmlUtils.xpathData("//div[@class='row']/div[@class='col-lg-7']/div[1]/div[@class='tag-block']/span[@data-type='author']/a[@class='btn btn-sm btn-primary']/text()")
book_msg = htmlUtils.xpathData("//div[@class='row']/div[@class='col-lg-7']/div[1]/div[@class='p-t-5 p-b-5']/text()")
jmid = book_msg[0]
dep = str(book_msg[1]).replace("叙述:","")
icon = htmlUtils.xpathData("//div[@class='row']//div[@class='thumb-overlay']/img/@src",num=0)
comicInfo.setIcon(icon)
comicInfo.setHomePage(c_url)
comicInfo.setComicName(book_name)
comicInfo.setAuthor(author)
comicInfo.setDep(dep)
comicInfo.setTags(tags)
comicInfo.setTag(tags)
comicInfo.setCBS("韩漫")
comicInfo.setLang("zh")
albumid = re.search(r'/album/(\d+)', c_url).group(1)
referer = re.search(r'(https://\w+\.\w+)/', c_url).group(1)
if nums:
list_chapter_name = []
list_chapter_href = []
cls.count_chapter = 0
for i in nums:
photo_name_list = i.xpath("li/text()")[0].split()
photo_date = i.xpath("li/span/text()")[0].split()
#print(re.findall(r'[\u4E00-\u9FA5]+.*?', i.xpath("li/text()")[0]))
try:
if re.findall(r'[\u4E00-\u9FA5]', photo_name_list[2]):
photo_name=re.sub(r'\s','',photo_name_list[0])+' '+photo_name_list[2]
else:photo_name=re.sub(r'\s','',photo_name_list[0])
except Exception as e:
photo_name = re.sub(r'\s', '', photo_name_list[0])
photo_name = re.sub(r'[\\\/\|\(\)\~\?\.\:\:\-\*\<\>\-]', '',photo_name)
#print(photo_name)
photoid=i.attrib['data-album']
cls.aid = photoid
comicInfo.setChapterName(photo_name)
comicInfo.setDate(photo_date[0],split='-')
comicInfo.setWeb(referer+i.attrib['href'])
is_scramble = False
if int(photoid) > 220980:
is_scramble = True
if not comicInfo.nextExistsGetPath("done_"):
comicEntity.comicChapter(referer+i.attrib['href'],scramble=is_scramble,sleep=random.randint(5,15))
#存在就校验CBZ包是否完整
if comicInfo.nextExistsGetPath("done_"):
verUtils.verCBZ()
cls.count_chapter += 1
#一本漫画下载后等待
#清空文件夹
path_dir_comic = comicInfo.getDirComic()
if os.path.exists(path_dir_comic):
shutil.rmtree(path_dir_comic)
if sleep != None:
time.sleep(sleep)
'''
读取某章节下所有图片
'''
@classmethod
def comicChapter(cls,chapter_url,scramble=None,sleep=None):
is_next = True
# try:
is_next = cls.Onechapter(chapter_url,scramble)
#进入下个阶段
if comicInfo.nextExistsGetPath("down_"):
#章节图片全部下载后,调用下载封面
netUtils.downloadComicIcon()
#下个阶段
if comicInfo.nextExistsGetPath("cbz_"):
time.sleep(0.1)
#下载后自动打包
is_next = CBZUtils.packAutoComicChapterCBZ()
#完成删除原文件
remove_path = comicInfo.getDirComicChapter()
shutil.rmtree(remove_path)
print(f"文件已删除: {remove_path}")
# except:
# ntfy.sendMsg(f"{comicInfo.getComicName()} 下载出错了")
# is_next = False
#ntfy.sendMsg(f"预计总章节大小:{cls.count_chapter + 1} / "+ str(comicInfo.getLenChapters()))
if sleep != None and is_next == True:
ntfy.sendMsg(f"{sleep} 秒后开始下载下一个章节")
time.sleep(sleep)
@classmethod
def Onechapter(cls,chapter_url,scramble=None):
if not str(chapter_url).startswith("http"):
chapter_url = comicInfo.getBaseUrl() + chapter_url
try:
is_next = cls.comicChapterDownload(chapter_url,scramble)
except:
htmlUtils.remove_HtmlCache(chapter_url)
cls.repeat = 0
is_next = cls.comicChapterDownload(chapter_url,scramble)
comicInfo.nextInfoToImgChapter()
#下载完成后, 开始解密图片
if scramble:
#获取章节图片路径
chapter_dir = comicInfo.getDirComicChapter()
dirs = os.listdir(chapter_dir)
for img in dirs:
is_scramble = str(img).startswith("scramble=")
if is_scramble:
c_path = os.path.join(chapter_dir, img)
#imageUtils.getScrambleImage(c_path)
cls.encode_scramble_image(c_path)
#进入下一阶段
comicInfo.nextImgToDownloadChapter()
return is_next
@classmethod
def comicChapterDownload(cls,chapter_url,c_scramble):
img_list =htmlUtils.xpathData("//div[@class='panel-body']/div/div[contains(@class,'center')]/img/@data-original",url=chapter_url,update=True)
pages_imgs =htmlUtils.xpathData("//div[@class='center scramble-page']/@id",url=chapter_url)
comicInfo.setPages(pages_imgs)
comicInfo.writeComicInfoXML(comicInfo.str_chapter)
#print("img_list:",len(img_list))
list_img = []
list_file_name = []
for i in img_list:
img_url= i
img_name = os.path.basename(img_url).split('.')[0]
if c_scramble:
img_name = "scramble="+str(cls.get_scramble_num(cls.aid,img_name))+"_"+img_name
#path_img = "%s\\%s.jpg" % (cls.aid, img_name)
path_img = "%s.jpg" % (img_name)
list_img.append(img_url)
list_file_name.append(path_img)
comicInfo.setChapterImgs(list_img)
#保存图像
comicInfo.nextSaveInfoChapter(comicInfo.str_chapter, list_img)
is_next = verUtils.verNextCBZ(list_img)
list_shunt = ["?shunt=2","?shunt=1","?shunt=3",""]
while not is_next:
time.sleep(1)
download_images(list_img,comicInfo.getDirComicChapter(), filesName=list_file_name,timeout=1500)
file_imgs = os.listdir(comicInfo.getDirComicChapter())
count_jpg = ",".join(file_imgs).split(".jpg")
is_next = len(count_jpg)-1 == len(list_img)
cls.repeat += 1
if cls.repeat > 3:
url = list_shunt[cls.repeat % len(list_shunt)]
print("分流中=",url)
cls.comicChapterDownload(str(chapter_url).split("?")[0]+url,c_scramble)
return True
@classmethod
def get_md5(cls,num):
result1 = hashlib.md5(num.encode()).hexdigest()
print('get_md5-', result1)
return result1
@classmethod
def get_scramble_num(cls,e, t):
#print(type(e),e, type(t),t)
a = 10
try:
num_dict = {}
for i in range(10):
num_dict[i] = i * 2 + 2
if (int(e) >= 268850):
n = str(e) + t;
# switch(n=(n = (n = md5(n)).substr(-1)), n %= 10) {
#print("n=",n)
tmp = ord(cls.get_md5(n)[-1])
result = num_dict[tmp % 10]
a = result
return a
except Exception as e:
print(e.__traceback__.tb_lineno,e)
return False
@classmethod
def encode_scramble_image(cls,imgpath):
image = Image.open(imgpath)
w, h = image.size
#image.show()
file_str = str(imgpath).split("=")
#10_29.jpg
base_dir = file_str[0].replace("scramble","")
base_name = file_str[-1]
base_fn = base_name.split("_")
save_name = base_fn[1]
save_name_delesu = save_name.split(".")[0]
blocks = int(base_fn[0])
img_type = os.path.basename(imgpath).split('.')[-1]
save_path = os.path.join(os.path.dirname(imgpath),save_name_delesu+"."+img_type)
# print(type(aid),type(img_name))
if blocks:
s = blocks # 随机值
# print(s)
l = h % s # 切割最后多余的值
box_list = []
hz = 0
for i in range(s):
c = math.floor(h / s)
g = i * c
hz += c
h2 = h - c * (i + 1) - l
if i == 0:
c += l;hz += l
else:
g += l
box_list.append((0, h2, w, h - g))
# print(box_list,len(box_list))
item_width = w
# box_list.reverse() #还原切图可以倒序列表
# print(box_list, len(box_list))
newh = 0
image_list = [image.crop(box) for box in box_list]
# print(box_list)
newimage = Image.new("RGB", (w, h))
for image in image_list:
# image.show()
b_w, b_h = image.size
newimage.paste(image, (0, newh))
newh += b_h
newimage.save(save_path)
if os.path.exists(imgpath):
os.remove(imgpath)
+1 -1
View File
@@ -101,7 +101,7 @@ class comicEntity:
shutil.rmtree(path_dir_comic)
if sleep != None:
time.sleep(sleep)
'''
读取某章节下所有图片