This commit is contained in:
cwx
2023-01-10 07:07:09 +08:00
parent 906184f8c6
commit 9658ebdfa3
16 changed files with 397 additions and 1179 deletions
+181
View File
@@ -0,0 +1,181 @@
import json,os,time,random,shutil
from utils.HtmlUtils import htmlUtils
from utils.ImageUtils import imageUtils
from utils.comic.ComicInfo import comicInfo
from utils.CBZUtils import CBZUtils
from utils.downloader import download_images
from utils.downloader import download_comic_icon
from utils.Ntfy import ntfy
from utils.CBZUtils import verUtils
from utils.entity.down.RouMan import comicCommon
class baseComic:
count_chapter = 0
@classmethod
def downladsComcis(cls,book_name,comic_href,updated=None):
comicInfo.setComicName(book_name)
if updated != None:
comicInfo.setUpdateAt(updated)
random_int = random.randint(5,20)
comicInfo.setComicName(book_name)
dir_conf_comic = comicInfo.getDirConfComic()
if not os.path.exists(dir_conf_comic):
ntfy.sendMsg(f"{random_int}秒后开始下载 漫画:{book_name}")
time.sleep(random_int)
else:
ntfy.sendMsg(f"已存在 漫画:{book_name}")
if comicInfo.isUpdateComic():
return comic_href
else:
ntfy.sendMsg(f"{book_name} 已是最新")
#print(books)
#for comicHref in comicsHref:
# cls.oneComic(comicHref,random.uniform(10,20))
@classmethod
def oneComic(cls,url,title,author,icon,tags,dep,chapters,chapter_href,alias=None,genre="韩漫",lang="zh",sleep=None):
author = str(author).replace("&",",").replace(" ",",")
comicInfo.setHomePage(url)
comicInfo.setComicName(str(title))
if alias != None:
comicInfo.setComicNames(title+","+alias)
comicInfo.setAuthor(author)
comicInfo.setIcon(icon)
comicInfo.setTags(tags)
comicInfo.setDep(dep)
#comicInfo.setCBS("韩漫")
comicInfo.setGenre(genre)
comicInfo.setLang(lang)
comicInfo.setListChapter(chapters)
#comicUtils.setComic(title,alias,icon,author,tags,action,dep,update_date,chapters,chapter_href)
cls.count_chapter = 0
for href in chapter_href:
chapter = chapters[cls.count_chapter]
comicInfo.setChapterName(chapter)
if not comicInfo.nextExistsGetPath("done_"):
cls.comicChapter(href,scramble=True,sleep=random.randint(5,15))
#存在就校验CBZ包是否完整
if comicInfo.nextExistsGetPath("done_"):
verUtils.verCBZ()
cls.count_chapter += 1
#一本漫画下载后等待
#清空文件夹
path_dir_comic = comicInfo.getDirComic()
if os.path.exists(path_dir_comic):
shutil.rmtree(path_dir_comic)
if sleep != None:
time.sleep(sleep)
'''
读取某章节下所有图片
'''
@classmethod
def comicChapter(cls,chapter_url,scramble=None,sleep=None):
is_next = True
#try:
is_next = cls.Onechapter(chapter_url,scramble)
#进入下个阶段
if comicInfo.nextExistsGetPath("down_"):
#章节图片全部下载后,调用下载封面
download_comic_icon()
#下个阶段
if comicInfo.nextExistsGetPath("cbz_"):
time.sleep(0.1)
#下载后自动打包
is_next = CBZUtils.packAutoComicChapterCBZ()
#完成删除原文件
remove_path = comicInfo.getDirComicChapter()
if os.path.exists(remove_path):
shutil.rmtree(remove_path)
print(f"文件已删除: {remove_path}")
#except Exception as e:
# print(e)
# ntfy.sendMsg(f"{comicInfo.getComicName()} 下载出错了")
# is_next = False
ntfy.sendMsg(f"预计总章节大小:{cls.count_chapter + 1} / "+ str(comicInfo.getLenChapters()))
if sleep != None and is_next == True:
ntfy.sendMsg(f"{sleep} 秒后开始下载下一个章节")
time.sleep(sleep)
@classmethod
def Onechapter(cls,chapter_url,scramble=None):
if not str(chapter_url).startswith("http"):
chapter_url = comicInfo.getBaseUrl() + chapter_url
try:
is_next = cls.comicChapterDownload(chapter_url)
except:
htmlUtils.remove_HtmlCache(chapter_url)
is_next = cls.comicChapterDownload(chapter_url)
comicInfo.nextInfoToImgChapter()
#下载完成后, 开始解密图片
chapter_dir = comicInfo.getDirComicChapter()
if scramble and os.path.exists(chapter_dir):
#获取章节图片路径
dirs = os.listdir(chapter_dir)
for img in dirs:
if img.startswith("scramble="):
c_path = os.path.join(chapter_dir, img)
#imageUtils.getScrambleImage(c_path)
imageUtils.encode_scramble_image(c_path)
#进入下一阶段
comicInfo.nextImgToDownloadChapter()
return is_next
@classmethod
def comicChapterDownload(cls,url):
comicCommon.comicChapterDownload(url)
list_img = comicInfo.getChapterListImg()
files_name = comicInfo.getChapterFilesName()
#netUtils.downloadComicChapterImages(list_img,scrambles=list_scramble)
chapter_name = comicInfo.getChapter()
book_name = comicInfo.getComicName()
comicInfo.setChapterImgs(list_img)
#保存图像
comicInfo.nextSaveInfoChapter(chapter_name, list_img)
#验证数据是已存在且是否完整
cbz_path = comicInfo.getDirCBZComicChapter()+".CBZ"
is_next = True
if os.path.exists(cbz_path):
try:
cbz_size = len(CBZUtils.zip_info(cbz_path)) - 1
except:
cbz_size = 0
if len(list_img) == cbz_size:
ntfy.sendMsg(f"{book_name} {chapter_name} 数据完整,已跳过")
comicInfo.nextDoneSave(list_img)
is_next = False
else:
ntfy.sendMsg(f"{book_name} {chapter_name} 数据不完整,尝试删除配置CBZ文件后重试")
try:
if cbz_size < len(list_img) or os.path.getsize(cbz_path) < 300000:
ntfy.sendMsg(f"删除 {cbz_path}")
os.remove(cbz_path)
else:
is_next = False
except:
ntfy(f"删除失败 {cbz_path}")
if is_next:
path_comic_info = comicInfo.getPathComicInfoXML()
if not os.path.exists(path_comic_info):
#print("不存在ComicInfo.xml 生成中...")
comicInfo.setPages(files_name)
comicInfo.writeComicInfoXML(chapter_name)
ntfy.sendMsg(f"{book_name} {chapter_name} 下载中")
is_next = verUtils.verNextCBZ(list_img)
repeat = 0
while not is_next or repeat <= 3:
download_images(list_img,comicInfo.getDirComicChapter(), filesName=files_name,timeout=180)
file_imgs = os.listdir(comicInfo.getDirComicChapter())
count_jpg = ",".join(file_imgs).split(".jpg")
is_next = len(count_jpg)-1 == len(list_img)
if not is_next:
sleep_time = 3+int(repeat)*2
time.sleep(sleep_time)
ntfy.sendMsg(f"下载数据({len(count_jpg)-1}/{len(list_img)})不完整,{sleep_time}秒钟后尝试第{repeat}")
repeat += 1
return is_next
+1 -6
View File
@@ -1,18 +1,13 @@
import hashlib
import json,os,time,random,shutil
import re,math
import execjs
from utils.NetUtils import netUtils
from utils.HtmlUtils import htmlUtils
from utils.ImageUtils import imageUtils
from utils.comic.ComicInfo import comicInfo
from utils.CBZUtils import CBZUtils
from utils.downloader import download_images
from utils.Ntfy import ntfy
from utils.VerUtils import verUtils
from PIL import Image
import get_jm_url
class comicEntity:
count_chapter = 0
@@ -21,7 +16,7 @@ class comicEntity:
@classmethod
def baseReUrl(cls,url):
newurl_list=get_jm_url.app()
newurl_list=""
if newurl_list:
if re.findall(r'https://(.*?)/\w+/\d+/',url)[0] not in newurl_list:
for newurl in newurl_list:
+8 -215
View File
@@ -1,17 +1,10 @@
import json,os,time,random,shutil
from utils.NetUtils import netUtils
import json
from utils.HtmlUtils import htmlUtils
from utils.ImageUtils import imageUtils
from utils.comic.ComicInfo import comicInfo
from utils.CBZUtils import CBZUtils
from utils.downloader import download_images
from utils.Ntfy import ntfy
from utils.VerUtils import verUtils
from utils.entity.BaseComicEntity import baseComic
class comicEntity:
count_chapter = 0
@classmethod
def baseComicData(cls,url,update=False):
data = htmlUtils.xpathData('//script[@id="__NEXT_DATA__"]/text()',url=url,update=update)
@@ -32,27 +25,10 @@ class comicEntity:
book_id = book.get("id")
book_name = book.get("name")
updated = book.get("updatedAt")
comicInfo.setComicName(book_name)
comicInfo.setUpdateAt(updated)
comic_href = base_url+"/books/"+book_id
random_int = random.randint(5,20)
comicInfo.setComicName(book_name)
dir_conf_comic = comicInfo.getDirConfComic()
if not os.path.exists(dir_conf_comic):
ntfy.sendMsg(f"{random_int}秒后开始下载 漫画:{book_name}")
time.sleep(random_int)
else:
ntfy.sendMsg(f"已存在 漫画:{book_name}")
if comicInfo.isUpdateComic():
cls.oneComic(comic_href, random.uniform(0,3))
comicInfo.updateComicDate()
else:
ntfy.sendMsg(f"{book_name} 已是最新")
#print(books)
#for comicHref in comicsHref:
# cls.oneComic(comicHref,random.uniform(10,20))
href = baseComic.downladsComcis(book_name=book_name,comic_href=comic_href,updated=updated)
cls.oneComic(href)
@classmethod
def oneComic(cls,c_url,sleep=None):
#漫画名
@@ -69,189 +45,6 @@ class comicEntity:
chapters = htmlUtils.xpathData('//div[contains(@class,"bookid_chapterBox")]//div[contains(@class,"bookid_chapter")]/a/text()')
chapter_href = htmlUtils.xpathData('//div[contains(@class,"bookid_chapterBox")]//div[contains(@class,"bookid_chapter")]/a/@href')
author = str(author).replace("&",",").replace(" ",",")
comicInfo.setHomePage(c_url)
comicInfo.setComicName(str(title))
comicInfo.setComicNames(title+","+alias)
comicInfo.setAuthor(author)
comicInfo.setIcon(icon)
comicInfo.setTags(tags)
comicInfo.setDep(dep)
#comicInfo.setCBS("韩漫")
comicInfo.setGenre("韩漫")
comicInfo.setLang("zh")
comicInfo.setComicNames(title+","+alias)
comicInfo.setListChapter(chapters)
#comicUtils.setComic(title,alias,icon,author,tags,action,dep,update_date,chapters,chapter_href)
cls.count_chapter = 0
for href in chapter_href:
chapter = chapters[cls.count_chapter]
comicInfo.setChapterName(chapter)
if not comicInfo.nextExistsGetPath("done_"):
comicEntity.comicChapter(href,scramble=True,sleep=random.randint(5,15))
#存在就校验CBZ包是否完整
if comicInfo.nextExistsGetPath("done_"):
verUtils.verCBZ()
cls.count_chapter += 1
#一本漫画下载后等待
#清空文件夹
path_dir_comic = comicInfo.getDirComic()
if os.path.exists(path_dir_comic):
shutil.rmtree(path_dir_comic)
if sleep != None:
time.sleep(sleep)
'''
读取某章节下所有图片
'''
@classmethod
def comicChapter(cls,chapter_url,scramble=None,sleep=None):
is_next = True
#try:
is_next = cls.Onechapter(chapter_url,scramble)
#进入下个阶段
if comicInfo.nextExistsGetPath("down_"):
#章节图片全部下载后,调用下载封面
netUtils.downloadComicIcon()
#下个阶段
if comicInfo.nextExistsGetPath("cbz_"):
time.sleep(0.1)
#下载后自动打包
is_next = CBZUtils.packAutoComicChapterCBZ()
#完成删除原文件
remove_path = comicInfo.getDirComicChapter()
if os.path.exists(remove_path):
shutil.rmtree(remove_path)
print(f"文件已删除: {remove_path}")
#except Exception as e:
# print(e)
# ntfy.sendMsg(f"{comicInfo.getComicName()} 下载出错了")
# is_next = False
ntfy.sendMsg(f"预计总章节大小:{cls.count_chapter + 1} / "+ str(comicInfo.getLenChapters()))
if sleep != None and is_next == True:
ntfy.sendMsg(f"{sleep} 秒后开始下载下一个章节")
time.sleep(sleep)
@classmethod
def Onechapter(cls,chapter_url,scramble=None):
if not str(chapter_url).startswith("http"):
chapter_url = comicInfo.getBaseUrl() + chapter_url
try:
is_next = cls.comicChapterDownload(chapter_url)
except:
htmlUtils.remove_HtmlCache(chapter_url)
is_next = cls.comicChapterDownload(chapter_url)
comicInfo.nextInfoToImgChapter()
#下载完成后, 开始解密图片
chapter_dir = comicInfo.getDirComicChapter()
if scramble and os.path.exists(chapter_dir):
#获取章节图片路径
dirs = os.listdir(chapter_dir)
for img in dirs:
if img.startswith("scramble="):
c_path = os.path.join(chapter_dir, img)
#imageUtils.getScrambleImage(c_path)
imageUtils.encode_scramble_image(c_path)
#进入下一阶段
comicInfo.nextImgToDownloadChapter()
return is_next
@classmethod
def comicChapterDownload(cls,chapter_url):
x = cls.baseComicData(chapter_url,update=True)
bookName = x.get("bookName")
chapterName = x.get("chapterName")
#fileUtils.saveConfComicChapterInfo(chapterName,x,bookName)
#if comicInfo.nextExistsGetPath("info_"):
# print(f"{bookName} {chapterName} info文件已存在跳过")
alias = x.get("alias")
description = x.get("description")
images = x.get("images")
chapterAPIPath = x.get("chapterAPIPath")
comicInfo.setComicName(bookName)
comicInfo.setChapterName(chapterName)
comicInfo.setDep(description)
if not chapterAPIPath == None:
chapterAPIPath = str(chapterAPIPath).encode('utf-8').decode('unicode_escape')
base_url = comicInfo.getBaseUrl(chapter_url)
chapterAPIUrl = base_url+chapterAPIPath
ntfy.sendMsg(f"chapterApiUrl= {chapterAPIUrl}",alert=False)
data = htmlUtils.getJSON(chapterAPIUrl)
if data != None:
data = data.get("chapter")
chapterName = data.get("name")
images = data.get("images")
if images == None:
ntfy.sendMsg(f"未获取到章节图像 comic_name={bookName} chapter={chapterName}")
tags = x.get("tags")
x = tags
count = 1
list_img = []
list_scramble = []
list_fileName = []
for image in images:
image_src = image.get("src")
scramble = image.get("scramble")
count_image = "{:0>3d}".format(count)
list_img.append(image_src)
image_src_prefix = "."+str(image_src).split(".")[-1]
if scramble:
su = "."+str(image_src).split(".")[-1]
de_str = str(image_src).split("/")[-1].replace(su,"==")
blocks = imageUtils.encodeImage(de_str)
count_image = "scramble="+str(blocks)+"_"+count_image
list_fileName.append(count_image+image_src_prefix)
count+=1
#print("count_all_img=", count)
#netUtils.downloadComicChapterImages(list_img,scrambles=list_scramble)
comicInfo.setChapterImgs(list_img)
#保存图像
comicInfo.nextSaveInfoChapter(chapterName, list_img)
#验证数据是已存在且是否完整
cbz_path = comicInfo.getDirCBZComicChapter()+".CBZ"
is_next = True
if os.path.exists(cbz_path):
try:
cbz_size = len(CBZUtils.zip_info(cbz_path)) - 1
except:
cbz_size = 0
if len(list_img) == cbz_size:
ntfy.sendMsg(f"{bookName} {chapterName} 数据完整,已跳过")
comicInfo.nextDoneSave(list_img)
is_next = False
else:
ntfy.sendMsg(f"{bookName} {chapterName} 数据不完整,尝试删除配置CBZ文件后重试")
htmlUtils.remove_HtmlCache(chapter_url)
try:
if cbz_size < len(list_img) or os.path.getsize(cbz_path) < 300000:
ntfy.sendMsg(f"删除 {cbz_path}")
os.remove(cbz_path)
else:
is_next = False
except:
ntfy(f"删除失败 {cbz_path}")
if is_next:
path_comic_info = comicInfo.getPathComicInfoXML()
if not os.path.exists(path_comic_info):
#print("不存在ComicInfo.xml 生成中...")
comicInfo.setPages(list_fileName)
comicInfo.writeComicInfoXML(chapterName)
ntfy.sendMsg(f"{bookName} {chapterName} 下载中")
is_next = verUtils.verNextCBZ(list_img)
repeat = 0
while not is_next or repeat <= 3:
download_images(list_img,comicInfo.getDirComicChapter(), filesName=list_fileName,timeout=180)
file_imgs = os.listdir(comicInfo.getDirComicChapter())
count_jpg = ",".join(file_imgs).split(".jpg")
is_next = len(count_jpg)-1 == len(list_img)
if not is_next:
sleep_time = 3+int(repeat)*2
time.sleep(sleep_time)
ntfy.sendMsg(f"下载数据({len(count_jpg)-1}/{len(list_img)})不完整,{sleep_time}秒钟后尝试第{repeat}")
repeat += 1
return is_next
baseComic.oneComic(url=c_url,title=title,author=author,
icon=icon,tags=tags,dep=dep,chapters=chapters,chapter_href=chapter_href,
alias=alias)
+63
View File
@@ -0,0 +1,63 @@
import json
from utils.HtmlUtils import htmlUtils
from utils.ImageUtils import imageUtils
from utils.comic.ComicInfo import comicInfo
from utils.Ntfy import ntfy
class comicCommon:
@classmethod
def baseComicData(cls,url,update=False):
data = htmlUtils.xpathData('//script[@id="__NEXT_DATA__"]/text()',url=url,update=update)
data = json.loads(data[0])
data = data.get("props")
x = data.get("pageProps")
return x
@classmethod
def comicChapterDownload(cls,chapter_url):
x = cls.baseComicData(chapter_url,update=True)
book_name = x.get("bookName")
chapter_name = x.get("chapterName")
#fileUtils.saveConfComicChapterInfo(chapterName,x,bookName)
#if comicInfo.nextExistsGetPath("info_"):
# print(f"{bookName} {chapterName} info文件已存在跳过")
alias = x.get("alias")
description = x.get("description")
images = x.get("images")
chapter_api_path = x.get("chapterAPIPath")
comicInfo.setComicName(book_name)
comicInfo.setChapterName(chapter_name)
comicInfo.setDep(description)
if chapter_api_path != None:
chapter_api_path = str(chapter_api_path).encode('utf-8').decode('unicode_escape')
base_url = comicInfo.getBaseUrl(chapter_url)
chapter_api_url = base_url+chapter_api_path
ntfy.sendMsg(f"chapterApiUrl= {chapter_api_url}",alert=False)
data = htmlUtils.getJSON(chapter_api_url)
if data != None:
data = data.get("chapter")
chapter_name = data.get("name")
images = data.get("images")
if images == None:
ntfy.sendMsg(f"未获取到章节图像 comic_name={book_name} chapter={chapter_name}")
count = 1
list_img = []
list_file_name = []
for image in images:
image_src = image.get("src")
scramble = image.get("scramble")
count_image = "{:0>3d}".format(count)
list_img.append(image_src)
image_src_prefix = "."+str(image_src).split(".")[-1]
if scramble:
su = "."+str(image_src).split(".")[-1]
de_str = str(image_src).split("/")[-1].replace(su,"==")
blocks = imageUtils.encodeImage(de_str)
count_image = "scramble="+str(blocks)+"_"+count_image
list_file_name.append(count_image+image_src_prefix)
count+=1
#print("count_all_img=", count)
#netUtils.downloadComicChapterImages(list_img,scrambles=list_scramble)
comicInfo.comicChapterDownload(list_img,list_file_name)