From fa0e929a1a5be1fe50df9fb226c59a3710a0aa5a Mon Sep 17 00:00:00 2001 From: caiwx86 Date: Sat, 14 Jan 2023 17:49:40 +0800 Subject: [PATCH] fix --- main.py | 2 + utils/comic/ComicInfo.py | 13 +++- utils/comic/PathStr.py | 11 +++- utils/entity/BaoZi.py | 54 ++++++++++++++++ utils/entity/BaseComicEntity.py | 1 + utils/entity/JM.py | 74 ++++++++++++++++++++++ utils/entity/JMTI.py | 4 +- utils/entity/RouMan.py | 9 ++- utils/entity/down/JM.py | 108 ++++++++++++++++++++++++++++++++ 9 files changed, 267 insertions(+), 9 deletions(-) create mode 100644 utils/entity/BaoZi.py create mode 100644 utils/entity/JM.py create mode 100644 utils/entity/down/JM.py diff --git a/main.py b/main.py index 43d4547..f719163 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,6 @@ import os from utils.entity.RouMan import comicEntity +from utils.comic.PathStr import pathStr #from utils.entity.JMTI import comicEntity def comics(): @@ -7,6 +8,7 @@ def comics(): comicEntity.downladsComcis("https://rm01.xyz/books?&page="+str(x)) if __name__ == '__main__': +# pathStr.setComicMainPath("JM") comics() # os.environ["http_proxy"] = "http://127.0.0.1:7890" # os.environ["https_proxy"] = "http://127.0.0.1:7890" diff --git a/utils/comic/ComicInfo.py b/utils/comic/ComicInfo.py index 1c95971..6c8d1c9 100644 --- a/utils/comic/ComicInfo.py +++ b/utils/comic/ComicInfo.py @@ -39,6 +39,7 @@ class comicInfo(): str_web = None str_list_img = None str_files_img = None + str_value1 = None chapter_node = None comicName_node = None @@ -132,6 +133,14 @@ class comicInfo(): def setChapterListImg(cls,value): cls.str_list_img=value + @classmethod + def setValue1(cls,value): + cls.str_value1 = value + + @classmethod + def getValue1(cls): + return cls.str_value1 + @classmethod def getChapterListImg(cls): return cls.str_list_img @@ -252,7 +261,7 @@ class comicInfo(): @classmethod def getDirCBZComic(cls): if not cls.str_comicName == None: - return os.path.join(pathStr.base_CBZ, cls.str_comicName) + return os.path.join(pathStr.base_cbz, cls.str_comicName) else: print("comicName不存在，退出中") exit() @@ -260,7 +269,7 @@ class comicInfo(): @classmethod def getDirCBZComicChapter(cls): if not cls.str_comicName == None and not cls.str_chapter == None: - return os.path.join(pathStr.base_CBZ,cls.str_comicName,cls.str_chapter) + return os.path.join(pathStr.base_cbz,cls.str_comicName,cls.str_chapter) else: print("comicName不存在，退出中") exit() diff --git a/utils/comic/PathStr.py b/utils/comic/PathStr.py index 7c5e846..31fe610 100644 --- a/utils/comic/PathStr.py +++ b/utils/comic/PathStr.py @@ -1,16 +1,23 @@ import os,datetime from time import strftime class pathStr: + comic_url_main = None #base_comic_out = "COMICOUT" #base_comic_out = os.path.join("/mnt", "bigTComics","JM") + if comic_url_main != None: + base_comic_out = os.path.join("/mnt", "bigTComics", comic_url_main) base_comic_out = os.path.join("/mnt", "bigTComics") - base_CBZ = os.path.join(base_comic_out,"CBZ") + base_cbz = os.path.join(base_comic_out,"CBZ") base_comic_img = os.path.join(base_comic_out,"outputComic") base_conf_path = os.path.join(base_comic_out,".conf") base_html_cache = os.path.join(base_comic_out,"html_cache") base_html_chapter = os.path.join(base_comic_out,"html_updated") base_comic_update = os.path.join(base_conf_path,"comic_update") - + + @classmethod + def setComicMainPath(cls,value): + cls.comic_url_main = value + @classmethod def base_html_week(cls): date_path = cls.getDatePath() diff --git a/utils/entity/BaoZi.py b/utils/entity/BaoZi.py new file mode 100644 index 0000000..1c1d487 --- /dev/null +++ b/utils/entity/BaoZi.py @@ -0,0 +1,54 @@ +import json +from utils.HtmlUtils import htmlUtils +from utils.comic.ComicInfo import comicInfo +from utils.downloader import download_images +from utils.entity.BaseComicEntity import baseComic + +class comicEntity: + @classmethod + def baseComicData(cls,url,update=False): + data = htmlUtils.xpathData('//script[@id="__NEXT_DATA__"]/text()',url=url,update=update) + data = json.loads(data[0]) + data = data.get("props") + x = data.get("pageProps") + return x + + @classmethod + def downladsComcis(cls,url): + #漫画名 + x = cls.baseComicData(url) + books = x.get("books") + len_books = len(books) + base_url = comicInfo.getBaseUrl(url) + for x in range(0, len_books): + book = books[x] + book_id = book.get("id") + book_name = book.get("name") + updated = book.get("updatedAt") + comic_href = base_url+"/books/"+book_id + href = baseComic.downladsComcis(book_name=book_name,comic_href=comic_href,updated=updated) + cls.oneComic(href) + comicInfo.updateComicDate() + + @classmethod + def oneComic(cls,c_url,sleep=None): + #漫画名 + title = htmlUtils.xpathData('//div[@class="gb-inside-container"]/h1[contains(@class,"gb-headline-text")]/text()',url=c_url,update=True) + #别名 + #alias = htmlUtils.xpathData('//span[contains(@class,"bookid_alias")]/text()',num=1) + #icon = htmlUtils.xpathData('//div[contains(@class,"gb-container")]/div[@class="gb-inside-container"]/img[contains(@class,"dynamic-featured-image")]/@src') + icon = htmlUtils.xpathData('//noscript/img[contains(@class,"dynamic-featured-image")]/@src') + author = htmlUtils.xpathData('//div[@class="gb-inside-container"]/div[@class="author-content"]/a/text()') + tags = htmlUtils.xpathData('//div[@class="gb-inside-container"]/div[@class="genres-content"]/a/text()') + dep = htmlUtils.xpathData('//div[@class="gb-inside-container"]/div[@class="descrip_manga_info"]//p/text()') + update_date = htmlUtils.xpathData('//div[@class="chapter-content-listing"]//a[@class="wp-manga-chapter"]//span[@class="chapter-release-date"]/i/text()') + c_url = str(c_url).replace("manga","chapterlist")+"#" + chapters = htmlUtils.xpathData('//div[@class="chapter-content-listing"]//a[@class="wp-manga-chapter"]/text()',not_eq=' ',url=c_url) + chapter_href = htmlUtils.xpathData('//div[@class="chapter-content-listing"]//a[@class="wp-manga-chapter"]/@href',url=c_url) + c_url = c_url+"#" + chapters = htmlUtils.xpathData('//div[@class="chapter-content-listing"]//a[@class="wp-manga-chapter"]/text()',not_eq=' ',url=c_url) + chapter_href = htmlUtils.xpathData('//div[@class="chapter-content-listing"]//a[@class="wp-manga-chapter"]/@href',url=c_url) + + baseComic.oneComic(url=c_url,title=title,author=author, + icon=icon,tags=tags,dep=dep,chapters=chapters,chapter_href=chapter_href, + alias=None) \ No newline at end of file diff --git a/utils/entity/BaseComicEntity.py b/utils/entity/BaseComicEntity.py index d02412c..dcae5c9 100644 --- a/utils/entity/BaseComicEntity.py +++ b/utils/entity/BaseComicEntity.py @@ -29,6 +29,7 @@ class baseComic: return comic_href else: ntfy.sendMsg(f"{book_name} 已是最新") + return None #print(books) #for comicHref in comicsHref: diff --git a/utils/entity/JM.py b/utils/entity/JM.py new file mode 100644 index 0000000..0839cb9 --- /dev/null +++ b/utils/entity/JM.py @@ -0,0 +1,74 @@ +import json +import re +from utils.HtmlUtils import htmlUtils +from utils.comic.ComicInfo import comicInfo +from utils.downloader import download_images +from utils.entity.BaseComicEntity import baseComic + +class comicEntity: + @classmethod + def baseComicData(cls,url,update=False): + data = htmlUtils.xpathData('//script[@id="__NEXT_DATA__"]/text()',url=url,update=update) + data = json.loads(data[0]) + data = data.get("props") + x = data.get("pageProps") + return x + + @classmethod + def downladsComcis(cls,url): + #漫画名 + x = cls.baseComicData(url) + books = x.get("books") + len_books = len(books) + base_url = comicInfo.getBaseUrl(url) + for x in range(0, len_books): + book = books[x] + book_id = book.get("id") + book_name = book.get("name") + updated = book.get("updatedAt") + comic_href = base_url+"/books/"+book_id + href = baseComic.downladsComcis(book_name=book_name,comic_href=comic_href,updated=updated) + cls.oneComic(href) + comicInfo.updateComicDate() + + @classmethod + def oneComic(cls,c_url,sleep=None): + nums = htmlUtils.xpathData("//div[@class='row']/div[6]/div[1]/div[1]/ul[contains(@class,'btn-toolbar')]/a",url=c_url,update=True) + book_name = htmlUtils.xpathData("//div[@itemprop='name']/h1[@id='book-name']/text()",num=0) + title = re.sub(r'[\\\/\|\(\)\~\?\.\:\：\-\*\<\>]', '', book_name) + tags = htmlUtils.xpathData("//div[@class='row']/div[@class='col-lg-7']/div[1]/div[@class='tag-block']/span[@data-type='tags']/a[@class='btn btn-sm btn-primary']/text()") + author = htmlUtils.xpathData("//div[@class='row']/div[@class='col-lg-7']/div[1]/div[@class='tag-block']/span[@data-type='author']/a[@class='btn btn-sm btn-primary']/text()") + book_msg = htmlUtils.xpathData("//div[@class='row']/div[@class='col-lg-7']/div[1]/div[@class='p-t-5 p-b-5']/text()") + jmid = book_msg[0] + dep = str(book_msg[1]).replace("叙述：","") + icon = htmlUtils.xpathData("//div[@class='row']//div[@class='thumb-overlay']/img/@src",num=0) + + referer = re.search(r'(https://\w+\.\w+)/', c_url).group(1) + if nums: + list_chapter_name = [] + list_chapter_href = [] + list_chapter_update = [] + cls.count_chapter = 0 + + for i in nums: + photo_name_list = i.xpath("li/text()")[0].split() + photo_date = i.xpath("li/span/text()")[0].split() + #print(re.findall(r'[\u4E00-\u9FA5]+.*?', i.xpath("li/text()")[0])) + try: + if re.findall(r'[\u4E00-\u9FA5]', photo_name_list[2]): + photo_name=re.sub(r'\s','',photo_name_list[0])+' '+photo_name_list[2] + else:photo_name=re.sub(r'\s','',photo_name_list[0]) + except Exception as e: + photo_name = re.sub(r'\s', '', photo_name_list[0]) + photo_name = re.sub(r'[\\\/\|\(\)\~\?\.\:\：\-\*\<\>\-]', '',photo_name) + #print(photo_name) + photoid=i.attrib['data-album'] + cls.aid = photoid + comicInfo.setValue1(cls.aid) + list_chapter_name.append(photo_name) + list_chapter_href.append(referer+i.attrib['href']) + list_chapter_update.append(photo_date[0]) + + baseComic.oneComic(url=c_url,title=title,author=author, + icon=icon,tags=tags,dep=dep,chapters=list_chapter_name,chapter_href=list_chapter_href, + alias=None) \ No newline at end of file diff --git a/utils/entity/JMTI.py b/utils/entity/JMTI.py index 7bc725a..fd97044 100644 --- a/utils/entity/JMTI.py +++ b/utils/entity/JMTI.py @@ -5,6 +5,7 @@ from utils.HtmlUtils import htmlUtils from utils.ImageUtils import imageUtils from utils.comic.ComicInfo import comicInfo from utils.CBZUtils import CBZUtils +from utils.CBZUtils import verUtils from utils.downloader import download_images from utils.Ntfy import ntfy from PIL import Image @@ -50,7 +51,7 @@ class comicEntity: @classmethod def oneComic(cls,c_url,sleep=None): nums = htmlUtils.xpathData("//div[@class='row']/div[6]/div[1]/div[1]/ul[contains(@class,'btn-toolbar')]/a",url=c_url,update=True) - book_name = htmlUtils.xpathData("//div[@itemprop='name']/h1[@id='book-name']/text()") + book_name = htmlUtils.xpathData("//div[@itemprop='name']/h1[@id='book-name']/text()",num=0) book_name = re.sub(r'[\\\/\|\(\)\~\?\.\:\：\-\*\<\>]', '', book_name) tags = htmlUtils.xpathData("//div[@class='row']/div[@class='col-lg-7']/div[1]/div[@class='tag-block']/span[@data-type='tags']/a[@class='btn btn-sm btn-primary']/text()") author = htmlUtils.xpathData("//div[@class='row']/div[@class='col-lg-7']/div[1]/div[@class='tag-block']/span[@data-type='author']/a[@class='btn btn-sm btn-primary']/text()") @@ -65,7 +66,6 @@ class comicEntity: comicInfo.setAuthor(author) comicInfo.setDep(dep) comicInfo.setTags(tags) - comicInfo.setTag(tags) comicInfo.setCBS("韩漫") comicInfo.setLang("zh") albumid = re.search(r'/album/(\d+)', c_url).group(1) diff --git a/utils/entity/RouMan.py b/utils/entity/RouMan.py index 77bd1fc..3831c8a 100644 --- a/utils/entity/RouMan.py +++ b/utils/entity/RouMan.py @@ -25,10 +25,13 @@ class comicEntity: book_id = book.get("id") book_name = book.get("name") updated = book.get("updatedAt") - comic_href = base_url+"/books/"+book_id + comic_href = base_url+"/books/"+book_id + comicInfo.setUpdateAt(updated) + comicInfo.setComicName(book_name) href = baseComic.downladsComcis(book_name=book_name,comic_href=comic_href,updated=updated) - cls.oneComic(href) - comicInfo.updateComicDate() + if href != None: + cls.oneComic(href) + comicInfo.updateComicDate() @classmethod def oneComic(cls,c_url,sleep=None): diff --git a/utils/entity/down/JM.py b/utils/entity/down/JM.py new file mode 100644 index 0000000..3ebdb81 --- /dev/null +++ b/utils/entity/down/JM.py @@ -0,0 +1,108 @@ +import hashlib +import json +import os +from utils.HtmlUtils import htmlUtils +from utils.ImageUtils import imageUtils +from utils.comic.ComicInfo import comicInfo +from utils.Ntfy import ntfy + +class comicCommon: + @classmethod + def comicChapterDownload(cls,chapter_url): + chapter_url = chapter_url+"?shunt=2" + img_list =htmlUtils.xpathData("//div[@class='panel-body']/div/div[contains(@class,'center scramble-page')]/img/@data-original",url=chapter_url,update=True) + pages_imgs =htmlUtils.xpathData("//div[@class='center scramble-page']/@id",url=chapter_url) + comicInfo.setPages(pages_imgs) + comicInfo.writeComicInfoXML(comicInfo.str_chapter) + #print("img_list:",len(img_list)) + list_img = [] + list_file_name = [] + for i in img_list: + img_url= i + img_name = os.path.basename(img_url).split('.')[0] + aid = int(comicInfo.getValue1()) + if aid > 220980: + #if is_scramble: + img_name = "scramble="+str(cls.get_scramble_num(aid,img_name))+"_"+img_name + #path_img = "%s\\%s.jpg" % (cls.aid, img_name) + path_img = "%s.jpg" % (img_name) + list_img.append(img_url) + list_file_name.append(path_img) + comicInfo.comicChapterDownload(list_img,list_file_name) + + @classmethod + def get_md5(cls,num): + result1 = hashlib.md5(num.encode()).hexdigest() + print('get_md5-', result1) + return result1 + + @classmethod + def get_scramble_num(cls,e, t): + #print(type(e),e, type(t),t) + a = 10 + try: + num_dict = {} + for i in range(10): + num_dict[i] = i * 2 + 2 + if (int(e) >= 268850): + n = str(e) + t; + # switch(n=(n = (n = md5(n)).substr(-1)), n %= 10) { + #print("n=",n) + tmp = ord(cls.get_md5(n)[-1]) + result = num_dict[tmp % 10] + a = result + return a + except Exception as e: + print(e.__traceback__.tb_lineno,e) + return False + + @classmethod + def encode_scramble_image(cls,imgpath): + image = Image.open(imgpath) + w, h = image.size + #image.show() + file_str = str(imgpath).split("=") + #10_29.jpg + base_dir = file_str[0].replace("scramble","") + base_name = file_str[-1] + base_fn = base_name.split("_") + save_name = base_fn[1] + save_name_delesu = save_name.split(".")[0] + blocks = int(base_fn[0]) + img_type = os.path.basename(imgpath).split('.')[-1] + save_path = os.path.join(os.path.dirname(imgpath),save_name_delesu+"."+img_type) + # print(type(aid),type(img_name)) + if blocks: + s = blocks # 随机值 + # print(s) + l = h % s # 切割最后多余的值 + box_list = [] + hz = 0 + for i in range(s): + c = math.floor(h / s) + g = i * c + hz += c + h2 = h - c * (i + 1) - l + if i == 0: + c += l;hz += l + else: + g += l + box_list.append((0, h2, w, h - g)) + + # print(box_list,len(box_list)) + item_width = w + # box_list.reverse() #还原切图可以倒序列表 + # print(box_list, len(box_list)) + newh = 0 + image_list = [image.crop(box) for box in box_list] + # print(box_list) + newimage = Image.new("RGB", (w, h)) + for image in image_list: + # image.show() + b_w, b_h = image.size + newimage.paste(image, (0, newh)) + + newh += b_h + newimage.save(save_path) + if os.path.exists(imgpath): + os.remove(imgpath) \ No newline at end of file