diff --git a/.gitignore b/.gitignore index cbb8ebb..ab581ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .scrapy/* images/* +json/* /**/__pycache__ \ No newline at end of file diff --git a/CBZ/私宅女主人/第1话-提供顶级服务的随行秘书.CBZ b/CBZ/私宅女主人/第1话-提供顶级服务的随行秘书.CBZ new file mode 100644 index 0000000..1caa4fc Binary files /dev/null and b/CBZ/私宅女主人/第1话-提供顶级服务的随行秘书.CBZ differ diff --git a/Comics/exporters.py b/Comics/exporters.py index 77faca6..d9f69f2 100644 --- a/Comics/exporters.py +++ b/Comics/exporters.py @@ -8,13 +8,15 @@ from Comics.items import ComicItem from Comics.settings import COMIC_INFO_XML_STORE from Comics.utils.Constant import ComicPath from scrapy.utils.python import is_listlike, to_bytes, to_unicode +from itemadapter import ItemAdapter + class ItemExporter(PythonItemExporter): def convert(self, data): if isinstance(data, bytes): return data.decode("utf-8") if isinstance(data, dict): return dict(map(self.convert, data.items())) if isinstance(data, tuple): return map(self.convert, data) - if isinstance(data, list): return [self.convert(i) for i in data] + if isinstance(data, list): return [self.convert(i) for i in data] return data def export_obj(self, obj_item): @@ -46,7 +48,12 @@ class ComicInfoXmlItemExporter(XmlItemExporter): def comic_to_info_item(self, comic_item): comic_info = {} - comic_info_dict = getattr(ComicItem, "data", 0) + info_item = ItemAdapter(ComicInfoItem()) + comic_info_dict = {} + for field in info_item.field_names(): + meta_info = info_item.get_field_meta(field).get('info') + if meta_info is not None: + comic_info_dict[meta_info] = field for key, value in ComicItem(comic_item).items(): new_key = comic_info_dict.get(key) if new_key is not None: @@ -63,7 +70,7 @@ class ComicInfoXmlItemExporter(XmlItemExporter): value = str(value).split(',') if value is not None or value != "": self._export_xml_field(name, value, depth=2, child_element=child_element) - self._beautify_indent(depth=1) + #self._beautify_indent(depth=1) return comic_info def _export_xml_field(self, name, serialized_value, depth, child_element="value"): @@ -71,8 +78,8 @@ class ComicInfoXmlItemExporter(XmlItemExporter): self.xg.startElement(name, {}) if hasattr(serialized_value, "items"): self._beautify_newline() - for subname, value in serialized_value.items(): - self._export_xml_field(subname, value, depth=depth + 1) + for sub_name, value in serialized_value.items(): + self._export_xml_field(sub_name, value, depth=depth + 1) self._beautify_indent(depth=depth) elif is_listlike(serialized_value): self._beautify_newline() diff --git a/Comics/items.py b/Comics/items.py index 7700738..2fb6541 100644 --- a/Comics/items.py +++ b/Comics/items.py @@ -4,18 +4,8 @@ # https://docs.org/en/latest/topics/items.html from scrapy.item import Item, Field from Comics.utils.Constant import ComicPath -from dataclasses import dataclass from scrapy.loader.processors import TakeFirst, MapCompose, Join - -data = {} -def setinfo(**kwds): - def decorate(f): - for k in kwds: data[k] = kwds[k] - setattr(f, "data", data) - return f - return decorate - def serialize_to_chinese(value): return ComicPath.chinese_convert(value) @@ -27,34 +17,29 @@ class ComicOItem(Item): name = Field() chapterItem = Field() -@setinfo(name="Series", chapter="Title", - author="Writer", tags="Tags", - dep="Summary", genre="Genre", - index="Number", images_name="Pages", - age_rating="AgeRating") class ComicItem(Item): # 编号 - index = Field() + index = Field(output_processor=TakeFirst()) # 漫画名 name = Field(serializer=serialize_to_fix_file, output_processor=TakeFirst()) # 章节名 - chapter = Field(serializer=serialize_to_fix_file) + chapter = Field(serializer=serialize_to_fix_file, output_processor=TakeFirst()) # 图片链接 list_img = Field() # 作者 author = Field(serialize_to_chinese=serialize_to_chinese, output_processor=TakeFirst()) # 封面链接 - icon = Field() + icon = Field(output_processor=TakeFirst()) # 标签 - tags = Field(serializer=serialize_to_chinese) + tags = Field(serializer=serialize_to_chinese, output_processor=TakeFirst()) # 概述 - dep = Field(serializer=serialize_to_chinese) + dep = Field(serializer=serialize_to_chinese, output_processor=TakeFirst()) # 时间 - date = Field() + date = Field(output_processor=TakeFirst()) # 流派 - genre = Field() + genre = Field(output_processor=TakeFirst()) # 年龄分级 - age_rating = Field() + age_rating = Field(output_processor=TakeFirst()) images = Field() images_name = Field() @@ -72,22 +57,22 @@ def serializer_info_writer(value): return ",".join(list_value) class ComicInfoItem(Item): - Title = Field()#"章节名",True] - Series = Field()# ","漫画名",True] - Number = Field()# ","编号",True] + Title = Field(info='chapter')#"章节名",True] + Series = Field(info='name')# ","漫画名",True] + Number = Field(info='index')# ","编号",True] SeriesGroup = Field()# ","别名",False] - Summary = Field()# ","概述",True] + Summary = Field(info='dep')# ","概述",True] Year = Field()# ","年",False] Month = Field()# ","月",False] Day = Field()# ","日",False] - Writer = Field(serializer=serializer_info_writer)# "作者",True] + Writer = Field(info='author',serializer=serializer_info_writer)# "作者",True] Publisher = Field()# ","出版社",False] - Genre = Field()# ","流派",True] - Tags = Field()# ","标签",True] + Genre = Field(info='genre')# ","流派",True] + Tags = Field(info='tags')# ","标签",True] Web = Field()# ","主页",False] PageCount = Field()# ","总页数",True] LanguageISO = Field()#","语言",True] - AgeRating = Field()#","年龄分级",False] - Pages = Field()#","页码",True] - Page = Field() - # ComicInfo.xml and ComicChapter.json end \ No newline at end of file + AgeRating = Field(info='age_rating')#","年龄分级",False] + Pages = Field(info='images_name')#","页码",True] + # ComicInfo.xml and ComicChapter.json end + diff --git a/Comics/loader.py b/Comics/loader.py new file mode 100644 index 0000000..ad4b8e4 --- /dev/null +++ b/Comics/loader.py @@ -0,0 +1,44 @@ +import json +from scrapy.loader import ItemLoader +class ComicLoader(ItemLoader): + def parseExec(cls,data,exec): + if data !=None and exec != None: + dots = str(exec).split(".") + if not isinstance(data,dict): data = json.loads(data) + for dot in dots: + data = data.get(dot) + return data + + def add_xpath(self, field_name, xpath, *processors, index=None, exec=None, re=None, **kw): + """ + Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a + value, which is used to extract a list of strings from the + selector associated with this :class:`ItemLoader`. + + See :meth:`get_xpath` for ``kwargs``. + + :param xpath: the XPath to extract data from + :type xpath: str + + Examples:: + + # HTML snippet:
Color TV
+ loader.add_xpath('name', '//p[@class="product-name"]') + # HTML snippet:the price is $1200
+ loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)') + + """ + values = self._get_xpathvalues(xpath, **kw) + if exec is not None: + values = self.parseExec(values, exec) + if index is not None: + values = values[index] + self.add_value(field_name, values, *processors, re=re, **kw) + + def add_exec(self, field_name, value, str_exec=None, *processors, re=None, **kw): + if str_exec is not None: + value = self.parseExec(value, str_exec) + self.add_value(field_name, value, *processors, re=re, **kw) + + def get_exec(self, value, str_exec): + return self.parseExec(value, str_exec) \ No newline at end of file diff --git a/Comics/middlewares.py b/Comics/middlewares.py index cc12d22..c5c85d2 100644 --- a/Comics/middlewares.py +++ b/Comics/middlewares.py @@ -4,14 +4,17 @@ # https://docs.scrapy.org/en/latest/topics/spider-middleware.html from scrapy import signals -import random +import random,logging +from pathlib import Path from Comics.settings import PROXY_LIST # useful for handling different item types with a single interface -from itemadapter import is_item, ItemAdapter + +logger = logging.getLogger(__name__) class ProxyMiddleware(object): def process_request(self, request, spider): - request.meta["proxy"] = random.choice(PROXY_LIST) + if len(PROXY_LIST) != 0: + request.meta["proxy"] = random.choice(PROXY_LIST) class ComicsSpiderMiddleware: # Not all methods need to be defined. If a method is not defined, diff --git a/Comics/pipelines.py b/Comics/pipelines.py index a114f8e..712c68f 100644 --- a/Comics/pipelines.py +++ b/Comics/pipelines.py @@ -5,7 +5,7 @@ # useful for handling different item types with a single interface -import os,requests,re,scrapy,logging +import os, scrapy from Comics import settings from Comics.utils.FileUtils import imageUtils from Comics.utils.FileUtils import fileUtils @@ -15,7 +15,7 @@ from Comics.items import ImageItem from scrapy.pipelines.images import ImagesPipeline from Comics.exporters import ComicInfoXmlItemExporter from Comics.exporters import ItemExporter -from Comics.utils.CBZUtils import CBZUtils +from Comics.utils.FileUtils import CBZUtils class ComicsPipeline: def open_spider(self, spider): @@ -24,10 +24,10 @@ class ComicsPipeline: def process_item(self, item, spider): if isinstance(item, ComicItem): item = ComicItem(ItemExporter().export_obj(item)) - file = os.path.join("json",item['name'],item['chapter']) - fileUtils.save_file(f"{file}.json",item) + file = os.path.join("json", item['name'], item['chapter']) + fileUtils.save_file(f"{file}.json", item) return item - #image解析 + # image解析 def close_spider(self,spider): pass @@ -38,16 +38,16 @@ class ImageParsePipeline: count = 1 images_item = [] for image in item['list_img']: - (image_src,scramble) = [image.get("src"),image.get("scramble")] + (image_src, scramble) = [image.get("src"), image.get("scramble")] count_image = "{:0>3d}".format(count) suffix = "."+str(image_src).split(".")[-1] image_name = count_image + suffix if scramble: - de_str = str(image_src).split("/")[-1].replace(suffix,"==") + de_str = str(image_src).split("/")[-1].replace(suffix, "==") blocks_num = imageUtils.encodeImage(de_str) - image_name = ComicPath.getFileScrambleImageName(count=count_image,block=blocks_num,suffix=suffix) - image_path = os.path.join(item['name'],item['chapter'], image_name) - images_item.append(ImageItem(image_name=count_image + suffix,image_url=image_src,image_path=image_path)) + image_name = ComicPath.getFileScrambleImageName(count=count_image, block=blocks_num, suffix=suffix) + image_path = os.path.join(item['name'], item['chapter'], image_name) + images_item.append(ImageItem(image_name=count_image + suffix, image_url=image_src, image_path=image_path)) count += 1 item['images'] = images_item return item @@ -64,18 +64,18 @@ class ImgDownloadPipeline(ImagesPipeline): def get_media_requests(self, item, info): for image in item['images']: - yield scrapy.Request(url= image['image_url'], meta= {'item' : image}) + yield scrapy.Request(url=image['image_url'], meta={'item': image}) def item_completed(self, results, item, info): info_img = [] for success, img in results: img_path = os.path.join(settings.IMAGES_STORE, img['path']) - #解密图片 + # 解密图片 img_path = imageUtils.deScrambleImagesByPath(img_path) info_img.append(os.path.basename(img_path).split('.')[0]) item['images_name'] = ",".join(info_img) - #return item - #ComicInfoXml 生成 + # return item + # ComicInfoXml 生成 ComicInfoXmlItemExporter(comic=item['name'], chapter=item['chapter']).export_xml(item) - #打包 - CBZUtils.packComicChapterCBZ(comic=item['name'], chapter=item['chapter'], remove= False) \ No newline at end of file + # 打包 + CBZUtils.packComicChapterCBZ(comic=item['name'], chapter=item['chapter'], remove=False) diff --git a/Comics/settings.py b/Comics/settings.py index 4d04dd7..bc5db37 100644 --- a/Comics/settings.py +++ b/Comics/settings.py @@ -34,7 +34,7 @@ DOWNLOAD_DELAY = 20 RETRY_ENABLED = True RETRY_TIMES = 10 # 想重试几次就写几 # 下面这行可要可不要 -#RETRY_HTTP_CODES = [500, 502, 503, 504, 408] +RETRY_HTTP_CODES = [500, 502, 503, 504, 408, 401] # The download delay setting will honor only one of: #CONCURRENT_REQUESTS_PER_DOMAIN = 16 #CONCURRENT_REQUESTS_PER_IP = 16 @@ -102,7 +102,8 @@ AUTOTHROTTLE_DEBUG = False HTTPCACHE_ENABLED = True HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_DIR = 'httpcache' -HTTPCACHE_IGNORE_HTTP_CODES = [500, 502, 404, 403] +HTTPCACHE_IGNORE_HTTP_CODES = [500, 502, 404, 403, 401] +#HTTPCACHE_STORAGE = 'Comics.middlewares.MyFilesystemCacheStorage' HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' CBZ_EXPORT_PATH = "CBZ" diff --git a/Comics/spiders/rm_comic.py b/Comics/spiders/rm_comic.py index 86fef07..07d8c03 100644 --- a/Comics/spiders/rm_comic.py +++ b/Comics/spiders/rm_comic.py @@ -1,9 +1,8 @@ -import urllib.parse - -import scrapy,json,requests +import scrapy from Comics.items import ComicItem -from Comics.utils.FileUtils import CommonUtils -from scrapy.loader import ItemLoader +from Comics.loader import ComicLoader +from itemadapter import ItemAdapter +from Comics.items import ComicInfoItem class RmComicSpider(scrapy.Spider): name = 'rm_comic' @@ -14,49 +13,50 @@ class RmComicSpider(scrapy.Spider): def start_requests(self): yield scrapy.Request('https://rm01.xyz' '/books/306ec1e2-f701-4fda-bb78-041ad6ec4020', callback=self.parse_comic) - + + # 获取某个漫画的相关数据 + # 获取到多个章节链接后进入下个流程 def parse_comic(self, response): - comic = ComicItem() -# comic_item = ItemLoader(item=ComicItem(), response=response) - comic['name'] = response.xpath('//div[@class="col"]/h5/text()').extract_first() - comic['icon'] = response.xpath('//img[@class="img-thumbnail"]/@src').extract_first() - comic['author'] = response.xpath('//div[contains(@class,"bookid_bookInfo")]/p[1]/text()').extract()[1] - comic['tags'] = response.xpath('//div[contains(@class,"bookid_bookInfo")]/p[3]/b/text()').extract_first() - comic['dep'] = response.xpath('//div[contains(@class,"bookid_bookInfo")]/p[4]/text()').extract()[1] - comic['date'] = response.xpath('//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()').extract()[1] - comic['genre'] = "韩漫" - comic['age_rating'] = "R18+" - chapters = response.xpath('//div[contains(@class,"bookid_chapterBox")]' - '//div[contains(@class,"bookid_chapter")]/a/text()').extract() - chapter_href = response.xpath('//div[contains(@class,"bookid_chapterBox")]' - '//div[contains(@class,"bookid_chapter")]/a/@href').extract() + comic_item = ComicLoader(item=ComicItem(), response=response) + comic_item.add_xpath('name', '//div[@class="col"]/h5/text()') + comic_item.add_xpath('icon', '//img[@class="img-thumbnail"]/@src') + comic_item.add_xpath('author', '//div[contains(@class,"bookid_bookInfo")]/p[1]/text()', index=1) + comic_item.add_xpath('tags', '//div[contains(@class,"bookid_bookInfo")]/p[3]/b/text()') + comic_item.add_xpath('dep', '//div[contains(@class,"bookid_bookInfo")]/p[4]/text()', index=1) + comic_item.add_xpath('date', '//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()', index=1) + comic_item.add_value('genre', "韩漫") + comic_item.add_value('age_rating', "R18+") + chapter_href = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]' + '//div[contains(@class,"bookid_chapter")]/a/@href') + #chapters = response.xpath('//div[contains(@class,"bookid_chapterBox")]' + # '//div[contains(@class,"bookid_chapter")]/a/text()').extract() #for chapter, link in zip(chapters, chapter_href): for i, link in enumerate(chapter_href, start=1): - yield scrapy.Request(self.main_url+link, meta={'item' : comic, 'number': i}, callback=self.parse_chapter) + yield scrapy.Request(self.main_url+link, meta={'item': comic_item.load_item(), 'num': i}, callback=self.parse_chapter) + # 读取某章节下的所有图片 def parse_chapter(self, response): - item = response.meta['item'] - number = response.meta['number'] - data = response.xpath('//script[@id="__NEXT_DATA__"]/text()').extract_first() + comic_item = ComicLoader(item=response.meta['item'], response=response) + data = comic_item.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0] str_exec = "props.pageProps." - comic_name = CommonUtils.parseExec(data, str_exec+"bookName") - chapterName = CommonUtils.parseExec(data, str_exec+"chapterName") - description = CommonUtils.parseExec(data, str_exec+"description") - images = CommonUtils.parseExec(data, str_exec+"images") - chapter_api_url = CommonUtils.parseExec(data, str_exec+"chapterAPIPath") - item['chapter'] = chapterName - item['list_img'] = images - item['index'] = number - if chapter_api_url != None: - yield scrapy.Request(self.main_url+ chapter_api_url,meta={'item' : item}, callback= self.parse_chapter_api) + #comic_item.add_exec('name', data, str_exec=str_exec+"bookName") + #comic_item.add_exec('dep', data, str_exec=str_exec+"description") + comic_item.add_value('index', response.meta['num']) + comic_item.add_exec('chapter', data, str_exec=str_exec + "chapterName") + comic_item.add_exec('list_img', data, str_exec+"images") + comic = comic_item.load_item() + chapter_api_url = comic_item.get_exec(data, str_exec+"chapterAPIPath") + if chapter_api_url is not None: + yield scrapy.Request(self.main_url + chapter_api_url, meta={'item': comic}, callback=self.parse_chapter_api) else: - yield item + yield comic + # 加密数据API处理 def parse_chapter_api(self, response): - item = response.meta['item'] - item['chapter'] = CommonUtils.parseExec(response.text, "chapter.name") - item['list_img'] = CommonUtils.parseExec(response.text, "chapter.images") - yield item + comic_item = ComicLoader(item=response.meta['item'], response=response) + comic_item.add_exec('chapter', response.text, str_exec='chapter.name') + comic_item.add_exec('list_img', response.text, str_exec='chapter.images') + yield comic_item.load_item() def parse(self, response): raise NotImplementedError \ No newline at end of file diff --git a/Comics/utils/CBZUtils.py b/Comics/utils/CBZUtils.py deleted file mode 100644 index 3eba3ff..0000000 --- a/Comics/utils/CBZUtils.py +++ /dev/null @@ -1,105 +0,0 @@ -import os, shutil, time, logging -from datetime import datetime -from pathlib import Path -from zipfile import ZipFile -from Comics.settings import COMIC_INFO_XML_FILE,CBZ_EXPORT_PATH,IMAGES_STORE - -class CBZUtils: - - @classmethod - def readDirsOrFiles(cls,dir,type): - data = [] - files = os.listdir(dir) - for file in files: - path = os.path.join(dir,file) - if type == "files" and os.path.isfile(path): - data.append(path) - if type == "dirs" and os.path.isdir(path): - data.append(path) - return data - - @classmethod - def zip_compression(cls, source_dir=None, target_file=None, remove=True): - target_dir = os.path.dirname(target_file) - if not os.path.exists(target_dir): - os.makedirs(target_dir) - if not os.path.exists(target_file) and source_dir != None: - with ZipFile(target_file, mode='w') as zf: - for path, dir_names, filenames in os.walk(source_dir): - path = Path(path) - arc_dir = path.relative_to(source_dir) - y = 0 - for filename in filenames: - y = y + 1 - print("打包中:" + str(y) + "/" + str(len(filenames)), os.path.join(source_dir, filename)) - zf.write(path.joinpath(filename), arc_dir.joinpath(filename)) - zf.close() - logging.info(f"打包完成:{target_file}") - - @classmethod - def packComicChapterCBZ(cls, comic, chapter, remove=True): - images_chapter_path = os.path.join(IMAGES_STORE, comic, chapter) - cbz_chapter_path = os.path.join(CBZ_EXPORT_PATH, comic, chapter)+".CBZ" - if os.path.exists(images_chapter_path): - dirs = os.listdir(images_chapter_path) - for file in dirs: - if file.startswith("scramble="): - try: - os.remove(file) - except: - print(f"删除 {file} 发生错误,已跳过") - return False - cls.zip_compression(images_chapter_path, cbz_chapter_path) - time.sleep(0.1) - if remove: shutil.rmtree(images_chapter_path) - return True - - @classmethod - def replaceZip(cls,filepath,unpack_dir=None): - if not cls.compareFileDate(filepath): return None - if unpack_dir == None: - unpack_dir = str(filepath).split(".")[0] - fz = ZipFile(filepath, 'r') - for file in fz.namelist(): - if file.endswith(".jpg"): - data = fz.read(file) - if len(data) < 500 and os.path.exists(filepath): - os.remove(filepath) - print(f"数据不完整,已删除:{filepath}") - if cls.compareFileDate(filepath): - os.utime(filepath) - print(f"已更新文件时间 {filepath}") - if os.path.exists(unpack_dir): - shutil.rmtree(unpack_dir) - # 删除删除main.ftl文件 - #delete_filename = '' - #if os.path.exists(delete_filename): - # os.remove(delete_filename) - # time.sleep(60) - # shutil.copy(文件的路径,另一个目录);拷贝main.ftl到准备压缩的目录下 - #cls.zip_compression() - #小于则运行 - @classmethod - def compareFileDate(cls,filepath): - if os.path.exists(filepath): - ctime = os.path.getmtime(filepath) - str_ctime = datetime.fromtimestamp(int(ctime)) - file_ctime = str(str_ctime.year)+"{:0>2d}".format(str_ctime.month)+"{:0>2d}".format(str_ctime.day)+"{:0>2d}".format(str_ctime.hour) - c_ctime = 2023011603 - else: - return False - if int(file_ctime) < c_ctime: - return True - return False - - @classmethod - def zip_info(cls, path, filter=True): - result = None - try: - with ZipFile(path, "r") as zip_file: - result = zip_file.namelist() - if filter: - result.remove(COMIC_INFO_XML_FILE) - except Exception as e: - print(e) - return result \ No newline at end of file diff --git a/Comics/utils/ComicInfo.py b/Comics/utils/ComicInfo.py deleted file mode 100644 index aa5e22f..0000000 --- a/Comics/utils/ComicInfo.py +++ /dev/null @@ -1,99 +0,0 @@ -import json,os -import logging -from xml.dom.minidom import Document -from Comics.utils.Constant import ComicPath -from itemadapter import is_item, ItemAdapter - -class ComicInfo: - IS_NEW_ICON = False - document = Document() - path_comic_info = None - - @classmethod - def setNodeAndValue(cls,node,value): - if value != None: - if isinstance(value,str): - c_node = cls.document.createElement(node) - child_node = cls.document.createTextNode(value) - c_node.appendChild(child_node) - return c_node - else: return value - return None - - #页数 - @classmethod - def setPages(cls,values=None): - #if values == None: values = Comic.getChapterFilesName() - if values != None and isinstance(values,list): - suffix = "."+str(values[0]).split(".")[-1] - join_list=",".join(values).replace(suffix,"") - values = join_list.split(",") - #Comic.setPageCount(len(values)+1 if cls.IS_NEW_ICON else len(values)) - root_node = cls.document.createElement("Pages") - if cls.IS_NEW_ICON: - #添加封面 - icon_node = cls.document.createElement("Page") - icon_node.setAttribute("Image",ComicPath.COMIC_ICON_NAME) - icon_node.setAttribute("Type","FrontCover") - root_node.appendChild(icon_node) - for page in values: - c_node = cls.document.createElement("Page") - page = page.split("_")[-1] - c_node.setAttribute("Image",page) - root_node.appendChild(c_node) - #Comic.dict_pages = Comic.setField(Comic.dict_pages,root_node,convert=False) - - @classmethod - def getBaseUrl(cls,url=None): - #if url == None: - # url = Comic.getHomePage() - (num,index) = [3,0] - for x in range(0, num): - index = str(url).find("/",index)+1 - return url[0:index-1] - - #XML根文档 - @classmethod - def root_node(cls,root_value): return cls.document.createElement(root_value) - - @classmethod - def add_nodes(cls,root,item): - item = ItemAdapter(item) - keys = item.keys() - files = item.field_names() - values = item.values() - print("test") - #if len(list_value) == 0: return list_value - #for value in list_value: - # #Comic.chapter - # if value[0] == None and value[4]: - # #数据为空 value[0], 但不允许为空value[4] = False - # msg = f"#数据为空 key={value[3]} value[0]={value[0]}, 但不允许为空value[4]={value[4]}" - # logging.error(msg) - # exit() - # if value[0] != None: root.appendChild(cls.setNodeAndValue(value[2],value[0])) - - @classmethod - def initComicInfoXML(cls): - cls.setPages() - - @classmethod - def writeComicInfoXML(cls,item,overlay=False): - #save_path = ComicPath.getPathComicInfoXML() - save_path = "ComicInfo.xml" - if os.path.exists(save_path): - if overlay: - os.remove(save_path) - logging.info(f"已存在文件,已删除... {save_path}") - else: - logging.info(f"已存在文件,跳过中... {save_path}") - return True - cls.initComicInfoXML() - root = cls.root_node("ComicInfo") - new_document = Document() - new_document.appendChild(root) - cls.add_nodes(root, item) - with open(save_path, "w", encoding="utf-8") as fo: - new_document.writexml(fo, indent='', addindent='\t', newl='\n', encoding="utf-8") - fo.close() - logging.info(f"已生成文件... {save_path}") \ No newline at end of file diff --git a/Comics/utils/Constant.py b/Comics/utils/Constant.py index f178afb..ebe3091 100644 --- a/Comics/utils/Constant.py +++ b/Comics/utils/Constant.py @@ -1,14 +1,15 @@ import os.path import re from opencc import OpenCC -from Comics.settings import IMAGES_STORE class ComicPath: + PREFIX_SCRAMBLE = "scramble=" + @classmethod def getDirComicChapter(cls): return None @classmethod - def getFileScrambleImageName(cls,count,block,suffix=".jpg"): return "scramble="+str(block)+"_"+str(count)+suffix + def getFileScrambleImageName(cls,count,block,suffix=".jpg"): return cls.PREFIX_SCRAMBLE+str(block)+"_"+str(count)+suffix @classmethod def getFileScrambleImageSave(cls,file): return str(file).split("_")[-1] diff --git a/Comics/utils/FileUtils.py b/Comics/utils/FileUtils.py index 4824c6b..2bde3c0 100644 --- a/Comics/utils/FileUtils.py +++ b/Comics/utils/FileUtils.py @@ -1,8 +1,10 @@ import base64,hashlib,os,shutil import math,time,json,datetime,logging from PIL import Image -from tinydb import TinyDB, Query from Comics.utils.Constant import ComicPath +from pathlib import Path +from zipfile import ZipFile +from Comics.settings import COMIC_INFO_XML_FILE,CBZ_EXPORT_PATH,IMAGES_STORE class fileUtils: @classmethod @@ -32,16 +34,16 @@ class imageUtils: if os.path.exists(chapter_dir): #获取章节图片路径 dirs = os.listdir(chapter_dir) for img in dirs: - if img.startswith("scramble="): + if img.startswith(ComicPath.PREFIX_SCRAMBLE): imageUtils.encode_scramble_image(os.path.join(chapter_dir,img)) scramble_count += 1 - logging.debug(f"scramble= {scramble_count}") + logging.debug(f"{ComicPath.PREFIX_SCRAMBLE} {scramble_count}") return scramble_count @classmethod - def deScrambleImagesByPath(cls,img_path,img_save=None): - if os.path.basename(img_path).startswith("scramble="): - img_path = imageUtils.encode_scramble_image(img_path,img_save) + def deScrambleImagesByPath(cls, img_path, img_save=None): + if os.path.basename(img_path).startswith(ComicPath.PREFIX_SCRAMBLE): + img_path = imageUtils.encode_scramble_image(img_path, img_save) return img_path @classmethod @@ -232,4 +234,107 @@ class imageUtils: if os.path.exists(imgpath): os.remove(imgpath) print("remove=",imgpath) - return save_path \ No newline at end of file + return save_path + + +class CBZUtils: + + @classmethod + def readDirsOrFiles(cls, dir, type): + data = [] + files = os.listdir(dir) + for file in files: + path = os.path.join(dir, file) + if type == "files" and os.path.isfile(path): + data.append(path) + if type == "dirs" and os.path.isdir(path): + data.append(path) + return data + + @classmethod + def zip_compression(cls, source_dir=None, target_file=None, remove=True): + target_dir = os.path.dirname(target_file) + if not os.path.exists(target_dir): + os.makedirs(target_dir) + if not os.path.exists(target_file) and source_dir is not None: + with ZipFile(target_file, mode='w') as zf: + for path, dir_names, filenames in os.walk(source_dir): + path = Path(path) + arc_dir = path.relative_to(source_dir) + y = 0 + for filename in filenames: + y = y + 1 + print("打包中:" + str(y) + "/" + str(len(filenames)), os.path.join(source_dir, filename)) + zf.write(path.joinpath(filename), arc_dir.joinpath(filename)) + zf.close() + logging.info(f"打包完成:{target_file}") + + @classmethod + def packComicChapterCBZ(cls, comic, chapter, remove=True): + images_chapter_path = os.path.join(IMAGES_STORE, comic, chapter) + cbz_chapter_path = os.path.join(CBZ_EXPORT_PATH, comic, chapter) + ".CBZ" + if os.path.exists(images_chapter_path): + dirs = os.listdir(images_chapter_path) + for file in dirs: + if file.startswith(ComicPath.PREFIX_SCRAMBLE): + try: + os.remove(file) + except Exception as e: + print(f"删除 {file} 发生错误 {e},已跳过") + return False + cls.zip_compression(images_chapter_path, cbz_chapter_path) + time.sleep(0.1) + if remove: shutil.rmtree(images_chapter_path) + return True + + @classmethod + def replaceZip(cls, filepath, unpack_dir=None): + if not cls.compareFileDate(filepath): return None + if unpack_dir == None: + unpack_dir = str(filepath).split(".")[0] + fz = ZipFile(filepath, 'r') + for file in fz.namelist(): + if file.endswith(".jpg"): + data = fz.read(file) + if len(data) < 500 and os.path.exists(filepath): + os.remove(filepath) + print(f"数据不完整,已删除:{filepath}") + if cls.compareFileDate(filepath): + os.utime(filepath) + print(f"已更新文件时间 {filepath}") + if os.path.exists(unpack_dir): + shutil.rmtree(unpack_dir) + # 删除删除main.ftl文件 + # delete_filename = '' + # if os.path.exists(delete_filename): + # os.remove(delete_filename) + # time.sleep(60) + # shutil.copy(文件的路径,另一个目录);拷贝main.ftl到准备压缩的目录下 + # cls.zip_compression() + # 小于则运行 + + @classmethod + def compareFileDate(cls, filepath): + if os.path.exists(filepath): + ctime = os.path.getmtime(filepath) + str_ctime = datetime.fromtimestamp(int(ctime)) + file_ctime = str(str_ctime.year) + "{:0>2d}".format(str_ctime.month) + "{:0>2d}".format( + str_ctime.day) + "{:0>2d}".format(str_ctime.hour) + c_ctime = 2023011603 + else: + return False + if int(file_ctime) < c_ctime: + return True + return False + + @classmethod + def zip_info(cls, path, filter=True): + result = None + try: + with ZipFile(path, "r") as zip_file: + result = zip_file.namelist() + if filter: + result.remove(COMIC_INFO_XML_FILE) + except Exception as e: + print(e) + return result \ No newline at end of file