From e4ac2cc6310a8156148bf357b9c69d0e02749bb2 Mon Sep 17 00:00:00 2001 From: caiwx86 Date: Wed, 6 Sep 2023 02:50:16 +0800 Subject: [PATCH] fix --- Comics/pipelines.py | 32 +++++++++++++++++++++----------- Comics/spiders/rm_comic.py | 2 +- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/Comics/pipelines.py b/Comics/pipelines.py index 3e8f8ee..9fd8845 100644 --- a/Comics/pipelines.py +++ b/Comics/pipelines.py @@ -11,7 +11,7 @@ from Comics.settings import CBZ_EXPORT_PATH,OUTPUT_DIR,PROJECT_KEY from Comics.utils.Constant import ComicPath from Comics.items import ComicItem from scrapy.pipelines.images import ImagesPipeline -from Comics.exporters import ComicInfoXmlItemExporter,JsonExport +from Comics.exporters import ComicInfoXmlItemExporter,JsonExport,ItemExporter from Comics.utils.FileUtils import CBZUtils from Comics.utils.FileUtils import fileUtils as fu @@ -24,12 +24,17 @@ class ComicsPipeline: def process_item(self, item, spider): if isinstance(item, ComicItem): # 'output/rm_comic/json/壞X/第1話 壞X' - file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter']) - item['count'] = len(item['images']) - item['index'] = item['chapters'].index(item['chapter']) + 1 + cbz_path = ComicPath.get_file_path(item, result_type="cbz", convert=True) + if os.path.exists(cbz_path): + item['image_urls'] = [] + item['images'] = [] + return ItemExporter().export_obj(item) + else: + file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter']) + item['count'] = len(item['images']) + item['index'] = item['chapters'].index(item['chapter']) + 1 - data = JsonExport(file=file).export_json(item, if_return=True) - return data + return JsonExport(file=file).export_json(item, if_return=True) # image解析 def close_spider(self, spider): @@ -99,14 +104,19 @@ class ImgDownloadPipeline(ImagesPipeline): def item_completed(self, results, item, info): # return item - # ComicInfoXml 生成 - comic_info = ComicInfoXmlItemExporter(dir=self.get_file_path(item=item, result_type="comic_info")).export_xml(item) # 打包 - if CBZUtils.packComicChapterCBZ(src_dir= self.get_file_path(item, result_type="images_dir"), - dts_path= self.get_file_path(item, result_type="cbz"), - comic_info_images= comic_info['Pages'], remove=True): + cbz_path = self.get_file_path(item, result_type="cbz") + if os.path.exists(cbz_path): self.update_icon(item) self.pack_icon(item) + else: + # ComicInfoXml 生成 + comic_info = ComicInfoXmlItemExporter(dir=self.get_file_path(item=item, result_type="comic_info")).export_xml(item) + if CBZUtils.packComicChapterCBZ(src_dir= self.get_file_path(item, result_type="images_dir"), + dts_path= cbz_path, + comic_info_images= comic_info['Pages'], remove=True): + self.update_icon(item) + self.pack_icon(item) # sleep_time = random.randint(5,30) # print(f'等待{sleep_time}秒后进行下一章节') diff --git a/Comics/spiders/rm_comic.py b/Comics/spiders/rm_comic.py index 370ba8b..bfb4451 100644 --- a/Comics/spiders/rm_comic.py +++ b/Comics/spiders/rm_comic.py @@ -49,7 +49,7 @@ class RmComicSpider(scrapy.Spider): cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True) if os.path.exists(cbz_path): logging.info(f"漫画 {cbz_path} 已存在, 跳过中...") - return + yield item else: yield scrapy.Request(self.main_url+link, meta={'item': item}, callback=self.parse_chapter)