diff --git a/Comics/pipelines.py b/Comics/pipelines.py index c468fa2..3e8f8ee 100644 --- a/Comics/pipelines.py +++ b/Comics/pipelines.py @@ -13,6 +13,8 @@ from Comics.items import ComicItem from scrapy.pipelines.images import ImagesPipeline from Comics.exporters import ComicInfoXmlItemExporter,JsonExport from Comics.utils.FileUtils import CBZUtils +from Comics.utils.FileUtils import fileUtils as fu + class ComicsPipeline: def open_spider(self, spider): @@ -21,11 +23,12 @@ class ComicsPipeline: # item就是yield后面的对象 def process_item(self, item, spider): if isinstance(item, ComicItem): + # 'output/rm_comic/json/壞X/第1話 壞X' file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter']) item['count'] = len(item['images']) item['index'] = item['chapters'].index(item['chapter']) + 1 + data = JsonExport(file=file).export_json(item, if_return=True) - #data[PROJECT_KEY] = spider.name return data # image解析 @@ -42,7 +45,7 @@ class ImgDownloadPipeline(ImagesPipeline): ## Icon Path : CBZ/NAME/CHAPTER.jpg def download_icon(self, item, result_type="download"): - icon_path = self.get_file_path(item, result_type="icon") + icon_path = self.get_file_path(item, result_type="icon_cache") if result_type == "fullpath": return os.path.join(settings.IMAGES_STORE, icon_path) if os.path.exists(icon_path): @@ -60,21 +63,40 @@ class ImgDownloadPipeline(ImagesPipeline): # 下载封面 self.download_icon(item) for image_url,image in zip(self.image_urls,self.images): + is_down = True image_path = self.get_file_path(item, image) if self.image_scramble_exits(item, image_path): - logging.info(f"file exists: IMAGE_STORE {image_path}") - else: + if image_path == self.get_file_path(item, result_type="icon_cache"): + logging.info(f"icon file exists: IMAGE_STORE {image_path}") + else: + is_down = False + logging.info(f"file exists: IMAGE_STORE {image_path}") + if is_down: logging.info(f"downloading {image_url} --> IMAGE_STORE {image_path}") yield scrapy.Request(url=image_url, meta={'path': image_path}) + # 打包cbz封面 def pack_icon(self, item): cbz_icon = self.get_file_path(item=item, result_type="cbz_icon") dwn_icon = self.get_file_path(item=item, result_type="down_icon") - logging.info(f"icon packing {dwn_icon} => {cbz_icon}") - cbz_icon_dir = os.path.dirname(cbz_icon) - if not os.path.exists(cbz_icon_dir): os.makedirs(cbz_icon_dir) - shutil.copyfile(dwn_icon, cbz_icon) - + base_dir = os.path.dirname(dwn_icon) + name = os.path.basename(dwn_icon).split(".")[0] + for dirname in os.listdir(base_dir): + path = os.path.join(base_dir, dirname) + if os.path.isfile(path) and dirname.startswith(name): + fu.update_icon(path, cbz_icon) + + + + # 判断是否需要更新封面 + def update_icon(self, item): + # 下载后的缓存封面路径 + image_path = self.get_file_path(item, result_type="down_cache_icon") + # 最终封面保存路径 + save_path = self.get_file_path(item=item, result_type="down_icon") + + fu.update_icon(image_path, save_path) + def item_completed(self, results, item, info): # return item # ComicInfoXml 生成 @@ -83,6 +105,7 @@ class ImgDownloadPipeline(ImagesPipeline): if CBZUtils.packComicChapterCBZ(src_dir= self.get_file_path(item, result_type="images_dir"), dts_path= self.get_file_path(item, result_type="cbz"), comic_info_images= comic_info['Pages'], remove=True): + self.update_icon(item) self.pack_icon(item) # sleep_time = random.randint(5,30) diff --git a/Comics/settings.py b/Comics/settings.py index 78d30ba..10adf9a 100644 --- a/Comics/settings.py +++ b/Comics/settings.py @@ -68,7 +68,7 @@ COOKIES_ENABLED = False # Enable or disable downloader middlewares # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html DOWNLOADER_MIDDLEWARES = { -# 'Comics.middlewares.ComicsDownloaderMiddleware': 543, + 'Comics.middlewares.ComicsDownloaderMiddleware': 543, # 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500, 'Comics.middlewares.ProxyMiddleware': 100, 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 400, diff --git a/Comics/spiders/rm_comic.py b/Comics/spiders/rm_comic.py index 67ed16a..370ba8b 100644 --- a/Comics/spiders/rm_comic.py +++ b/Comics/spiders/rm_comic.py @@ -16,7 +16,7 @@ class RmComicSpider(scrapy.Spider): for x in range(0,60): yield scrapy.Request(self.start_urls+"?&page="+str(x), callback=self.books_comic) - # 获取多个漫画信息 + # 获取多个漫画信息 def books_comic(self, response): comics = ComicLoader(item=ComicItem(), response=response) data = comics.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0] @@ -40,14 +40,9 @@ class RmComicSpider(scrapy.Spider): comic_item.add_value('age_rating', "R18+") chapter_href = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]' '//div[contains(@class,"bookid_chapter")]/a/@href') - #chapters = response.xpath('//div[contains(@class,"bookid_chapterBox")]' - # '//div[contains(@class,"bookid_chapter")]/a/text()').extract() - chapters = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]' '//div[contains(@class,"bookid_chapter")]/a/text()') for chapter, link in zip(chapters, chapter_href): - #for i, link in enumerate(chapter_href, start=1): - # yield scrapy.Request(self.main_url+link, meta={'item': comic_item.load_item(), 'num': i}, callback=self.parse_chapter) comic_item.add_value('chapter', chapter) comic_item.add_value('chapters', chapters) item = comic_item.load_item() diff --git a/Comics/utils/Constant.py b/Comics/utils/Constant.py index 90edb71..f177481 100644 --- a/Comics/utils/Constant.py +++ b/Comics/utils/Constant.py @@ -67,8 +67,12 @@ class ComicPath: file = os.path.join(settings.CBZ_EXPORT_PATH, PROJECT, name, chapter+".jpg") elif result_type == "down_icon": file = os.path.join(settings.IMAGES_STORE, cls.get_file_path(item=item, result_type="icon")) + elif result_type == "down_cache_icon": + file = os.path.join(settings.IMAGES_STORE, cls.get_file_path(item=item, result_type="icon_cache")) elif result_type == "icon": - file = os.path.join(PROJECT, "icons", name+".jpg") + file = os.path.join(PROJECT, "icons", name, name+".jpg") + elif result_type == "icon_cache": + file = os.path.join(PROJECT, "icons", ".cache", name+".jpg") elif result_type == "cbz": file = os.path.join(settings.CBZ_EXPORT_PATH, PROJECT, name, chapter+".CBZ") elif result_type == "images_dir": diff --git a/Comics/utils/FileUtils.py b/Comics/utils/FileUtils.py index 94e8816..87d494e 100644 --- a/Comics/utils/FileUtils.py +++ b/Comics/utils/FileUtils.py @@ -22,6 +22,52 @@ class fileUtils: base_dir = os.path.dirname(file) if not os.path.exists(base_dir): os.makedirs(base_dir) return file + + @classmethod + def compare_size(cls, dst, file): + if os.path.exists(dst) and os.path.exists(file): + return os.stat(dst).st_size == os.stat(file).st_size + else: + return 0 + + """ + 图像编号 image-1.jpg + 如：存在image.png 返回 image-1.png 反之 image.png + """ + @classmethod + def file_check(cls, file, result="file"): + temp_file_name = file + count = 1 + files_size = [] + name, suffix = temp_file_name.split(".") + while count: + if os.path.exists(temp_file_name): + files_size.append(os.stat(temp_file_name).st_size) + temp_file_name = name+"-"+str(count)+"."+suffix + count += 1 + else: + if result == "size": + return files_size + else: + return temp_file_name + + @classmethod + def file_update(cls, old_file, new_file): + is_update = False + if os.path.exists(old_file): + is_update = os.stat(old_file).st_size not in cls.file_check(new_file, result="size") + return is_update + + # 判断是否需要更新封面 + @classmethod + def update_icon(cls, image_path, save_path): + # 不存在则更新 + if cls.file_update(image_path, save_path): + save_dir = os.path.dirname(save_path) + if not os.path.exists(save_dir): os.makedirs(save_dir) + logging.info(f"update icon ... {image_path} ===> {cls.file_check(save_path)}") + shutil.copyfile(image_path, cls.file_check(save_path)) + class CommonUtils: @classmethod