fix

Comics/pipelines.py

@@ -13,6 +13,8 @@ from Comics.items import ComicItem
 from scrapy.pipelines.images import ImagesPipeline
 from Comics.exporters import ComicInfoXmlItemExporter,JsonExport
 from Comics.utils.FileUtils import CBZUtils
+from Comics.utils.FileUtils import fileUtils as fu 
+
 
 class ComicsPipeline:
     def open_spider(self, spider):
@@ -21,11 +23,12 @@ class ComicsPipeline:
     # item就是yield后面的对象
     def process_item(self, item, spider):
         if isinstance(item, ComicItem):
+            # 'output/rm_comic/json/壞X/第1話 壞X'
             file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter'])
             item['count'] = len(item['images'])
             item['index'] = item['chapters'].index(item['chapter']) + 1
+
             data = JsonExport(file=file).export_json(item, if_return=True)
-            #data[PROJECT_KEY] = spider.name
         return data 
         # image解析
 
@@ -42,7 +45,7 @@ class ImgDownloadPipeline(ImagesPipeline):
     
     ## Icon Path : CBZ/NAME/CHAPTER.jpg
     def download_icon(self, item, result_type="download"):
-        icon_path = self.get_file_path(item, result_type="icon")
+        icon_path = self.get_file_path(item, result_type="icon_cache")
         if result_type == "fullpath":
             return os.path.join(settings.IMAGES_STORE, icon_path)
         if os.path.exists(icon_path):
@@ -60,20 +63,39 @@ class ImgDownloadPipeline(ImagesPipeline):
         # 下载封面
         self.download_icon(item)
         for image_url,image in zip(self.image_urls,self.images):
+            is_down = True
             image_path = self.get_file_path(item, image)
             if self.image_scramble_exits(item, image_path):
-                logging.info(f"file exists: IMAGE_STORE {image_path}")
+                if image_path == self.get_file_path(item, result_type="icon_cache"):
+                    logging.info(f"icon file exists: IMAGE_STORE {image_path}")
                 else:
+                    is_down = False
+                    logging.info(f"file exists: IMAGE_STORE {image_path}")
+            if is_down:
                 logging.info(f"downloading {image_url} --> IMAGE_STORE {image_path}")
                 yield scrapy.Request(url=image_url, meta={'path': image_path})
 
+    # 打包cbz封面
     def pack_icon(self, item):
         cbz_icon = self.get_file_path(item=item, result_type="cbz_icon")
         dwn_icon = self.get_file_path(item=item, result_type="down_icon")
-        logging.info(f"icon packing {dwn_icon} => {cbz_icon}")
+        base_dir = os.path.dirname(dwn_icon)
-        cbz_icon_dir = os.path.dirname(cbz_icon)
+        name = os.path.basename(dwn_icon).split(".")[0]
-        if not os.path.exists(cbz_icon_dir): os.makedirs(cbz_icon_dir)
+        for dirname in os.listdir(base_dir):
-        shutil.copyfile(dwn_icon, cbz_icon)
+            path = os.path.join(base_dir, dirname) 
+            if os.path.isfile(path) and dirname.startswith(name):
+                fu.update_icon(path, cbz_icon)
+
+
+
+    # 判断是否需要更新封面
+    def update_icon(self, item):
+        # 下载后的缓存封面路径
+        image_path = self.get_file_path(item, result_type="down_cache_icon")
+        # 最终封面保存路径
+        save_path =  self.get_file_path(item=item, result_type="down_icon")
+
+        fu.update_icon(image_path, save_path)
 
     def item_completed(self, results, item, info):
         # return item
@@ -83,6 +105,7 @@ class ImgDownloadPipeline(ImagesPipeline):
         if CBZUtils.packComicChapterCBZ(src_dir= self.get_file_path(item, result_type="images_dir"),
                                         dts_path= self.get_file_path(item, result_type="cbz"),
                                      comic_info_images= comic_info['Pages'], remove=True):
+            self.update_icon(item)
             self.pack_icon(item)
         
 #        sleep_time = random.randint(5,30)

Comics/settings.py

@@ -68,7 +68,7 @@ COOKIES_ENABLED = False
 # Enable or disable downloader middlewares
 # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
 DOWNLOADER_MIDDLEWARES = {
-#    'Comics.middlewares.ComicsDownloaderMiddleware': 543,
+    'Comics.middlewares.ComicsDownloaderMiddleware': 543,
 #    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
     'Comics.middlewares.ProxyMiddleware': 100,
     'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 400,

Comics/spiders/rm_comic.py

@@ -40,14 +40,9 @@ class RmComicSpider(scrapy.Spider):
         comic_item.add_value('age_rating', "R18+")
         chapter_href = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]'
                                                '//div[contains(@class,"bookid_chapter")]/a/@href')
-        #chapters = response.xpath('//div[contains(@class,"bookid_chapterBox")]'
-        #                                   '//div[contains(@class,"bookid_chapter")]/a/text()').extract()
-
         chapters = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]'
                                            '//div[contains(@class,"bookid_chapter")]/a/text()')
         for chapter, link in zip(chapters, chapter_href):
-        #for i, link in enumerate(chapter_href, start=1):
-        #    yield scrapy.Request(self.main_url+link, meta={'item': comic_item.load_item(), 'num': i}, callback=self.parse_chapter)
             comic_item.add_value('chapter', chapter)
             comic_item.add_value('chapters', chapters)
             item = comic_item.load_item()

Comics/utils/Constant.py

@@ -67,8 +67,12 @@ class ComicPath:
             file = os.path.join(settings.CBZ_EXPORT_PATH, PROJECT, name, chapter+".jpg")
         elif result_type == "down_icon":
             file = os.path.join(settings.IMAGES_STORE, cls.get_file_path(item=item, result_type="icon"))
+        elif result_type == "down_cache_icon":
+            file = os.path.join(settings.IMAGES_STORE, cls.get_file_path(item=item, result_type="icon_cache"))
         elif result_type == "icon":    
-            file = os.path.join(PROJECT, "icons", name+".jpg")
+            file = os.path.join(PROJECT, "icons", name, name+".jpg")
+        elif result_type == "icon_cache":
+            file = os.path.join(PROJECT, "icons", ".cache", name+".jpg")
         elif result_type == "cbz":
             file = os.path.join(settings.CBZ_EXPORT_PATH, PROJECT, name, chapter+".CBZ")
         elif result_type == "images_dir":

Comics/utils/FileUtils.py

@@ -23,6 +23,52 @@ class fileUtils:
         if not os.path.exists(base_dir): os.makedirs(base_dir)
         return file
         
+    @classmethod
+    def compare_size(cls, dst, file):
+        if os.path.exists(dst) and os.path.exists(file):
+            return os.stat(dst).st_size == os.stat(file).st_size
+        else:
+            return 0
+
+    """
+      图像编号 image-1.jpg
+      如：存在image.png 返回 image-1.png 反之 image.png
+    """
+    @classmethod
+    def file_check(cls, file, result="file"):
+        temp_file_name = file
+        count = 1
+        files_size = []
+        name, suffix = temp_file_name.split(".")
+        while count:
+            if os.path.exists(temp_file_name):
+                files_size.append(os.stat(temp_file_name).st_size)
+                temp_file_name = name+"-"+str(count)+"."+suffix
+                count += 1
+            else:
+                if result == "size":
+                    return files_size
+                else:
+                    return temp_file_name
+
+    @classmethod
+    def file_update(cls, old_file, new_file):
+        is_update = False
+        if os.path.exists(old_file):
+            is_update =  os.stat(old_file).st_size not in cls.file_check(new_file, result="size")
+        return is_update
+
+    # 判断是否需要更新封面
+    @classmethod
+    def update_icon(cls, image_path, save_path):
+        # 不存在则更新
+        if cls.file_update(image_path, save_path):
+            save_dir = os.path.dirname(save_path)
+            if not os.path.exists(save_dir): os.makedirs(save_dir)
+            logging.info(f"update icon ... {image_path} ===> {cls.file_check(save_path)}")
+            shutil.copyfile(image_path, cls.file_check(save_path))
+
+
 class CommonUtils:
     @classmethod
     def parseExec(cls,data,exec):