update

Comics/items.py

@@ -29,7 +29,10 @@ def _serialize_to_images(value, result_type=None):
             blocks_num = imageUtils.encodeImage(de_str)
             image_name = ComicPath.getFileScrambleImageName(count=count_image, block=blocks_num, suffix=suffix)
         #images_item.append(ImagesItem(image_name=count_image + suffix, image_url=image_src, image_path=image_name))
-        images_item.append(image_name)
+        if str(image_src).startswith('http'):
+            images_item.append(image_name)
+        else:
+            images_item.append(image_src)
         image_urls.append(image_src)
         count += 1
     logging.info(f"images_len: {len(images_item)}")
@@ -70,7 +73,8 @@ class ComicItem(Item):
     genre = Field(output_processor=TakeFirst())
     # 年龄分级
     age_rating = Field(output_processor=TakeFirst())
-
+    
+    count = Field()
     images_old = Field(serializer=serialize_to_images)
     images = Field(serializer=serialize_to_images)
     image_urls = Field(serializer=serialize_to_image_urls)
@@ -141,8 +145,8 @@ class ComicInfoItem(Item):
     Genre = Field(info='genre')# ","流派",True]
     Tags = Field(info='tags')# ","标签",True]
     Web = Field()# ","主页",False]
-    #PageCount = Field()# ","总页数",True]
+    PageCount = Field(info='count')# ","总页数",True]
-    PageCount = Field(info='images',serializer=serializer_info_images_count)# ","总页数",True]
+    #PageCount = Field(info='count',serializer=serializer_info_images_count)# ","总页数",True]
     LanguageISO = Field()#","语言",True]
     AgeRating = Field(info='age_rating')#","年龄分级",False]
     #Pages = Field(info='images_name', serializer=serializer_info_images_completed)#","页码",True]

Comics/loader.py

@@ -1,4 +1,4 @@
-import json
+import json,logging
 from scrapy.loader import ItemLoader
 
 class ComicLoader(ItemLoader):
@@ -33,7 +33,11 @@ class ComicLoader(ItemLoader):
         if exec is not None:
             values = self.parseExec(values, exec)
         if index is not None:
-            values = values[index]
+            try:
+                values = values[index]
+            except Exception as e:
+                logging.error(f"values [{values}] error index [{index}]")
+                logging.error(e)
         self.add_value(field_name, values, *processors, re=re, **kw)
 
     def add_exec(self, field_name, value, str_exec=None, *processors, re=None, **kw):

Comics/middlewares.py

@@ -25,15 +25,18 @@ class ProxyMiddleware(object):
         if len(PROXY_LIST) != 0:
             request.meta["proxy"] = random.choice(PROXY_LIST)
 
+################################################################
+# HTTP/HTTPS文件缓存中间件
+################################################################
 class MyFilesystemCacheStorage(FilesystemCacheStorage):
     
     def verify_next(self, url):
-        logging.info(f"cache url=== {url} {str(url).split('.')[-1]}")
+        logging.debug(f"cache url=== {url} {str(url).split('.')[-1]}")
         if str(url).split('.')[-1] not in HTTPCACHE_ALLOW_PREFIXS:
-            logging.info(f"取消缓存 === {url}") 
+            logging.debug(f"取消缓存 === {url}") 
             return False
         else:
-            logging.info(f"开始缓存 === {url}") 
+            logging.debug(f"开始缓存 === {url}") 
             return True
     
     def retrieve_response(self, spider: Spider, request: Request):

Comics/pipelines.py

@@ -7,15 +7,10 @@
 # useful for handling different item types with a single interface
 import os, scrapy,logging,time,random
 from Comics import settings
-from Comics.utils.FileUtils import imageUtils
-from Comics.utils.FileUtils import fileUtils
 from Comics.utils.Constant import ComicPath
 from Comics.items import ComicItem
-from Comics.items import ImagesItem
 from scrapy.pipelines.images import ImagesPipeline
-from Comics.exporters import ComicInfoXmlItemExporter
+from Comics.exporters import ComicInfoXmlItemExporter,JsonExport
-from Comics.exporters import ItemExporter
-from Comics.exporters import JsonExport
 from Comics.utils.FileUtils import CBZUtils
 
 class ComicsPipeline:
@@ -25,6 +20,9 @@ class ComicsPipeline:
     def process_item(self, item, spider):
         if isinstance(item, ComicItem):
             file = os.path.join(settings.OUTPUT_DIR,"json", item['name'], item['chapter'])
+            item['count'] = len(item['images'])
+            item['images'].append({'src':'icon.jpg', 'scramble': False})
+            item['image_urls'].append({'src': item['icon'] , 'scramble': False})
             data = JsonExport(file=file).export_json(item, if_return=True)
             #item['images'] = data['images']
         return data 

Comics/spiders/rm_comic.py

@@ -49,8 +49,8 @@ class RmComicSpider(scrapy.Spider):
         comic_item = ComicLoader(item=response.meta['item'], response=response)
         data = comic_item.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
         str_exec = "props.pageProps."
-        #comic_item.add_exec('name', data, str_exec=str_exec+"bookName")
+        comic_item.add_exec('name', data, str_exec=str_exec+"bookName")
-        #comic_item.add_exec('dep', data, str_exec=str_exec+"description")
+        comic_item.add_exec('dep', data, str_exec=str_exec+"description")
         comic_item.add_value('index', response.meta['num'])
         comic_item.add_exec('chapter', data, str_exec=str_exec + "chapterName")
         comic_item.add_exec('image_urls', data, str_exec+"images")

Comics/utils/FileUtils.py

@@ -275,7 +275,7 @@ class CBZUtils:
                     y = 0
                     for filename in filenames:
                         y = y + 1
-                        print("打包中：" + str(y) + "/" + str(len(filenames)), os.path.join(source_dir, filename))
+                        logging.info(f"打包中：" + str(y) + "/" + str(len(filenames)), os.path.join(source_dir, filename))
                         zf.write(path.joinpath(filename), arc_dir.joinpath(filename))
                 zf.close()
                 logging.info(f"打包完成：{target_file}")