Enable HTTPCACHE_ALLOW_PREFIXS

Comics/middlewares.py

@@ -7,6 +7,15 @@ from scrapy import signals
 import random,logging
 from pathlib import Path
 from Comics.settings import PROXY_LIST
+# File cache settings begin
+import pickle,time
+from scrapy.http.request import Request
+from scrapy.spiders import Spider
+from scrapy.utils.python import to_bytes, to_unicode
+from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
+from scrapy.extensions.httpcache import FilesystemCacheStorage
+from Comics.settings import HTTPCACHE_ALLOW_PREFIXS
+# File cache settings end
 # useful for handling different item types with a single interface
 
 logger = logging.getLogger(__name__)
@@ -16,6 +25,27 @@ class ProxyMiddleware(object):
         if len(PROXY_LIST) != 0:
             request.meta["proxy"] = random.choice(PROXY_LIST)
 
+class MyFilesystemCacheStorage(FilesystemCacheStorage):
+    
+    def verify_next(self, url):
+        logging.info(f"cache url=== {url} {str(url).split('.')[-1]}")
+        if str(url).split('.')[-1] not in HTTPCACHE_ALLOW_PREFIXS:
+            logging.info(f"取消缓存 === {url}") 
+            return False
+        else:
+            logging.info(f"开始缓存 === {url}") 
+            return True
+    
+    def retrieve_response(self, spider: Spider, request: Request):
+        if self.verify_next(request._get_url()):
+            super().retrieve_response(spider, request)
+
+        
+    def store_response(self, spider: Spider, request: Request, response):
+        if self.verify_next(request._get_url()):
+            super().store_response(spider, request, response)
+    
+    
 class ComicsSpiderMiddleware:
     # Not all methods need to be defined. If a method is not defined,
     # scrapy acts as if the spider middleware does not modify the
@@ -107,4 +137,4 @@ class ComicsDownloaderMiddleware:
         pass
 
     def spider_opened(self, spider):
-        spider.logger.info('Spider opened: %s' % spider.name)
+        spider.logger.info('Spider opened: %s' % spider.name)

Comics/settings.py

@@ -106,8 +106,8 @@ HTTPCACHE_ENABLED = True
 HTTPCACHE_EXPIRATION_SECS = 0
 HTTPCACHE_DIR = 'httpcache'
 HTTPCACHE_IGNORE_HTTP_CODES = [500, 502, 404]
-#HTTPCACHE_STORAGE = 'Comics.middlewares.MyFilesystemCacheStorage'
-HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+HTTPCACHE_ALLOW_PREFIXS = [ 'jpg', 'png', 'gif']
+HTTPCACHE_STORAGE = 'Comics.middlewares.MyFilesystemCacheStorage'
 
 # Logging configuration
 LOG_LEVEL = "INFO" # 日志等级