update 20230806

Comics/items.py

@@ -57,6 +57,8 @@ class ListComicItem(Item):
 class ComicItem(Item):
     # 工程
     current_project = Field()
+    # link
+    link = Field()
     # 编号
     index = Field(output_processor=TakeFirst())
     # 漫画名

Comics/loader.py

@@ -10,7 +10,7 @@ class ComicLoader(ItemLoader):
                 data = data.get(dot)
         return data
 
-    def add_xpath(self, field_name, xpath, *processors, index=None, exec=None, re=None, **kw):
+    def add_xpath(self, field_name, xpath, *processors, index=None, exec=None, re=None, is_null=None, **kw):
         """
         Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a
         value, which is used to extract a list of strings from the
@@ -33,11 +33,15 @@ class ComicLoader(ItemLoader):
         if exec is not None:
             values = self.parseExec(values, exec)
         if index is not None:
+            if len(values) - 1 >= index:
                 try:
                     values = values[index]
                 except Exception as e:
                     logging.error(f"values [{values}] error index [{index}]")
                     logging.error(e)
+            else:
+                if is_null is not None: values = is_null
+                else: values=""
         self.add_value(field_name, values, *processors, re=re, **kw)
 
     def add_exec(self, field_name, value, str_exec=None, *processors, re=None, **kw):

Comics/middlewares.py

@@ -22,8 +22,13 @@ logger = logging.getLogger(__name__)
 
 class ProxyMiddleware(object):
     def process_request(self, request, spider):
+        url = request.url
+        logging.debug(f"proxy url=== {url} {str(url).split('.')[-1]}")
+        if str(url).split('.')[-1] not in HTTPCACHE_ALLOW_PREFIXS:
             if len(PROXY_LIST) != 0:
                 request.meta["proxy"] = random.choice(PROXY_LIST)
+        else:
+            logging.info(f"skip proxy... {url}")
 
 ################################################################
 # HTTP/HTTPS文件缓存中间件

Comics/pipelines.py

@@ -17,37 +17,23 @@ from Comics.utils.FileUtils import CBZUtils
 class ComicsPipeline:
     def open_spider(self, spider):
         pass
+
     # item就是yield后面的对象
     def process_item(self, item, spider):
         if isinstance(item, ComicItem):
             file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter'])
             item['count'] = len(item['images'])
             data = JsonExport(file=file).export_json(item, if_return=True)
-            data[PROJECT_KEY] = spider.name
+            #data[PROJECT_KEY] = spider.name
         return data 
         # image解析
 
-    def close_spider(self,spider):
+    def close_spider(self, spider):
         pass
 
 class ImgDownloadPipeline(ImagesPipeline):
     def get_file_path(self, item, file=None, result_type="image"):
-        if result_type == "image":
+        return ComicPath.get_file_path(item=item, file=file, result_type= result_type)
-            if os.path.sep not in file:
-                file = os.path.join(item[settings.PROJECT_KEY], "images", item['name'], item['chapter'], file)
-        elif result_type == "comic_info":
-            file = os.path.join(item[settings.PROJECT_KEY], "images", item['name'], item['chapter'])
-        elif result_type == "cbz_icon":
-            file = os.path.join(CBZ_EXPORT_PATH, item[settings.PROJECT_KEY], item['name'], item['chapter']+".jpg")
-        elif result_type == "down_icon":
-            file = self.download_icon(item,result_type='fullpath')
-        elif result_type == "icon":    
-            file = os.path.join(item[settings.PROJECT_KEY], "icons", item['name'], item['chapter']+".jpg")
-        elif result_type == "cbz":
-            file = os.path.join(CBZ_EXPORT_PATH, item[settings.PROJECT_KEY], item['name'], item['chapter']+".CBZ")
-        elif result_type == "images_dir":
-            file = os.path.join(settings.IMAGES_STORE, item[settings.PROJECT_KEY], "images", item['name'], item['chapter'])
-        return file 
 
     def image_scramble_exits(self, item,image_path):
         en_image_path = ComicPath().getFileScrambleImageSave(image_path, relative="fullpath") 

Comics/settings.py

@@ -18,6 +18,7 @@ OUTPUT_DIR = "output"
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
 #USER_AGENT = 'Comics (+http://www.yourdomain.com)'
 USER_AGENT = UserAgent().random
+# Override the default request headers:
 # Obey robots.txt rules
 ROBOTSTXT_OBEY = False
 

Comics/spiders/rm_comic.py

@@ -1,32 +1,34 @@
-import scrapy,logging,time
+import scrapy,logging,time,os
 from Comics.items import ComicItem
 from Comics.loader import ComicLoader
 from Comics.items import ListComicItem
+from Comics.utils.Constant import ComicPath
+from Comics.settings import PROJECT_KEY
 
 class RmComicSpider(scrapy.Spider):
     name = 'rm_comic'
-    allowed_domains = ['rm01.xyz']
+    allowed_domains = ['rouman5.com']
-    main_url = 'https://rm01.xyz'
+    main_url = 'https://'+allowed_domains[0]
-    start_urls = 'https://rm01.xyz/books'
+    start_urls = main_url+'/books'
     
     def start_requests(self):
         yield scrapy.Request(self.start_urls, callback=self.books_comic)
     
     def books_comic(self, response):
-        books_comic = ComicLoader(item=ListComicItem(), response=response)
+        #books_comic = ComicLoader(item=ListComicItem(), response=response)
+        books_comic = ComicLoader(item=ComicItem(), response=response)
         data = books_comic.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
         str_exec = "props.pageProps.books"
         books = books_comic.get_exec(data, str_exec=str_exec)
         for book in  books:
-            books_comic.add_value('link', book['id'])
+            books_comic.add_value('link', self.start_urls+"/"+book['id'])
-            time.sleep(3)
             yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic)
         
-        
     # 获取某个漫画的相关数据
     # 获取到多个章节链接后进入下个流程
     def parse_comic(self, response):
         comic_item = ComicLoader(item=ComicItem(), response=response)
+        comic_item.add_value(PROJECT_KEY, self.name)
         comic_item.add_xpath('name', '//div[@class="col"]/h5/text()')
         comic_item.add_xpath('icon', '//img[@class="img-thumbnail"]/@src')
         comic_item.add_xpath('author', '//div[contains(@class,"bookid_bookInfo")]/p[1]/text()', index=1)
@@ -39,9 +41,21 @@ class RmComicSpider(scrapy.Spider):
                                                '//div[contains(@class,"bookid_chapter")]/a/@href')
         #chapters = response.xpath('//div[contains(@class,"bookid_chapterBox")]'
         #                                   '//div[contains(@class,"bookid_chapter")]/a/text()').extract()
+
+        chapters = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]'
+                                           '//div[contains(@class,"bookid_chapter")]/a/text()')
         #for chapter, link in zip(chapters, chapter_href):
         for i, link in enumerate(chapter_href, start=1):
-            yield scrapy.Request(self.main_url+link, meta={'item': comic_item.load_item(), 'num': i}, callback=self.parse_chapter)
+        #    yield scrapy.Request(self.main_url+link, meta={'item': comic_item.load_item(), 'num': i}, callback=self.parse_chapter)
+            comic_item.add_value('chapter', chapters[i-1])
+            item = comic_item.load_item()
+            cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True)
+            if os.path.exists(cbz_path):
+                logging.info(f"漫画 {cbz_path} 已存在, 跳过中...")
+                return
+            else:
+                yield scrapy.Request(self.main_url+link, meta={'item': item, 'num': i}, callback=self.parse_chapter)
+
 
     # 读取某章节下的所有图片
     def parse_chapter(self, response):
@@ -72,3 +86,6 @@ class RmComicSpider(scrapy.Spider):
 
     def parse(self, response):
         raise NotImplementedError
+    
+    def error_parse(self, response):
+        raise NotImplementedError

Comics/utils/Constant.py

@@ -1,5 +1,6 @@
 import os.path
 import re
+from Comics import settings
 from opencc import OpenCC
 class ComicPath:
     PREFIX_SCRAMBLE = "scramble="
@@ -46,3 +47,30 @@ class ComicPath:
                 filename = filename[0:len(filename)+1-count]
                 break
         return filename
+
+    @classmethod
+    def get_file_path(cls, item, result_type="image", file=None, convert=False):
+        PROJECT = item[settings.PROJECT_KEY][0]
+        if not convert:
+            name = item['name']
+            chapter = item['chapter']
+        else:
+            name = cls.fix_file_name(cls.chinese_convert(item['name']))
+            chapter = cls.fix_file_name(cls.chinese_convert(item['chapter']))
+
+        if result_type == "image":
+            if os.path.sep not in file:
+                file = os.path.join(PROJECT, "images", name, chapter, file)
+        elif result_type == "comic_info":
+            file = os.path.join(PROJECT, "images", name, chapter)
+        elif result_type == "cbz_icon":
+            file = os.path.join(settings.CBZ_EXPORT_PATH, PROJECT, name, chapter+".jpg")
+        elif result_type == "down_icon":
+            file = os.path.join(settings.IMAGES_STORE, cls.get_file_path(item=item, result_type="icon"))
+        elif result_type == "icon":    
+            file = os.path.join(PROJECT, "icons", name+".jpg")
+        elif result_type == "cbz":
+            file = os.path.join(settings.CBZ_EXPORT_PATH, PROJECT, name, chapter+".CBZ")
+        elif result_type == "images_dir":
+            file = os.path.join(settings.IMAGES_STORE, PROJECT, "images", name, chapter)
+        return file