update 20230806

Comics/spiders/rm_comic.py

@@ -1,32 +1,34 @@
-import scrapy,logging,time
+import scrapy,logging,time,os
 from Comics.items import ComicItem
 from Comics.loader import ComicLoader
 from Comics.items import ListComicItem
+from Comics.utils.Constant import ComicPath
+from Comics.settings import PROJECT_KEY
 
 class RmComicSpider(scrapy.Spider):
     name = 'rm_comic'
-    allowed_domains = ['rm01.xyz']
-    main_url = 'https://rm01.xyz'
-    start_urls = 'https://rm01.xyz/books'
+    allowed_domains = ['rouman5.com']
+    main_url = 'https://'+allowed_domains[0]
+    start_urls = main_url+'/books'
     
     def start_requests(self):
         yield scrapy.Request(self.start_urls, callback=self.books_comic)
     
     def books_comic(self, response):
-        books_comic = ComicLoader(item=ListComicItem(), response=response)
+        #books_comic = ComicLoader(item=ListComicItem(), response=response)
+        books_comic = ComicLoader(item=ComicItem(), response=response)
         data = books_comic.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
         str_exec = "props.pageProps.books"
         books = books_comic.get_exec(data, str_exec=str_exec)
         for book in  books:
-            books_comic.add_value('link', book['id'])
-            time.sleep(3)
-            yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic) 
-        
+            books_comic.add_value('link', self.start_urls+"/"+book['id'])
+            yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic)
         
     # 获取某个漫画的相关数据
     # 获取到多个章节链接后进入下个流程
     def parse_comic(self, response):
         comic_item = ComicLoader(item=ComicItem(), response=response)
+        comic_item.add_value(PROJECT_KEY, self.name)
         comic_item.add_xpath('name', '//div[@class="col"]/h5/text()')
         comic_item.add_xpath('icon', '//img[@class="img-thumbnail"]/@src')
         comic_item.add_xpath('author', '//div[contains(@class,"bookid_bookInfo")]/p[1]/text()', index=1)
@@ -39,9 +41,21 @@ class RmComicSpider(scrapy.Spider):
                                                '//div[contains(@class,"bookid_chapter")]/a/@href')
         #chapters = response.xpath('//div[contains(@class,"bookid_chapterBox")]'
         #                                   '//div[contains(@class,"bookid_chapter")]/a/text()').extract()
+
+        chapters = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]'
+                                           '//div[contains(@class,"bookid_chapter")]/a/text()')
         #for chapter, link in zip(chapters, chapter_href):
         for i, link in enumerate(chapter_href, start=1):
-            yield scrapy.Request(self.main_url+link, meta={'item': comic_item.load_item(), 'num': i}, callback=self.parse_chapter)
+        #    yield scrapy.Request(self.main_url+link, meta={'item': comic_item.load_item(), 'num': i}, callback=self.parse_chapter)
+            comic_item.add_value('chapter', chapters[i-1])
+            item = comic_item.load_item()
+            cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True)
+            if os.path.exists(cbz_path):
+                logging.info(f"漫画 {cbz_path} 已存在, 跳过中...")
+                return
+            else:
+                yield scrapy.Request(self.main_url+link, meta={'item': item, 'num': i}, callback=self.parse_chapter)
+
 
     # 读取某章节下的所有图片
     def parse_chapter(self, response):
@@ -71,4 +85,7 @@ class RmComicSpider(scrapy.Spider):
 
 
     def parse(self, response):
+        raise NotImplementedError
+    
+    def error_parse(self, response):
         raise NotImplementedError