update

Comics/spiders/rm_comic.py

@@ -1,26 +1,9 @@
+import urllib.parse
+
 import scrapy,json,requests
 from Comics.items import ComicItem
 from Comics.utils.FileUtils import CommonUtils
-import threading
-import toml
-
-class ErrorLog:
-    def __init__(self) -> None:
-        self.lock = threading.Lock()
-
-    def err_ls(self, dic):
-        self.lock.acquire()
-        with open('error.toml', 'r+t') as f:
-            data = toml.load('error.toml')
-            f.seek(0, 0)
-            f.truncate()
-            dic_name = f'err_{len(data)}'
-            data[dic_name] = dic
-            _ = toml.dump(data, f)
-        self.lock.release()
-
-
-error_logger = ErrorLog()
+from scrapy.loader import ItemLoader
 
 class RmComicSpider(scrapy.Spider):
     name = 'rm_comic'
@@ -29,45 +12,51 @@ class RmComicSpider(scrapy.Spider):
     #start_urls = ['https://rm01.xyz/books/63b65185-f798-4c8f-a0b0-8811615908fd/0']
     
     def start_requests(self):
-        yield scrapy.Request(self.main_url + '/books/0a7e8bd1-4cfa-481a-b067-1df663fb2017', callback=self.parse_comic)
+        yield scrapy.Request('https://rm01.xyz'
+                             '/books/306ec1e2-f701-4fda-bb78-041ad6ec4020', callback=self.parse_comic)
     
     def parse_comic(self, response):
         comic = ComicItem()
+#        comic_item = ItemLoader(item=ComicItem(), response=response)
         comic['name'] = response.xpath('//div[@class="col"]/h5/text()').extract_first()
         comic['icon'] = response.xpath('//img[@class="img-thumbnail"]/@src').extract_first()
         comic['author'] = response.xpath('//div[contains(@class,"bookid_bookInfo")]/p[1]/text()').extract()[1]
         comic['tags'] = response.xpath('//div[contains(@class,"bookid_bookInfo")]/p[3]/b/text()').extract_first()
         comic['dep'] = response.xpath('//div[contains(@class,"bookid_bookInfo")]/p[4]/text()').extract()[1]
         comic['date'] = response.xpath('//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()').extract()[1]
-        comic['chapters'] = response.xpath('//div[contains(@class,"bookid_chapterBox")]//div[contains(@class,"bookid_chapter")]/a/text()').extract()
-        comic['chapter_href'] = response.xpath('//div[contains(@class,"bookid_chapterBox")]//div[contains(@class,"bookid_chapter")]/a/@href').extract()
-        for link in comic['chapter_href']:
-            yield scrapy.Request(self.main_url+link,meta={'item' : comic} , callback=self.parse_chapter,errback=self.err)
-    
-    def err(self):
-        print("Error=====")
-        
+        comic['genre'] = "韩漫"
+        comic['age_rating'] = "R18+"
+        chapters = response.xpath('//div[contains(@class,"bookid_chapterBox")]'
+                                           '//div[contains(@class,"bookid_chapter")]/a/text()').extract()
+        chapter_href = response.xpath('//div[contains(@class,"bookid_chapterBox")]'
+                                               '//div[contains(@class,"bookid_chapter")]/a/@href').extract()
+        #for chapter, link in zip(chapters, chapter_href):
+        for i, link in enumerate(chapter_href, start=1):
+                yield scrapy.Request(self.main_url+link, meta={'item' : comic, 'number': i}, callback=self.parse_chapter)
+
     def parse_chapter(self, response):
         item = response.meta['item']
+        number = response.meta['number']
         data = response.xpath('//script[@id="__NEXT_DATA__"]/text()').extract_first()
-        str_exec="props.pageProps."
-        comic_name = CommonUtils.parseExec(data,str_exec+"bookName")
-        chapterName = CommonUtils.parseExec(data,str_exec+"chapterName")
-        description = CommonUtils.parseExec(data,str_exec+"description")
-        images = CommonUtils.parseExec(data,str_exec+"images")
-        chapter_api_url = CommonUtils.parseExec(data,str_exec+"chapterAPIPath")
+        str_exec = "props.pageProps."
+        comic_name = CommonUtils.parseExec(data, str_exec+"bookName")
+        chapterName = CommonUtils.parseExec(data, str_exec+"chapterName")
+        description = CommonUtils.parseExec(data, str_exec+"description")
+        images = CommonUtils.parseExec(data, str_exec+"images")
+        chapter_api_url = CommonUtils.parseExec(data, str_exec+"chapterAPIPath")
         item['chapter'] = chapterName
         item['list_img'] = images
+        item['index'] = number
         if chapter_api_url != None:
-            yield scrapy.Request(url=self.main_url+chapter_api_url,meta={'item' : item}, callback=self.parse_chapter_api, errback=self.err)
+            yield scrapy.Request(self.main_url+ chapter_api_url,meta={'item' : item}, callback= self.parse_chapter_api)
         else:
-            item['list_img'] = images
             yield item
-    
-    def parse_chapter_api(self,response,item):
-        data = response.meta['item']
-        print(item)
-        return response
-         
+
+    def parse_chapter_api(self, response):
+        item = response.meta['item']
+        item['chapter'] = CommonUtils.parseExec(response.text, "chapter.name")
+        item['list_img'] = CommonUtils.parseExec(response.text, "chapter.images")
+        yield item
+
     def parse(self, response):
         raise NotImplementedError