fix

Comics/items.py

@@ -24,9 +24,13 @@ def _serialize_to_images(value, result_type=None):
     images_item = []
     image_urls = []
     for image in value:
-        (image_src, scramble) = [image.get("src"), image.get("scramble")]
+        try:
+            (image_src, scramble) = [image.get("src"), image.get("scramble")]
+        except:
+            (image_src, scramble) = [image, False]
         count_image = settings.IMAGES_NAME_FORMAT.format(count)
-        suffix = "."+str(image_src).split(".")[-1]
+#        suffix = "."+str(image_src).split(".")[-1]
+        suffix = ".jpg"
         image_name = count_image + suffix
         if scramble:
             de_str = str(image_src).split("/")[-1].replace(suffix, "==")

Comics/loader.py

@@ -1,8 +1,9 @@
 import json,logging
 from scrapy.loader import ItemLoader
+from Comics.settings import PROJECT_KEY
 
 class ComicLoader(ItemLoader):
-    def parseExec(cls,data,exec):
+    def parseExec(self,data,exec):
         if data !=None and exec != None:
             dots = str(exec).split(".")
             if not isinstance(data,dict): data = json.loads(data)
@@ -62,3 +63,71 @@ class ComicLoader(ItemLoader):
             self._replace_value(field_name, value)
             return False 
         else: return True
+
+    
+    # 设置漫画属性
+    def set_properties(self, name, value=None, xpath=None, index=None, sexec=None):
+        if value != None and sexec==None:
+            self.add_value(field_name=name, value=value)
+        if xpath != None:
+            self.add_xpath(field_name=name, xpath=xpath, index=index)
+        if sexec != None:
+            self.add_exec(field_name=name, value=value, str_exec=sexec)
+
+    # 工程名
+    def project_name(self, project_name): self.add_value(PROJECT_KEY, project_name)
+    # 漫画名
+    def name(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('name', value, xpath, index, sexec)
+    # 漫画封面链接
+    def icon(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('icon', value, xpath, index, sexec)
+    # 作者
+    def author(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('author', value, xpath, index, sexec)
+    # 标签
+    def tags(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('tags', value, xpath, index, sexec)
+    # 概述
+    def dep(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('dep', value, xpath, index, sexec)
+    # 时间
+    def date(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('date', value, xpath, index, sexec)
+    # 流派
+    def genre(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('genre', value, xpath, index, sexec)
+    # 年龄分级
+    def age_rating(self, value=None, xpath=None, index=None, sexec=None): 
+        self.set_properties('age_rating', value, xpath, index, sexec)
+    # 全部章节
+    def chapters(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('chapters', value, xpath, index, sexec)
+    # 单一章节
+    def chapter(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('chapter', value, xpath, index, sexec)
+    # 图像名称
+    def images(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('images', value, xpath, index, sexec)
+    # 图像链接
+    def image_urls(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('image_urls', value, xpath, index, sexec)
+
+class ComicEntity:
+
+    ENTITY = None
+
+    def __init__(self, entity):
+        self.ENTITY = entity
+
+    # 属性获取
+    def get_dict(self, key):
+        try:
+            return self.ENTITY[key]
+        except:
+            return []
+
+    def set(self, key, value): self.ENTITY[key] = value
+    def set_count(self, value): self.set('count', value)
+    def set_index(self, value): self.set('index', value)
+    def images(self): return self.get_dict("images")
+    def image_urls(self): return self.get_dict("image_urls")
+    def chapters(self): return self.get_dict("chapters")
+    def chapter(self): return self.get_dict("chapter")
+    def icon(self): return self.get_dict("icon")
+    def count(self): return self.set_count(len(self.images()))
+    def index(self): return self.set_index(self.chapters().index(self.chapter()) + 1)
+
+    def item(self): 
+        self.count()
+        self.index()
+        return self.ENTITY

Comics/pipelines.py

@@ -14,7 +14,7 @@ from scrapy.pipelines.images import ImagesPipeline
 from Comics.exporters import ComicInfoXmlItemExporter,JsonExport,ItemExporter
 from Comics.utils.FileUtils import CBZUtils
 from Comics.utils.FileUtils import fileUtils as fu 
-
+from Comics.loader import ComicEntity
 
 class ComicsPipeline:
     def open_spider(self, spider):
@@ -24,15 +24,11 @@ class ComicsPipeline:
     def process_item(self, item, spider):
         if isinstance(item, ComicItem):
             # 'output/rm_comic/json/壞X/第1話 壞X'
-            cbz_path = ComicPath.get_file_path(item, result_type="cbz", convert=True)
+            if os.path.exists(ComicPath.CBZ(item=item)):
-            if os.path.exists(cbz_path):
-                item['image_urls'] = item['images'] = []
                 return ItemExporter().export_obj(item)
             else:
                 file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter'])
-                item['count'] = len(item['images'])
+                return JsonExport(file=file).export_json(ComicEntity(item).item(), if_return=True)
-                item['index'] = item['chapters'].index(item['chapter']) + 1
-                return JsonExport(file=file).export_json(item, if_return=True)
         # image解析
 
     def close_spider(self, spider):
@@ -61,8 +57,9 @@ class ImgDownloadPipeline(ImagesPipeline):
     def file_path(self, request, response=None, info=None, *, item=None): return request.meta['path']
 
     def get_media_requests(self, item, info):
-        self.image_urls = item['image_urls']
+        comic = ComicEntity(item)
-        self.images = item['images']
+        self.image_urls = comic.image_urls() 
+        self.images = comic.images()
         # 下载封面
         self.download_icon(item)
         for image_url,image in zip(self.image_urls,self.images):

Comics/spiders/rm_comic.py

@@ -22,29 +22,29 @@ class RmComicSpider(scrapy.Spider):
         data = comics.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
         for book in comics.get_exec(data, str_exec="props.pageProps.books"):
             comics.add_value('link', self.start_urls+"/"+book['id'])
-            if not book['name'] in skip.skip_comic:
+            if book['name'] not in skip.skip_comic:
                 yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic)
 
     # 获取某个漫画的相关数据
     # 获取到多个章节链接后进入下个流程
     def parse_comic(self, response):
         comic_item = ComicLoader(item=ComicItem(), response=response)
-        comic_item.add_value(PROJECT_KEY, self.name)
+        comic_item.project_name(self.name)
-        comic_item.add_xpath('name', '//div[@class="col"]/h5/text()')
+        comic_item.name(xpath='//div[@class="col"]/h5/text()')
-        comic_item.add_xpath('icon', '//img[@class="img-thumbnail"]/@src')
+        comic_item.icon(xpath='//img[@class="img-thumbnail"]/@src')
-        comic_item.add_xpath('author', '//div[contains(@class,"bookid_bookInfo")]/p[1]/text()', index=1)
+        comic_item.author(xpath='//div[contains(@class,"bookid_bookInfo")]/p[1]/text()', index=1)
-        comic_item.add_xpath('tags', '//div[contains(@class,"bookid_bookInfo")]/p[3]/b/text()')
+        comic_item.tags(xpath='//div[contains(@class,"bookid_bookInfo")]/p[3]/b/text()')
-        comic_item.add_xpath('dep', '//div[contains(@class,"bookid_bookInfo")]/p[4]/text()', index=1)
+        comic_item.dep(xpath='//div[contains(@class,"bookid_bookInfo")]/p[4]/text()', index=1)
-        comic_item.add_xpath('date', '//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()', index=1)
+        comic_item.date(xpath='//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()', index=1)
-        comic_item.add_value('genre', "韩漫")
+        comic_item.genre(value="韩漫")
-        comic_item.add_value('age_rating', "R18+")
+        comic_item.age_rating(value="R18+")
         chapter_href = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]'
                                                '//div[contains(@class,"bookid_chapter")]/a/@href')
         chapters = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]'
                                            '//div[contains(@class,"bookid_chapter")]/a/text()')
         for chapter, link in zip(chapters, chapter_href):
-            comic_item.add_value('chapter', chapter)
+            comic_item.chapters(value=chapters)
-            comic_item.add_value('chapters', chapters)
+            comic_item.chapter(value=chapter)
             item = comic_item.load_item()
             cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True)
             if os.path.exists(cbz_path):
@@ -59,11 +59,11 @@ class RmComicSpider(scrapy.Spider):
         comic_item = ComicLoader(item=response.meta['item'], response=response)
         data = comic_item.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
         str_exec = "props.pageProps."
-        comic_item.add_exec('name', data, str_exec=str_exec+"bookName")
+        comic_item.name(value=data, sexec=str_exec+"bookName")
-        comic_item.add_exec('dep', data, str_exec=str_exec+"description")
+        comic_item.dep(value=data, sexec=str_exec+"description")
-        comic_item.add_exec('chapter', data, str_exec=str_exec + "chapterName")
+        comic_item.chapter(value=data, sexec=str_exec+"chapterName")
-        comic_item.add_exec('image_urls', data, str_exec+"images")
+        comic_item.image_urls(value=data, sexec=str_exec+"images")
-        comic_item.add_exec('images', data, str_exec+"images")
+        comic_item.images(value=data, sexec=str_exec+"images")
         comic = comic_item.load_item()
         chapter_api_url = comic_item.get_exec(data, str_exec+"chapterAPIPath")
         if chapter_api_url is not None:
@@ -74,9 +74,9 @@ class RmComicSpider(scrapy.Spider):
     # 加密数据API处理
     def parse_chapter_api(self, response):
         comic_item = ComicLoader(item=response.meta['item'], response=response)
-        comic_item.add_exec('chapter', response.text, str_exec='chapter.name')
+        comic_item.chapter(value=response.text, sexec='chapter.name')
-        comic_item.add_exec('image_urls', response.text, str_exec='chapter.images')
+        comic_item.image_urls(value=response.text, sexec='chapter.images')
-        comic_item.add_exec('images', response.text, str_exec='chapter.images')
+        comic_item.images(value=response.text, sexec='chapter.images')
         yield comic_item.load_item()
 
 

Comics/spiders/yh_comic.py

@@ -0,0 +1,69 @@
+import scrapy,logging,time,os
+from Comics.items import ComicItem
+from Comics.loader import ComicLoader
+from Comics.items import ListComicItem
+from Comics.utils.Constant import ComicPath
+from Comics.settings import PROJECT_KEY
+import skip
+
+class RmComicSpider(scrapy.Spider):
+    name = 'yh_comic'
+    allowed_domains = ['www.shuanglilock.com.cn']
+    main_url = 'https://'+allowed_domains[0]
+    start_urls = main_url+'/info'
+    
+    def start_requests(self):
+#        for x in range(0,60):
+            yield scrapy.Request("https://www.shuanglilock.com.cn/info/27145/", callback=self.parse_comic)
+
+    # 获取多个漫画信息
+#    def books_comic(self, response):
+#        comics = ComicLoader(item=ComicItem(), response=response)
+#        data = comics.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
+#        for book in comics.get_exec(data, str_exec="props.pageProps.books"):
+#            comics.add_value('link', self.start_urls+"/"+book['id'])
+#            if book['name'] not in skip.skip_comic:
+#                yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic)
+
+    # 获取某个漫画的相关数据
+    # 获取到多个章节链接后进入下个流程
+    def parse_comic(self, response):
+        comic_item = ComicLoader(item=ComicItem(), response=response)
+        comic_item.project_name(self.name)
+        comic_item.name(xpath='//div[@class="comics-detail__info"]/h1[@class="comics-detail__title"]/text()')
+        comic_item.icon(xpath='//div[@class="pure-u-1-1 pure-u-sm-1-3 pure-u-md-1-6"]/img/@src')
+        comic_item.author(xpath='//div[@class="comics-detail__info"]/h2[@class="comics-detail__author"]/text()')
+        comic_item.tags(xpath='//div[@class="tag-list"]/a[@class="tag"]/text()')
+        comic_item.dep(xpath='//p[contains(@class,"comics-detail__desc")]/text()')
+        #comic_item.date(xpath='//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()', index=1)
+        comic_item.genre(value="樱花漫画")
+        #comic_item.age_rating(value="R18+")
+        chapter_href = comic_item.get_xpath('//div[contains(@id,"chapter-items")]'
+                                               '//a[@class="comics-chapters__item"]/@href')
+        chapters = comic_item.get_xpath('//div[contains(@id,"chapter-items")]'
+                                           '//a[@class="comics-chapters__item"]//span/text()')
+        for chapter, link in zip(chapters, chapter_href):
+            comic_item.chapters(value=chapters)
+            comic_item.chapter(value=chapter)
+            item = comic_item.load_item()
+            cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True)
+            if os.path.exists(cbz_path):
+                logging.info(f"漫画 {cbz_path} 已存在, 跳过中...")
+                yield item
+            else:
+                yield scrapy.Request(self.main_url+link, meta={'item': item}, callback=self.parse_chapter)
+
+
+    # 读取某章节下的所有图片
+    def parse_chapter(self, response):
+        comic_item = ComicLoader(item=response.meta['item'], response=response)
+        comic_item.image_urls(xpath='//div[@class="comiclist"]/div[@class="comicpage"]/div/img/@data-original')
+        comic_item.images(xpath='//div[@class="comiclist"]/div[@class="comicpage"]/div/img/@data-original')
+        comic = comic_item.load_item()
+        yield comic 
+
+    def parse(self, response):
+        raise NotImplementedError
+    
+    def error_parse(self, response):
+        raise NotImplementedError

Comics/utils/Constant.py

@@ -79,6 +79,11 @@ class ComicPath:
             file = os.path.join(settings.IMAGES_STORE, PROJECT, "images", name, chapter)
         return file
 
+    @classmethod
+    def CBZ(cls, item):
+        return cls.get_file_path(item, result_type="cbz", convert=True)
+
+
 class ntfy:
     @classmethod
     def sendMsg(cls, msg,alert=False,sleep=None,error=None):