diff --git a/Comics/items.py b/Comics/items.py index 92fc8d4..cf2eeda 100644 --- a/Comics/items.py +++ b/Comics/items.py @@ -29,7 +29,10 @@ def _serialize_to_images(value, result_type=None): blocks_num = imageUtils.encodeImage(de_str) image_name = ComicPath.getFileScrambleImageName(count=count_image, block=blocks_num, suffix=suffix) #images_item.append(ImagesItem(image_name=count_image + suffix, image_url=image_src, image_path=image_name)) - images_item.append(image_name) + if str(image_src).startswith('http'): + images_item.append(image_name) + else: + images_item.append(image_src) image_urls.append(image_src) count += 1 logging.info(f"images_len: {len(images_item)}") @@ -70,7 +73,8 @@ class ComicItem(Item): genre = Field(output_processor=TakeFirst()) # 年龄分级 age_rating = Field(output_processor=TakeFirst()) - + + count = Field() images_old = Field(serializer=serialize_to_images) images = Field(serializer=serialize_to_images) image_urls = Field(serializer=serialize_to_image_urls) @@ -141,8 +145,8 @@ class ComicInfoItem(Item): Genre = Field(info='genre')# ","流派",True] Tags = Field(info='tags')# ","标签",True] Web = Field()# ","主页",False] - #PageCount = Field()# ","总页数",True] - PageCount = Field(info='images',serializer=serializer_info_images_count)# ","总页数",True] + PageCount = Field(info='count')# ","总页数",True] + #PageCount = Field(info='count',serializer=serializer_info_images_count)# ","总页数",True] LanguageISO = Field()#","语言",True] AgeRating = Field(info='age_rating')#","年龄分级",False] #Pages = Field(info='images_name', serializer=serializer_info_images_completed)#","页码",True] diff --git a/Comics/loader.py b/Comics/loader.py index 28d3bb6..0be9359 100644 --- a/Comics/loader.py +++ b/Comics/loader.py @@ -1,4 +1,4 @@ -import json +import json,logging from scrapy.loader import ItemLoader class ComicLoader(ItemLoader): @@ -33,7 +33,11 @@ class ComicLoader(ItemLoader): if exec is not None: values = self.parseExec(values, exec) if index is not None: - values = values[index] + try: + values = values[index] + except Exception as e: + logging.error(f"values [{values}] error index [{index}]") + logging.error(e) self.add_value(field_name, values, *processors, re=re, **kw) def add_exec(self, field_name, value, str_exec=None, *processors, re=None, **kw): diff --git a/Comics/middlewares.py b/Comics/middlewares.py index e988091..5c015a5 100644 --- a/Comics/middlewares.py +++ b/Comics/middlewares.py @@ -25,15 +25,18 @@ class ProxyMiddleware(object): if len(PROXY_LIST) != 0: request.meta["proxy"] = random.choice(PROXY_LIST) +################################################################ +# HTTP/HTTPS文件缓存中间件 +################################################################ class MyFilesystemCacheStorage(FilesystemCacheStorage): def verify_next(self, url): - logging.info(f"cache url=== {url} {str(url).split('.')[-1]}") + logging.debug(f"cache url=== {url} {str(url).split('.')[-1]}") if str(url).split('.')[-1] not in HTTPCACHE_ALLOW_PREFIXS: - logging.info(f"取消缓存 === {url}") + logging.debug(f"取消缓存 === {url}") return False else: - logging.info(f"开始缓存 === {url}") + logging.debug(f"开始缓存 === {url}") return True def retrieve_response(self, spider: Spider, request: Request): diff --git a/Comics/pipelines.py b/Comics/pipelines.py index 948c674..225a3b3 100644 --- a/Comics/pipelines.py +++ b/Comics/pipelines.py @@ -7,15 +7,10 @@ # useful for handling different item types with a single interface import os, scrapy,logging,time,random from Comics import settings -from Comics.utils.FileUtils import imageUtils -from Comics.utils.FileUtils import fileUtils from Comics.utils.Constant import ComicPath from Comics.items import ComicItem -from Comics.items import ImagesItem from scrapy.pipelines.images import ImagesPipeline -from Comics.exporters import ComicInfoXmlItemExporter -from Comics.exporters import ItemExporter -from Comics.exporters import JsonExport +from Comics.exporters import ComicInfoXmlItemExporter,JsonExport from Comics.utils.FileUtils import CBZUtils class ComicsPipeline: @@ -25,6 +20,9 @@ class ComicsPipeline: def process_item(self, item, spider): if isinstance(item, ComicItem): file = os.path.join(settings.OUTPUT_DIR,"json", item['name'], item['chapter']) + item['count'] = len(item['images']) + item['images'].append({'src':'icon.jpg', 'scramble': False}) + item['image_urls'].append({'src': item['icon'] , 'scramble': False}) data = JsonExport(file=file).export_json(item, if_return=True) #item['images'] = data['images'] return data diff --git a/Comics/spiders/rm_comic.py b/Comics/spiders/rm_comic.py index 71d062b..4cf4e9c 100644 --- a/Comics/spiders/rm_comic.py +++ b/Comics/spiders/rm_comic.py @@ -49,8 +49,8 @@ class RmComicSpider(scrapy.Spider): comic_item = ComicLoader(item=response.meta['item'], response=response) data = comic_item.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0] str_exec = "props.pageProps." - #comic_item.add_exec('name', data, str_exec=str_exec+"bookName") - #comic_item.add_exec('dep', data, str_exec=str_exec+"description") + comic_item.add_exec('name', data, str_exec=str_exec+"bookName") + comic_item.add_exec('dep', data, str_exec=str_exec+"description") comic_item.add_value('index', response.meta['num']) comic_item.add_exec('chapter', data, str_exec=str_exec + "chapterName") comic_item.add_exec('image_urls', data, str_exec+"images") diff --git a/Comics/utils/FileUtils.py b/Comics/utils/FileUtils.py index 69072a0..036fa4a 100644 --- a/Comics/utils/FileUtils.py +++ b/Comics/utils/FileUtils.py @@ -275,7 +275,7 @@ class CBZUtils: y = 0 for filename in filenames: y = y + 1 - print("打包中：" + str(y) + "/" + str(len(filenames)), os.path.join(source_dir, filename)) + logging.info(f"打包中：" + str(y) + "/" + str(len(filenames)), os.path.join(source_dir, filename)) zf.write(path.joinpath(filename), arc_dir.joinpath(filename)) zf.close() logging.info(f"打包完成：{target_file}")