cwx/ComicScrapy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
caiwx86 2023-09-06 02:50:16 +08:00
parent 746879d985
commit e4ac2cc631
2 changed files with 22 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Comics/pipelines.py
View File

@ -11,7 +11,7 @@ from Comics.settings import CBZ_EXPORT_PATH,OUTPUT_DIR,PROJECT_KEY
from Comics.utils.Constant import ComicPath from Comics.utils.Constant import ComicPath
from Comics.items import ComicItem from Comics.items import ComicItem
from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.images import ImagesPipeline
from Comics.exporters import ComicInfoXmlItemExporter,JsonExport from Comics.exporters import ComicInfoXmlItemExporter,JsonExport,ItemExporter
from Comics.utils.FileUtils import CBZUtils from Comics.utils.FileUtils import CBZUtils
from Comics.utils.FileUtils import fileUtils as fu from Comics.utils.FileUtils import fileUtils as fu
@ -24,12 +24,17 @@ class ComicsPipeline:
def process_item(self, item, spider): def process_item(self, item, spider):
if isinstance(item, ComicItem): if isinstance(item, ComicItem):
# 'output/rm_comic/json/壞X/第1話 壞X' # 'output/rm_comic/json/壞X/第1話 壞X'
cbz_path = ComicPath.get_file_path(item, result_type="cbz", convert=True)
if os.path.exists(cbz_path):
item['image_urls'] = []
item['images'] = []
return ItemExporter().export_obj(item)
else:
file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter']) file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter'])
item['count'] = len(item['images']) item['count'] = len(item['images'])
item['index'] = item['chapters'].index(item['chapter']) + 1 item['index'] = item['chapters'].index(item['chapter']) + 1
data = JsonExport(file=file).export_json(item, if_return=True) return JsonExport(file=file).export_json(item, if_return=True)
return data
# image解析 # image解析
def close_spider(self, spider): def close_spider(self, spider):
@ -99,11 +104,16 @@ class ImgDownloadPipeline(ImagesPipeline):
def item_completed(self, results, item, info): def item_completed(self, results, item, info):
# return item # return item
# 打包
cbz_path = self.get_file_path(item, result_type="cbz")
if os.path.exists(cbz_path):
self.update_icon(item)
self.pack_icon(item)
else:
# ComicInfoXml 生成 # ComicInfoXml 生成
comic_info = ComicInfoXmlItemExporter(dir=self.get_file_path(item=item, result_type="comic_info")).export_xml(item) comic_info = ComicInfoXmlItemExporter(dir=self.get_file_path(item=item, result_type="comic_info")).export_xml(item)
# 打包
if CBZUtils.packComicChapterCBZ(src_dir= self.get_file_path(item, result_type="images_dir"), if CBZUtils.packComicChapterCBZ(src_dir= self.get_file_path(item, result_type="images_dir"),
dts_path= self.get_file_path(item, result_type="cbz"), dts_path= cbz_path,
comic_info_images= comic_info['Pages'], remove=True): comic_info_images= comic_info['Pages'], remove=True):
self.update_icon(item) self.update_icon(item)
self.pack_icon(item) self.pack_icon(item)

2
Comics/spiders/rm_comic.py
View File

@ -49,7 +49,7 @@ class RmComicSpider(scrapy.Spider):
cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True) cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True)
if os.path.exists(cbz_path): if os.path.exists(cbz_path):
logging.info(f"漫画 {cbz_path} 已存在, 跳过中...") logging.info(f"漫画 {cbz_path} 已存在, 跳过中...")
return yield item
else: else:
yield scrapy.Request(self.main_url+link, meta={'item': item}, callback=self.parse_chapter) yield scrapy.Request(self.main_url+link, meta={'item': item}, callback=self.parse_chapter)