82 lines
3.4 KiB
Python
82 lines
3.4 KiB
Python
# Define your item pipelines here
|
|
#
|
|
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
|
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
|
|
|
|
|
|
# useful for handling different item types with a single interface
|
|
import os, scrapy
|
|
from Comics import settings
|
|
from Comics.utils.FileUtils import imageUtils
|
|
from Comics.utils.FileUtils import fileUtils
|
|
from Comics.utils.Constant import ComicPath
|
|
from Comics.items import ComicItem
|
|
from Comics.items import ImageItem
|
|
from scrapy.pipelines.images import ImagesPipeline
|
|
from Comics.exporters import ComicInfoXmlItemExporter
|
|
from Comics.exporters import ItemExporter
|
|
from Comics.utils.FileUtils import CBZUtils
|
|
|
|
class ComicsPipeline:
|
|
def open_spider(self, spider):
|
|
pass
|
|
# item就是yield后面的对象
|
|
def process_item(self, item, spider):
|
|
if isinstance(item, ComicItem):
|
|
item = ComicItem(ItemExporter().export_obj(item))
|
|
file = os.path.join("json", item['name'], item['chapter'])
|
|
fileUtils.save_file(f"{file}.json", item)
|
|
return item
|
|
# image解析
|
|
|
|
def close_spider(self,spider):
|
|
pass
|
|
|
|
class ImageParsePipeline:
|
|
def process_item(self, item, spider):
|
|
if isinstance(item, ComicItem):
|
|
count = 1
|
|
images_item = []
|
|
for image in item['list_img']:
|
|
(image_src, scramble) = [image.get("src"), image.get("scramble")]
|
|
count_image = "{:0>3d}".format(count)
|
|
suffix = "."+str(image_src).split(".")[-1]
|
|
image_name = count_image + suffix
|
|
if scramble:
|
|
de_str = str(image_src).split("/")[-1].replace(suffix, "==")
|
|
blocks_num = imageUtils.encodeImage(de_str)
|
|
image_name = ComicPath.getFileScrambleImageName(count=count_image, block=blocks_num, suffix=suffix)
|
|
image_path = os.path.join(item['name'], item['chapter'], image_name)
|
|
images_item.append(ImageItem(image_name=count_image + suffix, image_url=image_src, image_path=image_path))
|
|
count += 1
|
|
item['images'] = images_item
|
|
return item
|
|
|
|
class ImgDownloadPipeline(ImagesPipeline):
|
|
def file_path(self, request, response=None, info=None, *, item=None):
|
|
image = request.meta['item']
|
|
image_path = image['image_path']
|
|
en_image_path = os.path.join(os.path.dirname(image_path), image['image_name'])
|
|
if os.path.exists(os.path.join(settings.IMAGES_STORE, en_image_path)):
|
|
return en_image_path
|
|
else:
|
|
return image_path
|
|
|
|
def get_media_requests(self, item, info):
|
|
for image in item['images']:
|
|
yield scrapy.Request(url=image['image_url'], meta={'item': image})
|
|
|
|
def item_completed(self, results, item, info):
|
|
info_img = []
|
|
for success, img in results:
|
|
img_path = os.path.join(settings.IMAGES_STORE, img['path'])
|
|
# 解密图片
|
|
img_path = imageUtils.deScrambleImagesByPath(img_path)
|
|
info_img.append(os.path.basename(img_path).split('.')[0])
|
|
item['images_name'] = ",".join(info_img)
|
|
# return item
|
|
# ComicInfoXml 生成
|
|
ComicInfoXmlItemExporter(comic=item['name'], chapter=item['chapter']).export_xml(item)
|
|
# 打包
|
|
CBZUtils.packComicChapterCBZ(comic=item['name'], chapter=item['chapter'], remove=False)
|