76 lines
3.1 KiB
Python
76 lines
3.1 KiB
Python
# Define your item pipelines here
|
|
#
|
|
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
|
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
|
|
|
|
|
|
# useful for handling different item types with a single interface
|
|
import os,requests,re,scrapy,logging
|
|
from Comics import settings
|
|
from Comics.utils.FileUtils import imageUtils
|
|
from Comics.utils.Constant import ComicPath
|
|
from Comics.items import ComicItem
|
|
from Comics.items import ImageItem
|
|
from scrapy.pipelines.images import ImagesPipeline
|
|
from scrapy.exporters import XmlItemExporter
|
|
|
|
class ComicsPipeline:
|
|
def open_spider(self,spider):
|
|
self.fp = open('book.json','w',encoding='utf-8')
|
|
|
|
# item就是yield后面的对象
|
|
def process_item(self, item, spider):
|
|
self.fp.write(str(item))
|
|
return item
|
|
#image解析
|
|
|
|
def close_spider(self,spider):
|
|
self.fp.close()
|
|
|
|
class ImageParsePipeline:
|
|
def process_item(self, item, spider):
|
|
if isinstance(item, ComicItem):
|
|
list_img = item['list_img']
|
|
count = 1
|
|
scramble_count = 0
|
|
list_image_item = []
|
|
for image in list_img:
|
|
(image_src,scramble) = [image.get("src"),image.get("scramble")]
|
|
count_image = "{:0>3d}".format(count)
|
|
image_src_suffix = "."+str(image_src).split(".")[-1]
|
|
image_file_name = count_image+image_src_suffix
|
|
if scramble:
|
|
de_str = str(image_src).split("/")[-1].replace(image_src_suffix,"==")
|
|
blocks_num = imageUtils.encodeImage(de_str)
|
|
scramble_image_file_name = ComicPath.getFileScrambleImageName(count=count_image,block=blocks_num,suffix=image_src_suffix)
|
|
scramble_count += 1
|
|
image_path = os.path.join(item['name'],item['chapter'],scramble_image_file_name)
|
|
image_path = ComicPath.ChineseConvert(image_path)
|
|
list_image_item.append(ImageItem(image_name=image_file_name,image_url=image_src,image_path=image_path))
|
|
count+=1
|
|
return list_image_item
|
|
|
|
class ImgDownloadPipeline(ImagesPipeline):
|
|
def file_path(self, request, response=None, info=None, *, item=None):
|
|
image = request.meta['item']
|
|
image_path = image['image_path']
|
|
en_image_path = os.path.join(os.path.dirname(image_path),image['image_name'])
|
|
if os.path.exists(en_image_path): return en_image_path
|
|
else: return image_path
|
|
|
|
def get_media_requests(self, item, info):
|
|
for image in item:
|
|
host = re.sub(r'(http://|https://)', '', image['image_url']).split('/')[0]
|
|
yield scrapy.Request(url= image['image_url'], meta= {'item' : image})
|
|
|
|
def item_completed(self, results, item, info):
|
|
if len(results) == len(item):
|
|
for image in results:
|
|
success = image[0]
|
|
img = image[1]
|
|
img_path = os.path.join(settings.IMAGES_STORE,img['path'])
|
|
#解密图片
|
|
imageUtils.deScrambleImagesByPath(img_path)
|
|
XmlItemExporter()
|
|
return item
|