45 lines
1.7 KiB
Python
45 lines
1.7 KiB
Python
# Define your item pipelines here
|
|
#
|
|
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
|
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
|
|
|
|
|
|
# useful for handling different item types with a single interface
|
|
from itemadapter import ItemAdapter
|
|
from scrapy.pipelines.images import ImagesPipeline
|
|
|
|
class ComicsPipeline:
|
|
def open_spider(self,spider):
|
|
self.fp = open('book.json','w',encoding='utf-8')
|
|
|
|
def comic_process(self,images):
|
|
count = 1
|
|
scramble_count = 0
|
|
(files_name,images_url) = [[],[]]
|
|
for image in images:
|
|
(image_src,scramble) = [image.get("src"),image.get("scramble")]
|
|
count_image = "{:0>3d}".format(count)
|
|
image_src_suffix = "."+str(image_src).split(".")[-1]
|
|
image_file_name = count_image+image_src_suffix
|
|
if scramble:
|
|
de_str = str(image_src).split("/")[-1].replace(image_src_suffix,"==")
|
|
#blocks_num = imageUtils.encodeImage(de_str)
|
|
#image_file_name = ComicPath.getFileScrambleImageName(count=count_image,block=blocks_num,suffix=image_src_suffix)
|
|
scramble_count += 1
|
|
files_name.append(image_file_name)
|
|
images_url.append(image_src)
|
|
#downUtils.putDownImageUrlDirFile(image_src,ComicPath.getDirComicChapter(),image_file_name)
|
|
count+=1
|
|
|
|
# item就是yield后面的对象
|
|
def process_item(self, item, spider):
|
|
self.fp.write(str(item))
|
|
self.comic_process(item["list_img"])
|
|
return item
|
|
|
|
def close_spider(self,spider):
|
|
self.fp.close()
|
|
|
|
class ImgDownloadPipeline(ImagesPipeline):
|
|
def get_
|
|
|