123 lines
5.3 KiB
Python
123 lines
5.3 KiB
Python
# Define your item pipelines here
|
|
#
|
|
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
|
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
|
|
|
|
|
|
# useful for handling different item types with a single interface
|
|
import os, scrapy,logging,time,random,shutil
|
|
from Comics import settings
|
|
from Comics.settings import CBZ_EXPORT_PATH,OUTPUT_DIR,PROJECT_KEY
|
|
from Comics.utils.Constant import ComicPath
|
|
from Comics.items import ComicItem
|
|
from scrapy.pipelines.images import ImagesPipeline
|
|
from Comics.exporters import ComicInfoXmlItemExporter,JsonExport,ItemExporter
|
|
from Comics.utils.FileUtils import CBZUtils
|
|
from Comics.utils.FileUtils import fileUtils as fu
|
|
|
|
|
|
class ComicsPipeline:
|
|
def open_spider(self, spider):
|
|
pass
|
|
|
|
# item就是yield后面的对象
|
|
def process_item(self, item, spider):
|
|
if isinstance(item, ComicItem):
|
|
# 'output/rm_comic/json/壞X/第1話 壞X'
|
|
cbz_path = ComicPath.get_file_path(item, result_type="cbz", convert=True)
|
|
if os.path.exists(cbz_path):
|
|
item['image_urls'] = []
|
|
item['images'] = []
|
|
return ItemExporter().export_obj(item)
|
|
else:
|
|
file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter'])
|
|
item['count'] = len(item['images'])
|
|
item['index'] = item['chapters'].index(item['chapter']) + 1
|
|
|
|
return JsonExport(file=file).export_json(item, if_return=True)
|
|
# image解析
|
|
|
|
def close_spider(self, spider):
|
|
pass
|
|
|
|
class ImgDownloadPipeline(ImagesPipeline):
|
|
def get_file_path(self, item, file=None, result_type="image"):
|
|
return ComicPath.get_file_path(item=item, file=file, result_type= result_type)
|
|
|
|
def image_scramble_exits(self, item,image_path):
|
|
en_image_path = ComicPath().getFileScrambleImageSave(image_path, relative="fullpath")
|
|
return os.path.exists(os.path.join(settings.IMAGES_STORE, self.get_file_path(item, en_image_path)))
|
|
|
|
## Icon Path : CBZ/NAME/CHAPTER.jpg
|
|
def download_icon(self, item, result_type="download"):
|
|
icon_path = self.get_file_path(item, result_type="icon_cache")
|
|
if result_type == "fullpath":
|
|
return os.path.join(settings.IMAGES_STORE, icon_path)
|
|
if os.path.exists(icon_path):
|
|
return False
|
|
else:
|
|
self.image_urls.append(item['icon'])
|
|
self.images.append(icon_path)
|
|
return True
|
|
|
|
def file_path(self, request, response=None, info=None, *, item=None): return request.meta['path']
|
|
|
|
def get_media_requests(self, item, info):
|
|
self.image_urls = item['image_urls']
|
|
self.images = item['images']
|
|
# 下载封面
|
|
self.download_icon(item)
|
|
for image_url,image in zip(self.image_urls,self.images):
|
|
is_down = True
|
|
image_path = self.get_file_path(item, image)
|
|
if self.image_scramble_exits(item, image_path):
|
|
if image_path == self.get_file_path(item, result_type="icon_cache"):
|
|
logging.info(f"icon file exists: IMAGE_STORE {image_path}")
|
|
else:
|
|
is_down = False
|
|
logging.info(f"file exists: IMAGE_STORE {image_path}")
|
|
if is_down:
|
|
logging.info(f"downloading {image_url} --> IMAGE_STORE {image_path}")
|
|
yield scrapy.Request(url=image_url, meta={'path': image_path})
|
|
|
|
# 打包cbz封面
|
|
def pack_icon(self, item):
|
|
cbz_icon = self.get_file_path(item=item, result_type="cbz_icon")
|
|
dwn_icon = self.get_file_path(item=item, result_type="down_icon")
|
|
base_dir = os.path.dirname(dwn_icon)
|
|
name = os.path.basename(dwn_icon).split(".")[0]
|
|
for dirname in os.listdir(base_dir):
|
|
path = os.path.join(base_dir, dirname)
|
|
if os.path.isfile(path) and dirname.startswith(name):
|
|
fu.update_icon(path, cbz_icon)
|
|
|
|
|
|
|
|
# 判断是否需要更新封面
|
|
def update_icon(self, item):
|
|
# 下载后的缓存封面路径
|
|
image_path = self.get_file_path(item, result_type="down_cache_icon")
|
|
# 最终封面保存路径
|
|
save_path = self.get_file_path(item=item, result_type="down_icon")
|
|
|
|
fu.update_icon(image_path, save_path)
|
|
|
|
def item_completed(self, results, item, info):
|
|
# return item
|
|
# 打包
|
|
cbz_path = self.get_file_path(item, result_type="cbz")
|
|
if os.path.exists(cbz_path):
|
|
self.update_icon(item)
|
|
self.pack_icon(item)
|
|
else:
|
|
# ComicInfoXml 生成
|
|
comic_info = ComicInfoXmlItemExporter(dir=self.get_file_path(item=item, result_type="comic_info")).export_xml(item)
|
|
if CBZUtils.packComicChapterCBZ(src_dir= self.get_file_path(item, result_type="images_dir"),
|
|
dts_path= cbz_path,
|
|
comic_info_images= comic_info['Pages'], remove=True):
|
|
self.update_icon(item)
|
|
self.pack_icon(item)
|
|
|
|
sleep_time = random.randint(3,15)
|
|
print(f'等待{sleep_time}秒后进行下一章节')
|
|
time.sleep(int(sleep_time)) |