195 lines
8.9 KiB
Python
195 lines
8.9 KiB
Python
# Define your item pipelines here
|
||
#
|
||
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
||
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
|
||
|
||
|
||
# useful for handling different item types with a single interface
|
||
import os,scrapy,logging,shutil
|
||
from Comics import settings
|
||
from Comics.settings import IMAGES_STORE
|
||
from Comics._utils.items import ComicItem
|
||
from Comics._utils.loader import ComicLoader
|
||
from Comics._utils.utils import CBZUtils,fileUtils as fu
|
||
from Comics._utils.utils import ComicPath
|
||
from Comics._utils.utils import oldUtils
|
||
from Comics._utils.exporters import JsonExport,ItemExporter
|
||
from scrapy.pipelines.images import ImagesPipeline
|
||
from Comics._utils.ComicInfo import ComicInfoXml
|
||
from Comics._utils.downloader import download_images
|
||
|
||
class ComicsPipeline():
|
||
# 500kb
|
||
def remove_min_file(self, file_path, min_size=500*1024):
|
||
try:
|
||
# 使用 os.path.getsize() 获取文件大小（以字节为单位）
|
||
file_size = os.path.getsize(file_path)
|
||
if file_size < min_size:
|
||
os.remove(file_path)
|
||
logging.info(f"清除错误文件： {file_path}")
|
||
except Exception as e:
|
||
print(f"获取文件大小失败: {e}")
|
||
|
||
'''
|
||
解析前端传入的item数据
|
||
将数据进行序列化后传出
|
||
'''
|
||
# item就是yield后面的对象
|
||
def process_item(self, item: ComicItem, spider):
|
||
if isinstance(item, ComicItem):
|
||
# 已存在漫画CBZ文件 调用转换
|
||
result_item = None
|
||
if fu.exists(ComicPath(item).PATH_CBZ()): result_item = ItemExporter().export_obj(item)
|
||
# 不存在漫画CBZ文件
|
||
else: result_item = JsonExport(file=ComicPath(item).getDirJosnComicChapter()).export_json(ComicLoader(item).load_item(), if_return=True)
|
||
cbz_path = ComicPath(item=item).PATH_CBZ()
|
||
if not fu.exists(cbz_path):
|
||
return result_item
|
||
else: self.remove_min_file(cbz_path)
|
||
return None
|
||
|
||
class BaseImagesPipeline(ImagesPipeline):
|
||
|
||
def image_scramble_exits(self, item,image_path):
|
||
en_image_path = ComicPath(item).getFileScrambleImageSave(image_path, relative="fullpath")
|
||
return fu.exists(fu.join(settings.IMAGES_STORE, self.get_file_path(item, en_image_path)))
|
||
|
||
def get_file_path(self, item, file=None, result_type="image"):
|
||
return ComicPath(item).file_path(file=file, result_type= result_type)
|
||
|
||
# 封面路径
|
||
def file_path(self, request, response=None, info=None, *, item=None): return request.meta['path']
|
||
|
||
# 判断是否需要更新封面
|
||
def update_icon(self, item):
|
||
# 下载后的缓存封面路径
|
||
image_path = self.get_file_path(item, result_type="down_cache_icon")
|
||
# 最终封面保存路径
|
||
save_path = self.get_file_path(item=item, result_type="down_icon")
|
||
|
||
fu.update_icon(image_path, save_path)
|
||
|
||
def success_completed(self, item, results):
|
||
is_success = False
|
||
fail_data = []
|
||
for result in results:
|
||
# 失败数据
|
||
if not result[0]: fail_data.append(result[1])
|
||
if len(fail_data) == 0 and len(results) != 0: is_success = True
|
||
return is_success
|
||
|
||
class ImgDownloadPipeline(BaseImagesPipeline):
|
||
|
||
def get_media_requests(self, item, info):
|
||
comic = ComicLoader(item=item)
|
||
# 获取需要解析下载的图像
|
||
images_item = comic.parse_images()
|
||
donwloaded_images = []
|
||
for image_item in images_item:
|
||
image_url, image_path = [ image_item["image_url"], image_item["image_path"]]
|
||
if image_item["image_type"] == "Icon":
|
||
image_path = super().get_file_path(item, result_type="icon_cache")
|
||
# 图像（含加密图像）存在
|
||
# if super().image_scramble_exits(item, image_path):
|
||
if os.path.exists(image_path):
|
||
logging.info(f"file exists: IMAGE_STORE {image_path}")
|
||
donwloaded_images.append(image_path)
|
||
logging.info(f"images count= {len(images_item)} downloaded_images_count= {len(donwloaded_images)}")
|
||
else:
|
||
logging.info(f"downloading {image_url} --> IMAGE_STORE {image_path}")
|
||
yield scrapy.Request(url=image_url, meta={'path': image_path })
|
||
|
||
# 打包cbz封面
|
||
def pack_icon(self, item):
|
||
cbz_icon = super().get_file_path(item=item, result_type="cbz_icon")
|
||
dwn_icon = super().get_file_path(item=item, result_type="down_icon")
|
||
base_dir = fu.dirname(dwn_icon)
|
||
name = fu.basename(dwn_icon).split(".")[0]
|
||
for dirname in os.listdir(base_dir):
|
||
path = fu.join(base_dir, dirname)
|
||
if os.path.isfile(path) and dirname.startswith(name):
|
||
fu.update_icon(path, cbz_icon)
|
||
|
||
def download_validate(self, item):
|
||
"""图片校验完整则返回TRUE
|
||
|
||
Args:
|
||
item (_type_): _description_
|
||
|
||
Returns:
|
||
_type_: _description_
|
||
"""
|
||
comic = ComicLoader(item=item)
|
||
# 获取需要解析下载的图像
|
||
images_item = comic.parse_images()
|
||
exist_images = []
|
||
for image_item in images_item:
|
||
image_path = os.path.join(IMAGES_STORE, image_item["image_path"])
|
||
if image_item["image_type"] == "Image" and os.path.exists(image_path): exist_images.append(image_path)
|
||
if len(comic.get_image_urls()) == len(exist_images) and len(exist_images) != 0 : return True
|
||
return False
|
||
|
||
def download_done(self, item):
|
||
# super().update_icon(item)
|
||
cbz_path = super().get_file_path(item, result_type="cbz")
|
||
chapter_dir = ComicPath(item=item).file_path(result_type=ComicPath().MAPPING_IMAGES_DIR)
|
||
# images_file = oldUtils().old_images(folder=chapter_dir)
|
||
# images_urls = ComicLoader(item=item).get_image_urls()
|
||
# 校验数据是正确
|
||
# if len(images_file) != len(images_urls) or len(images_urls) == 0: return
|
||
super().update_icon(item)
|
||
# CBZ文件是否已存在
|
||
if fu.exists(cbz_path):
|
||
logging.info(f"file exists {cbz_path}")
|
||
#self.update_icon(item)
|
||
chapter = os.path.basename(cbz_path).split(".")[0]
|
||
if os.path.exists(chapter_dir) and chapter in chapter_dir:
|
||
print(f"正在删除多余章节文件夹 {chapter_dir}")
|
||
shutil.rmtree(chapter_dir)
|
||
self.pack_icon(item)
|
||
else:
|
||
# ComicInfoXml 生成
|
||
ComicInfoXml().scrapy_xml_by_json(item, save_dir= chapter_dir)
|
||
if CBZUtils.comicChapterCBZPack(src_dir= chapter_dir, dts_path= cbz_path, remove=True):
|
||
super().update_icon(item)
|
||
self.pack_icon(item)
|
||
|
||
def down_image(self, item):
|
||
"""图片下载完成，开始CBZ打包
|
||
不再做数据完整检验，CBZUtils后续会自动检验
|
||
传入图像路径即可
|
||
Args:
|
||
results (_type_): 当前下载完成的图片，已下载的已忽略
|
||
item (_type_): Comic item数据
|
||
info (_type_): 信息
|
||
"""
|
||
comic = ComicLoader(item=item)
|
||
# 获取需要解析下载的图像
|
||
images_item = comic.parse_images()
|
||
donwloaded_images = []
|
||
down_queue = []
|
||
for image_item in images_item:
|
||
image_url, image_path = [ image_item["image_url"], image_item["image_path"]]
|
||
if image_item["image_type"] == "Image":
|
||
is_next = not super().image_scramble_exits(item, image_path)
|
||
# 图像（含加密图像）存在
|
||
if not is_next:
|
||
logging.info(f"file exists: IMAGE_STORE {image_path}")
|
||
donwloaded_images.append(image_path)
|
||
logging.info(f"images count= {len(images_item)} downloaded_images_count= {len(donwloaded_images)}")
|
||
# 如果图像已全部下载则直接跳至下一步（压缩CBZ）
|
||
# if len(donwloaded_images) == len(images_item):
|
||
# logging.info(f"len(donwloaded_images) == len(images_item)")
|
||
# self.download_done(item)
|
||
if is_next:
|
||
# logging.info(f"downloading {image_url} --> IMAGE_STORE {image_path}")
|
||
down_queue.append((image_url, os.path.join(IMAGES_STORE, image_path)))
|
||
if len(down_queue) > 0:
|
||
download_images(down_queue, max_retries=3)
|
||
|
||
def item_completed(self, results, item, info):
|
||
cbz_path = super().get_file_path(item, result_type="cbz")
|
||
if not fu.exists(cbz_path):
|
||
self.down_image(item)
|
||
# 存在未下载图像数据则重试
|
||
self.download_done(item) |