fix
This commit is contained in:
parent
25dfc474ee
commit
3d56218b09
@ -13,6 +13,8 @@ from Comics.items import ComicItem
|
||||
from scrapy.pipelines.images import ImagesPipeline
|
||||
from Comics.exporters import ComicInfoXmlItemExporter,JsonExport
|
||||
from Comics.utils.FileUtils import CBZUtils
|
||||
from Comics.utils.FileUtils import fileUtils as fu
|
||||
|
||||
|
||||
class ComicsPipeline:
|
||||
def open_spider(self, spider):
|
||||
@ -21,11 +23,12 @@ class ComicsPipeline:
|
||||
# item就是yield后面的对象
|
||||
def process_item(self, item, spider):
|
||||
if isinstance(item, ComicItem):
|
||||
# 'output/rm_comic/json/壞X/第1話 壞X'
|
||||
file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter'])
|
||||
item['count'] = len(item['images'])
|
||||
item['index'] = item['chapters'].index(item['chapter']) + 1
|
||||
|
||||
data = JsonExport(file=file).export_json(item, if_return=True)
|
||||
#data[PROJECT_KEY] = spider.name
|
||||
return data
|
||||
# image解析
|
||||
|
||||
@ -42,7 +45,7 @@ class ImgDownloadPipeline(ImagesPipeline):
|
||||
|
||||
## Icon Path : CBZ/NAME/CHAPTER.jpg
|
||||
def download_icon(self, item, result_type="download"):
|
||||
icon_path = self.get_file_path(item, result_type="icon")
|
||||
icon_path = self.get_file_path(item, result_type="icon_cache")
|
||||
if result_type == "fullpath":
|
||||
return os.path.join(settings.IMAGES_STORE, icon_path)
|
||||
if os.path.exists(icon_path):
|
||||
@ -60,20 +63,39 @@ class ImgDownloadPipeline(ImagesPipeline):
|
||||
# 下载封面
|
||||
self.download_icon(item)
|
||||
for image_url,image in zip(self.image_urls,self.images):
|
||||
is_down = True
|
||||
image_path = self.get_file_path(item, image)
|
||||
if self.image_scramble_exits(item, image_path):
|
||||
logging.info(f"file exists: IMAGE_STORE {image_path}")
|
||||
if image_path == self.get_file_path(item, result_type="icon_cache"):
|
||||
logging.info(f"icon file exists: IMAGE_STORE {image_path}")
|
||||
else:
|
||||
is_down = False
|
||||
logging.info(f"file exists: IMAGE_STORE {image_path}")
|
||||
if is_down:
|
||||
logging.info(f"downloading {image_url} --> IMAGE_STORE {image_path}")
|
||||
yield scrapy.Request(url=image_url, meta={'path': image_path})
|
||||
|
||||
# 打包cbz封面
|
||||
def pack_icon(self, item):
|
||||
cbz_icon = self.get_file_path(item=item, result_type="cbz_icon")
|
||||
dwn_icon = self.get_file_path(item=item, result_type="down_icon")
|
||||
logging.info(f"icon packing {dwn_icon} => {cbz_icon}")
|
||||
cbz_icon_dir = os.path.dirname(cbz_icon)
|
||||
if not os.path.exists(cbz_icon_dir): os.makedirs(cbz_icon_dir)
|
||||
shutil.copyfile(dwn_icon, cbz_icon)
|
||||
base_dir = os.path.dirname(dwn_icon)
|
||||
name = os.path.basename(dwn_icon).split(".")[0]
|
||||
for dirname in os.listdir(base_dir):
|
||||
path = os.path.join(base_dir, dirname)
|
||||
if os.path.isfile(path) and dirname.startswith(name):
|
||||
fu.update_icon(path, cbz_icon)
|
||||
|
||||
|
||||
|
||||
# 判断是否需要更新封面
|
||||
def update_icon(self, item):
|
||||
# 下载后的缓存封面路径
|
||||
image_path = self.get_file_path(item, result_type="down_cache_icon")
|
||||
# 最终封面保存路径
|
||||
save_path = self.get_file_path(item=item, result_type="down_icon")
|
||||
|
||||
fu.update_icon(image_path, save_path)
|
||||
|
||||
def item_completed(self, results, item, info):
|
||||
# return item
|
||||
@ -83,6 +105,7 @@ class ImgDownloadPipeline(ImagesPipeline):
|
||||
if CBZUtils.packComicChapterCBZ(src_dir= self.get_file_path(item, result_type="images_dir"),
|
||||
dts_path= self.get_file_path(item, result_type="cbz"),
|
||||
comic_info_images= comic_info['Pages'], remove=True):
|
||||
self.update_icon(item)
|
||||
self.pack_icon(item)
|
||||
|
||||
# sleep_time = random.randint(5,30)
|
||||
|
||||
@ -68,7 +68,7 @@ COOKIES_ENABLED = False
|
||||
# Enable or disable downloader middlewares
|
||||
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||
DOWNLOADER_MIDDLEWARES = {
|
||||
# 'Comics.middlewares.ComicsDownloaderMiddleware': 543,
|
||||
'Comics.middlewares.ComicsDownloaderMiddleware': 543,
|
||||
# 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
|
||||
'Comics.middlewares.ProxyMiddleware': 100,
|
||||
'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 400,
|
||||
|
||||
@ -40,14 +40,9 @@ class RmComicSpider(scrapy.Spider):
|
||||
comic_item.add_value('age_rating', "R18+")
|
||||
chapter_href = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]'
|
||||
'//div[contains(@class,"bookid_chapter")]/a/@href')
|
||||
#chapters = response.xpath('//div[contains(@class,"bookid_chapterBox")]'
|
||||
# '//div[contains(@class,"bookid_chapter")]/a/text()').extract()
|
||||
|
||||
chapters = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]'
|
||||
'//div[contains(@class,"bookid_chapter")]/a/text()')
|
||||
for chapter, link in zip(chapters, chapter_href):
|
||||
#for i, link in enumerate(chapter_href, start=1):
|
||||
# yield scrapy.Request(self.main_url+link, meta={'item': comic_item.load_item(), 'num': i}, callback=self.parse_chapter)
|
||||
comic_item.add_value('chapter', chapter)
|
||||
comic_item.add_value('chapters', chapters)
|
||||
item = comic_item.load_item()
|
||||
|
||||
@ -67,8 +67,12 @@ class ComicPath:
|
||||
file = os.path.join(settings.CBZ_EXPORT_PATH, PROJECT, name, chapter+".jpg")
|
||||
elif result_type == "down_icon":
|
||||
file = os.path.join(settings.IMAGES_STORE, cls.get_file_path(item=item, result_type="icon"))
|
||||
elif result_type == "down_cache_icon":
|
||||
file = os.path.join(settings.IMAGES_STORE, cls.get_file_path(item=item, result_type="icon_cache"))
|
||||
elif result_type == "icon":
|
||||
file = os.path.join(PROJECT, "icons", name+".jpg")
|
||||
file = os.path.join(PROJECT, "icons", name, name+".jpg")
|
||||
elif result_type == "icon_cache":
|
||||
file = os.path.join(PROJECT, "icons", ".cache", name+".jpg")
|
||||
elif result_type == "cbz":
|
||||
file = os.path.join(settings.CBZ_EXPORT_PATH, PROJECT, name, chapter+".CBZ")
|
||||
elif result_type == "images_dir":
|
||||
|
||||
@ -23,6 +23,52 @@ class fileUtils:
|
||||
if not os.path.exists(base_dir): os.makedirs(base_dir)
|
||||
return file
|
||||
|
||||
@classmethod
|
||||
def compare_size(cls, dst, file):
|
||||
if os.path.exists(dst) and os.path.exists(file):
|
||||
return os.stat(dst).st_size == os.stat(file).st_size
|
||||
else:
|
||||
return 0
|
||||
|
||||
"""
|
||||
图像编号 image-1.jpg
|
||||
如：存在image.png 返回 image-1.png 反之 image.png
|
||||
"""
|
||||
@classmethod
|
||||
def file_check(cls, file, result="file"):
|
||||
temp_file_name = file
|
||||
count = 1
|
||||
files_size = []
|
||||
name, suffix = temp_file_name.split(".")
|
||||
while count:
|
||||
if os.path.exists(temp_file_name):
|
||||
files_size.append(os.stat(temp_file_name).st_size)
|
||||
temp_file_name = name+"-"+str(count)+"."+suffix
|
||||
count += 1
|
||||
else:
|
||||
if result == "size":
|
||||
return files_size
|
||||
else:
|
||||
return temp_file_name
|
||||
|
||||
@classmethod
|
||||
def file_update(cls, old_file, new_file):
|
||||
is_update = False
|
||||
if os.path.exists(old_file):
|
||||
is_update = os.stat(old_file).st_size not in cls.file_check(new_file, result="size")
|
||||
return is_update
|
||||
|
||||
# 判断是否需要更新封面
|
||||
@classmethod
|
||||
def update_icon(cls, image_path, save_path):
|
||||
# 不存在则更新
|
||||
if cls.file_update(image_path, save_path):
|
||||
save_dir = os.path.dirname(save_path)
|
||||
if not os.path.exists(save_dir): os.makedirs(save_dir)
|
||||
logging.info(f"update icon ... {image_path} ===> {cls.file_check(save_path)}")
|
||||
shutil.copyfile(image_path, cls.file_check(save_path))
|
||||
|
||||
|
||||
class CommonUtils:
|
||||
@classmethod
|
||||
def parseExec(cls,data,exec):
|
||||
|
||||
Loading…
Reference in New Issue
Block a user