update
This commit is contained in:
parent
e6eadefe34
commit
7672b7c84b
@ -52,9 +52,9 @@ class JsonExport(JsonItemExporter):
|
||||
|
||||
class ComicInfoXmlItemExporter(XmlItemExporter):
|
||||
custom_root_element = "ComicInfo"
|
||||
def __init__(self, comic, chapter):
|
||||
file_path = os.path.join(COMIC_INFO_XML_STORE, comic,
|
||||
chapter, f"{self.custom_root_element}.xml")
|
||||
def __init__(self, dir):
|
||||
file_path = os.path.join(COMIC_INFO_XML_STORE, dir,
|
||||
f"{self.custom_root_element}.xml")
|
||||
dir_path = os.path.dirname(file_path)
|
||||
if not os.path.exists(dir_path): os.makedirs(dir_path)
|
||||
self.xml_file = open(file_path, "wb")
|
||||
|
||||
@ -6,7 +6,11 @@ import os,Comics.settings as settings,logging
|
||||
from scrapy.item import Item, Field
|
||||
from Comics.utils.Constant import ComicPath
|
||||
from Comics.utils.FileUtils import imageUtils
|
||||
from scrapy.loader.processors import TakeFirst, MapCompose, Join
|
||||
from itemloaders.processors import TakeFirst, MapCompose, Join
|
||||
from scrapy.spiders import Spider
|
||||
|
||||
def current_project():
|
||||
return Spider.name
|
||||
|
||||
def serialize_to_chinese(value):
|
||||
return ComicPath.chinese_convert(value)
|
||||
@ -35,7 +39,7 @@ def _serialize_to_images(value, result_type=None):
|
||||
images_item.append(image_src)
|
||||
image_urls.append(image_src)
|
||||
count += 1
|
||||
logging.info(f"images_len: {len(images_item)}")
|
||||
logging.debug(f"images_len: {len(images_item)}")
|
||||
if result_type == "image_urls": return image_urls
|
||||
else: return images_item
|
||||
|
||||
@ -51,6 +55,8 @@ class ListComicItem(Item):
|
||||
|
||||
|
||||
class ComicItem(Item):
|
||||
# 工程
|
||||
current_project = Field()
|
||||
# 编号
|
||||
index = Field(output_processor=TakeFirst())
|
||||
# 漫画名
|
||||
@ -113,6 +119,7 @@ def _serializer_info_imagesa(value, result_type=None):
|
||||
def _serialize_info_images(value, result_type=None):
|
||||
images = []
|
||||
for image in value:
|
||||
if os.sep not in image:
|
||||
images.append(ComicPath().getFileScrambleImageSave(image,True,False))
|
||||
if result_type == "count":
|
||||
return len(images)
|
||||
|
||||
@ -5,8 +5,9 @@
|
||||
|
||||
|
||||
# useful for handling different item types with a single interface
|
||||
import os, scrapy,logging,time,random
|
||||
import os, scrapy,logging,time,random,shutil
|
||||
from Comics import settings
|
||||
from Comics.settings import CBZ_EXPORT_PATH,OUTPUT_DIR,PROJECT_KEY
|
||||
from Comics.utils.Constant import ComicPath
|
||||
from Comics.items import ComicItem
|
||||
from scrapy.pipelines.images import ImagesPipeline
|
||||
@ -19,43 +20,81 @@ class ComicsPipeline:
|
||||
# item就是yield后面的对象
|
||||
def process_item(self, item, spider):
|
||||
if isinstance(item, ComicItem):
|
||||
file = os.path.join(settings.OUTPUT_DIR,"json", item['name'], item['chapter'])
|
||||
file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter'])
|
||||
item['count'] = len(item['images'])
|
||||
item['images'].append({'src':'icon.jpg', 'scramble': False})
|
||||
item['image_urls'].append({'src': item['icon'] , 'scramble': False})
|
||||
data = JsonExport(file=file).export_json(item, if_return=True)
|
||||
#item['images'] = data['images']
|
||||
data[PROJECT_KEY] = spider.name
|
||||
return data
|
||||
# image解析
|
||||
|
||||
def close_spider(self,spider):
|
||||
pass
|
||||
|
||||
|
||||
class ImgDownloadPipeline(ImagesPipeline):
|
||||
def file_exits(self, image_path):
|
||||
en_image_path = ComicPath().getFileScrambleImageSave(image_path, relative="fullpath")
|
||||
return os.path.exists(os.path.join(settings.IMAGES_STORE, en_image_path))
|
||||
def get_file_path(self, item, file=None, result_type="image"):
|
||||
if result_type == "image":
|
||||
if os.path.sep not in file:
|
||||
file = os.path.join(item[settings.PROJECT_KEY], "images", item['name'], item['chapter'], file)
|
||||
elif result_type == "comic_info":
|
||||
file = os.path.join(item[settings.PROJECT_KEY], "images", item['name'], item['chapter'])
|
||||
elif result_type == "cbz_icon":
|
||||
file = os.path.join("CBZ", item[settings.PROJECT_KEY], item['name'], item['chapter']+".jpg")
|
||||
elif result_type == "down_icon":
|
||||
file = self.download_icon(item,result_type='fullpath')
|
||||
elif result_type == "icon":
|
||||
file = os.path.join(item[settings.PROJECT_KEY], "icons", item['name'], item['chapter']+".jpg")
|
||||
elif result_type == "cbz":
|
||||
file = os.path.join("CBZ", item[settings.PROJECT_KEY], item['name'], item['chapter']+".CBZ")
|
||||
elif result_type == "images_dir":
|
||||
file = os.path.join(settings.IMAGES_STORE, item[settings.PROJECT_KEY], "images", item['name'], item['chapter'])
|
||||
return file
|
||||
|
||||
def file_full_path(self, item, image): return os.path.join(item['name'], item['chapter'], image)
|
||||
def image_scramble_exits(self, item,image_path):
|
||||
en_image_path = ComicPath().getFileScrambleImageSave(image_path, relative="fullpath")
|
||||
return os.path.exists(os.path.join(settings.IMAGES_STORE, self.get_file_path(item, en_image_path)))
|
||||
|
||||
## Icon Path : CBZ/NAME/CHAPTER.jpg
|
||||
def download_icon(self, item, result_type="download"):
|
||||
icon_path = self.get_file_path(item, result_type="icon")
|
||||
if result_type == "fullpath":
|
||||
return os.path.join(settings.IMAGES_STORE, icon_path)
|
||||
if os.path.exists(icon_path):
|
||||
return False
|
||||
else:
|
||||
self.image_urls.append(item['icon'])
|
||||
self.images.append(icon_path)
|
||||
return True
|
||||
|
||||
def file_path(self, request, response=None, info=None, *, item=None): return request.meta['path']
|
||||
|
||||
def get_media_requests(self, item, info):
|
||||
for image_url,image_path in zip(item['image_urls'],item['images']):
|
||||
image_path = self.file_full_path(item, image_path)
|
||||
if self.file_exits(image_path):
|
||||
logging.info(f"file exists: {image_path}")
|
||||
self.image_urls = item['image_urls']
|
||||
self.images = item['images']
|
||||
# 下载封面
|
||||
self.download_icon(item)
|
||||
for image_url,image in zip(self.image_urls,self.images):
|
||||
image_path = self.get_file_path(item, image)
|
||||
if self.image_scramble_exits(item, image_path):
|
||||
logging.info(f"file exists: IMAGE_STORE {image_path}")
|
||||
else:
|
||||
logging.info(f"downloading {image_url} --> {image_path}")
|
||||
logging.info(f"downloading {image_url} --> IMAGE_STORE {image_path}")
|
||||
yield scrapy.Request(url=image_url, meta={'path': image_path})
|
||||
|
||||
def pack_icon(self, item):
|
||||
cbz_icon = self.get_file_path(item=item, result_type="cbz_icon")
|
||||
dwn_icon = self.get_file_path(item=item, result_type="down_icon")
|
||||
logging.info(f"icon packing {dwn_icon} => {cbz_icon}")
|
||||
cbz_icon_dir = os.path.dirname(cbz_icon)
|
||||
if not os.path.exists(cbz_icon_dir): os.makedirs(cbz_icon_dir)
|
||||
shutil.copyfile(dwn_icon, cbz_icon)
|
||||
|
||||
def item_completed(self, results, item, info):
|
||||
item['images_name'] = results
|
||||
# return item
|
||||
# ComicInfoXml 生成
|
||||
comic_info = ComicInfoXmlItemExporter(comic=item['name'], chapter=item['chapter']).export_xml(item)
|
||||
comic_info = ComicInfoXmlItemExporter(dir=self.get_file_path(item=item, result_type="comic_info")).export_xml(item)
|
||||
# 打包
|
||||
CBZUtils.packComicChapterCBZ(comic=item['name'], chapter=item['chapter'],
|
||||
comic_info_images= comic_info["Pages"], remove=False)
|
||||
if CBZUtils.packComicChapterCBZ(src_dir= self.get_file_path(item, result_type="images_dir"),
|
||||
dts_path= self.get_file_path(item, result_type="cbz"),
|
||||
comic_info_images= comic_info['Pages'], remove=False):
|
||||
self.pack_icon(item)
|
||||
time.sleep(random.randint(5,10))
|
||||
@ -9,11 +9,11 @@
|
||||
from fake_useragent import UserAgent
|
||||
import os
|
||||
|
||||
PROJECT_KEY = "current_project"
|
||||
BOT_NAME = 'Comics'
|
||||
|
||||
SPIDER_MODULES = ['Comics.spiders']
|
||||
NEWSPIDER_MODULE = 'Comics.spiders'
|
||||
|
||||
OUTPUT_DIR = "output"
|
||||
# Crawl responsibly by identifying yourself (and your website) on the user-agent
|
||||
#USER_AGENT = 'Comics (+http://www.yourdomain.com)'
|
||||
@ -28,7 +28,7 @@ CONCURRENT_REQUESTS = 16
|
||||
# Configure a delay for requests for the same website (default: 0)
|
||||
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
|
||||
# See also autothrottle settings and docs
|
||||
IMAGES_STORE = os.path.join(OUTPUT_DIR, 'images')
|
||||
IMAGES_STORE = "output"
|
||||
IMAGES_NAME_FORMAT = "{:0>3d}"
|
||||
COMIC_INFO_XML_STORE = IMAGES_STORE
|
||||
DOWNLOAD_DELAY = 0
|
||||
@ -36,7 +36,7 @@ DOWNLOAD_DELAY = 0
|
||||
RETRY_ENABLED = True
|
||||
RETRY_TIMES = 10 # 想重试几次就写几
|
||||
# 下面这行可要可不要
|
||||
RETRY_HTTP_CODES = [500, 502, 503, 504, 408, 401]
|
||||
# RETRY_HTTP_CODES = [500, 502, 503, 504, 408, 401]
|
||||
# The download delay setting will honor only one of:
|
||||
CONCURRENT_REQUESTS_PER_DOMAIN = 16
|
||||
CONCURRENT_REQUESTS_PER_IP = 16
|
||||
@ -105,7 +105,6 @@ AUTOTHROTTLE_DEBUG = False
|
||||
HTTPCACHE_ENABLED = True
|
||||
HTTPCACHE_EXPIRATION_SECS = 0
|
||||
HTTPCACHE_DIR = 'httpcache'
|
||||
HTTPCACHE_IGNORE_HTTP_CODES = [500, 502, 404]
|
||||
HTTPCACHE_ALLOW_PREFIXS = [ 'jpg', 'png', 'gif']
|
||||
HTTPCACHE_STORAGE = 'Comics.middlewares.MyFilesystemCacheStorage'
|
||||
|
||||
|
||||
@ -19,7 +19,6 @@ class RmComicSpider(scrapy.Spider):
|
||||
books = books_comic.get_exec(data, str_exec=str_exec)
|
||||
for book in books:
|
||||
books_comic.add_value('link', book['id'])
|
||||
logging.info(f"downloading books %s" % book['name'])
|
||||
time.sleep(3)
|
||||
yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic)
|
||||
|
||||
|
||||
@ -275,30 +275,27 @@ class CBZUtils:
|
||||
y = 0
|
||||
for filename in filenames:
|
||||
y = y + 1
|
||||
logging.info(f"打包中：" + str(y) + "/" + str(len(filenames)), os.path.join(source_dir, filename))
|
||||
print("打包中：" + str(y) + "/" + str(len(filenames)), os.path.join(source_dir, filename))
|
||||
zf.write(path.joinpath(filename), arc_dir.joinpath(filename))
|
||||
zf.close()
|
||||
logging.info(f"打包完成：{target_file}")
|
||||
|
||||
@classmethod
|
||||
def packComicChapterCBZ(cls, comic, chapter, comic_info_images, remove=True):
|
||||
images_chapter_path = os.path.join(IMAGES_STORE, comic, chapter)
|
||||
cbz_chapter_path = os.path.join(CBZ_EXPORT_PATH, comic, chapter) + ".CBZ"
|
||||
if os.path.exists(images_chapter_path):
|
||||
dirs = os.listdir(images_chapter_path)
|
||||
def packComicChapterCBZ(cls, src_dir, dts_path, comic_info_images, remove=True):
|
||||
if os.path.exists(src_dir):
|
||||
dirs = os.listdir(src_dir)
|
||||
for file in dirs:
|
||||
if file.startswith(ComicPath.PREFIX_SCRAMBLE):
|
||||
try:
|
||||
imageUtils.deScrambleImagesByPath(os.path.join(images_chapter_path,file))
|
||||
imageUtils.deScrambleImagesByPath(os.path.join(src_dir,file))
|
||||
except Exception as e:
|
||||
print(f"删除 {file} 发生错误 {e}，已跳过")
|
||||
return False
|
||||
cls.zip_compression(images_chapter_path, cbz_chapter_path)
|
||||
cls.zip_compression(src_dir, dts_path)
|
||||
time.sleep(0.1)
|
||||
if remove: shutil.rmtree(images_chapter_path)
|
||||
if remove: shutil.rmtree(src_dir)
|
||||
# validation
|
||||
cls.cbz_validate(cbz_chapter_path, comic_info_images)
|
||||
return True
|
||||
return cls.cbz_validate(dts_path, comic_info_images)
|
||||
|
||||
@classmethod
|
||||
def replaceZip(cls, filepath, unpack_dir=None):
|
||||
@ -356,6 +353,8 @@ class CBZUtils:
|
||||
def cbz_validate(cls, zip_path, comic_info_images):
|
||||
if len(cls.zip_info(zip_path)) == len(comic_info_images):
|
||||
logging.info(f"validating successfully === {zip_path}")
|
||||
return True
|
||||
else:
|
||||
os.remove(zip_path)
|
||||
logging.error(f"validating fail === {zip_path}")
|
||||
return False
|
||||
Loading…
Reference in New Issue
Block a user