update
This commit is contained in:
parent
e8b153dfbf
commit
e6eadefe34
@ -29,7 +29,10 @@ def _serialize_to_images(value, result_type=None):
|
||||
blocks_num = imageUtils.encodeImage(de_str)
|
||||
image_name = ComicPath.getFileScrambleImageName(count=count_image, block=blocks_num, suffix=suffix)
|
||||
#images_item.append(ImagesItem(image_name=count_image + suffix, image_url=image_src, image_path=image_name))
|
||||
images_item.append(image_name)
|
||||
if str(image_src).startswith('http'):
|
||||
images_item.append(image_name)
|
||||
else:
|
||||
images_item.append(image_src)
|
||||
image_urls.append(image_src)
|
||||
count += 1
|
||||
logging.info(f"images_len: {len(images_item)}")
|
||||
@ -70,7 +73,8 @@ class ComicItem(Item):
|
||||
genre = Field(output_processor=TakeFirst())
|
||||
# 年龄分级
|
||||
age_rating = Field(output_processor=TakeFirst())
|
||||
|
||||
|
||||
count = Field()
|
||||
images_old = Field(serializer=serialize_to_images)
|
||||
images = Field(serializer=serialize_to_images)
|
||||
image_urls = Field(serializer=serialize_to_image_urls)
|
||||
@ -141,8 +145,8 @@ class ComicInfoItem(Item):
|
||||
Genre = Field(info='genre')# ","流派",True]
|
||||
Tags = Field(info='tags')# ","标签",True]
|
||||
Web = Field()# ","主页",False]
|
||||
#PageCount = Field()# ","总页数",True]
|
||||
PageCount = Field(info='images',serializer=serializer_info_images_count)# ","总页数",True]
|
||||
PageCount = Field(info='count')# ","总页数",True]
|
||||
#PageCount = Field(info='count',serializer=serializer_info_images_count)# ","总页数",True]
|
||||
LanguageISO = Field()#","语言",True]
|
||||
AgeRating = Field(info='age_rating')#","年龄分级",False]
|
||||
#Pages = Field(info='images_name', serializer=serializer_info_images_completed)#","页码",True]
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import json
|
||||
import json,logging
|
||||
from scrapy.loader import ItemLoader
|
||||
|
||||
class ComicLoader(ItemLoader):
|
||||
@ -33,7 +33,11 @@ class ComicLoader(ItemLoader):
|
||||
if exec is not None:
|
||||
values = self.parseExec(values, exec)
|
||||
if index is not None:
|
||||
values = values[index]
|
||||
try:
|
||||
values = values[index]
|
||||
except Exception as e:
|
||||
logging.error(f"values [{values}] error index [{index}]")
|
||||
logging.error(e)
|
||||
self.add_value(field_name, values, *processors, re=re, **kw)
|
||||
|
||||
def add_exec(self, field_name, value, str_exec=None, *processors, re=None, **kw):
|
||||
|
||||
@ -25,15 +25,18 @@ class ProxyMiddleware(object):
|
||||
if len(PROXY_LIST) != 0:
|
||||
request.meta["proxy"] = random.choice(PROXY_LIST)
|
||||
|
||||
################################################################
|
||||
# HTTP/HTTPS文件缓存中间件
|
||||
################################################################
|
||||
class MyFilesystemCacheStorage(FilesystemCacheStorage):
|
||||
|
||||
def verify_next(self, url):
|
||||
logging.info(f"cache url=== {url} {str(url).split('.')[-1]}")
|
||||
logging.debug(f"cache url=== {url} {str(url).split('.')[-1]}")
|
||||
if str(url).split('.')[-1] not in HTTPCACHE_ALLOW_PREFIXS:
|
||||
logging.info(f"取消缓存 === {url}")
|
||||
logging.debug(f"取消缓存 === {url}")
|
||||
return False
|
||||
else:
|
||||
logging.info(f"开始缓存 === {url}")
|
||||
logging.debug(f"开始缓存 === {url}")
|
||||
return True
|
||||
|
||||
def retrieve_response(self, spider: Spider, request: Request):
|
||||
|
||||
@ -7,15 +7,10 @@
|
||||
# useful for handling different item types with a single interface
|
||||
import os, scrapy,logging,time,random
|
||||
from Comics import settings
|
||||
from Comics.utils.FileUtils import imageUtils
|
||||
from Comics.utils.FileUtils import fileUtils
|
||||
from Comics.utils.Constant import ComicPath
|
||||
from Comics.items import ComicItem
|
||||
from Comics.items import ImagesItem
|
||||
from scrapy.pipelines.images import ImagesPipeline
|
||||
from Comics.exporters import ComicInfoXmlItemExporter
|
||||
from Comics.exporters import ItemExporter
|
||||
from Comics.exporters import JsonExport
|
||||
from Comics.exporters import ComicInfoXmlItemExporter,JsonExport
|
||||
from Comics.utils.FileUtils import CBZUtils
|
||||
|
||||
class ComicsPipeline:
|
||||
@ -25,6 +20,9 @@ class ComicsPipeline:
|
||||
def process_item(self, item, spider):
|
||||
if isinstance(item, ComicItem):
|
||||
file = os.path.join(settings.OUTPUT_DIR,"json", item['name'], item['chapter'])
|
||||
item['count'] = len(item['images'])
|
||||
item['images'].append({'src':'icon.jpg', 'scramble': False})
|
||||
item['image_urls'].append({'src': item['icon'] , 'scramble': False})
|
||||
data = JsonExport(file=file).export_json(item, if_return=True)
|
||||
#item['images'] = data['images']
|
||||
return data
|
||||
|
||||
@ -49,8 +49,8 @@ class RmComicSpider(scrapy.Spider):
|
||||
comic_item = ComicLoader(item=response.meta['item'], response=response)
|
||||
data = comic_item.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
|
||||
str_exec = "props.pageProps."
|
||||
#comic_item.add_exec('name', data, str_exec=str_exec+"bookName")
|
||||
#comic_item.add_exec('dep', data, str_exec=str_exec+"description")
|
||||
comic_item.add_exec('name', data, str_exec=str_exec+"bookName")
|
||||
comic_item.add_exec('dep', data, str_exec=str_exec+"description")
|
||||
comic_item.add_value('index', response.meta['num'])
|
||||
comic_item.add_exec('chapter', data, str_exec=str_exec + "chapterName")
|
||||
comic_item.add_exec('image_urls', data, str_exec+"images")
|
||||
|
||||
@ -275,7 +275,7 @@ class CBZUtils:
|
||||
y = 0
|
||||
for filename in filenames:
|
||||
y = y + 1
|
||||
print("打包中：" + str(y) + "/" + str(len(filenames)), os.path.join(source_dir, filename))
|
||||
logging.info(f"打包中：" + str(y) + "/" + str(len(filenames)), os.path.join(source_dir, filename))
|
||||
zf.write(path.joinpath(filename), arc_dir.joinpath(filename))
|
||||
zf.close()
|
||||
logging.info(f"打包完成：{target_file}")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user