fix
This commit is contained in:
parent
e64ba840bf
commit
76c7aa69a8
@ -24,9 +24,13 @@ def _serialize_to_images(value, result_type=None):
|
||||
images_item = []
|
||||
image_urls = []
|
||||
for image in value:
|
||||
(image_src, scramble) = [image.get("src"), image.get("scramble")]
|
||||
try:
|
||||
(image_src, scramble) = [image.get("src"), image.get("scramble")]
|
||||
except:
|
||||
(image_src, scramble) = [image, False]
|
||||
count_image = settings.IMAGES_NAME_FORMAT.format(count)
|
||||
suffix = "."+str(image_src).split(".")[-1]
|
||||
# suffix = "."+str(image_src).split(".")[-1]
|
||||
suffix = ".jpg"
|
||||
image_name = count_image + suffix
|
||||
if scramble:
|
||||
de_str = str(image_src).split("/")[-1].replace(suffix, "==")
|
||||
|
||||
@ -1,8 +1,9 @@
|
||||
import json,logging
|
||||
from scrapy.loader import ItemLoader
|
||||
from Comics.settings import PROJECT_KEY
|
||||
|
||||
class ComicLoader(ItemLoader):
|
||||
def parseExec(cls,data,exec):
|
||||
def parseExec(self,data,exec):
|
||||
if data !=None and exec != None:
|
||||
dots = str(exec).split(".")
|
||||
if not isinstance(data,dict): data = json.loads(data)
|
||||
@ -62,3 +63,71 @@ class ComicLoader(ItemLoader):
|
||||
self._replace_value(field_name, value)
|
||||
return False
|
||||
else: return True
|
||||
|
||||
|
||||
# 设置漫画属性
|
||||
def set_properties(self, name, value=None, xpath=None, index=None, sexec=None):
|
||||
if value != None and sexec==None:
|
||||
self.add_value(field_name=name, value=value)
|
||||
if xpath != None:
|
||||
self.add_xpath(field_name=name, xpath=xpath, index=index)
|
||||
if sexec != None:
|
||||
self.add_exec(field_name=name, value=value, str_exec=sexec)
|
||||
|
||||
# 工程名
|
||||
def project_name(self, project_name): self.add_value(PROJECT_KEY, project_name)
|
||||
# 漫画名
|
||||
def name(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('name', value, xpath, index, sexec)
|
||||
# 漫画封面链接
|
||||
def icon(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('icon', value, xpath, index, sexec)
|
||||
# 作者
|
||||
def author(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('author', value, xpath, index, sexec)
|
||||
# 标签
|
||||
def tags(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('tags', value, xpath, index, sexec)
|
||||
# 概述
|
||||
def dep(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('dep', value, xpath, index, sexec)
|
||||
# 时间
|
||||
def date(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('date', value, xpath, index, sexec)
|
||||
# 流派
|
||||
def genre(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('genre', value, xpath, index, sexec)
|
||||
# 年龄分级
|
||||
def age_rating(self, value=None, xpath=None, index=None, sexec=None):
|
||||
self.set_properties('age_rating', value, xpath, index, sexec)
|
||||
# 全部章节
|
||||
def chapters(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('chapters', value, xpath, index, sexec)
|
||||
# 单一章节
|
||||
def chapter(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('chapter', value, xpath, index, sexec)
|
||||
# 图像名称
|
||||
def images(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('images', value, xpath, index, sexec)
|
||||
# 图像链接
|
||||
def image_urls(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('image_urls', value, xpath, index, sexec)
|
||||
|
||||
class ComicEntity:
|
||||
|
||||
ENTITY = None
|
||||
|
||||
def __init__(self, entity):
|
||||
self.ENTITY = entity
|
||||
|
||||
# 属性获取
|
||||
def get_dict(self, key):
|
||||
try:
|
||||
return self.ENTITY[key]
|
||||
except:
|
||||
return []
|
||||
|
||||
def set(self, key, value): self.ENTITY[key] = value
|
||||
def set_count(self, value): self.set('count', value)
|
||||
def set_index(self, value): self.set('index', value)
|
||||
def images(self): return self.get_dict("images")
|
||||
def image_urls(self): return self.get_dict("image_urls")
|
||||
def chapters(self): return self.get_dict("chapters")
|
||||
def chapter(self): return self.get_dict("chapter")
|
||||
def icon(self): return self.get_dict("icon")
|
||||
def count(self): return self.set_count(len(self.images()))
|
||||
def index(self): return self.set_index(self.chapters().index(self.chapter()) + 1)
|
||||
|
||||
def item(self):
|
||||
self.count()
|
||||
self.index()
|
||||
return self.ENTITY
|
||||
|
||||
@ -14,7 +14,7 @@ from scrapy.pipelines.images import ImagesPipeline
|
||||
from Comics.exporters import ComicInfoXmlItemExporter,JsonExport,ItemExporter
|
||||
from Comics.utils.FileUtils import CBZUtils
|
||||
from Comics.utils.FileUtils import fileUtils as fu
|
||||
|
||||
from Comics.loader import ComicEntity
|
||||
|
||||
class ComicsPipeline:
|
||||
def open_spider(self, spider):
|
||||
@ -24,15 +24,11 @@ class ComicsPipeline:
|
||||
def process_item(self, item, spider):
|
||||
if isinstance(item, ComicItem):
|
||||
# 'output/rm_comic/json/壞X/第1話 壞X'
|
||||
cbz_path = ComicPath.get_file_path(item, result_type="cbz", convert=True)
|
||||
if os.path.exists(cbz_path):
|
||||
item['image_urls'] = item['images'] = []
|
||||
if os.path.exists(ComicPath.CBZ(item=item)):
|
||||
return ItemExporter().export_obj(item)
|
||||
else:
|
||||
file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter'])
|
||||
item['count'] = len(item['images'])
|
||||
item['index'] = item['chapters'].index(item['chapter']) + 1
|
||||
return JsonExport(file=file).export_json(item, if_return=True)
|
||||
return JsonExport(file=file).export_json(ComicEntity(item).item(), if_return=True)
|
||||
# image解析
|
||||
|
||||
def close_spider(self, spider):
|
||||
@ -61,8 +57,9 @@ class ImgDownloadPipeline(ImagesPipeline):
|
||||
def file_path(self, request, response=None, info=None, *, item=None): return request.meta['path']
|
||||
|
||||
def get_media_requests(self, item, info):
|
||||
self.image_urls = item['image_urls']
|
||||
self.images = item['images']
|
||||
comic = ComicEntity(item)
|
||||
self.image_urls = comic.image_urls()
|
||||
self.images = comic.images()
|
||||
# 下载封面
|
||||
self.download_icon(item)
|
||||
for image_url,image in zip(self.image_urls,self.images):
|
||||
|
||||
@ -22,29 +22,29 @@ class RmComicSpider(scrapy.Spider):
|
||||
data = comics.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
|
||||
for book in comics.get_exec(data, str_exec="props.pageProps.books"):
|
||||
comics.add_value('link', self.start_urls+"/"+book['id'])
|
||||
if not book['name'] in skip.skip_comic:
|
||||
if book['name'] not in skip.skip_comic:
|
||||
yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic)
|
||||
|
||||
# 获取某个漫画的相关数据
|
||||
# 获取到多个章节链接后进入下个流程
|
||||
def parse_comic(self, response):
|
||||
comic_item = ComicLoader(item=ComicItem(), response=response)
|
||||
comic_item.add_value(PROJECT_KEY, self.name)
|
||||
comic_item.add_xpath('name', '//div[@class="col"]/h5/text()')
|
||||
comic_item.add_xpath('icon', '//img[@class="img-thumbnail"]/@src')
|
||||
comic_item.add_xpath('author', '//div[contains(@class,"bookid_bookInfo")]/p[1]/text()', index=1)
|
||||
comic_item.add_xpath('tags', '//div[contains(@class,"bookid_bookInfo")]/p[3]/b/text()')
|
||||
comic_item.add_xpath('dep', '//div[contains(@class,"bookid_bookInfo")]/p[4]/text()', index=1)
|
||||
comic_item.add_xpath('date', '//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()', index=1)
|
||||
comic_item.add_value('genre', "韩漫")
|
||||
comic_item.add_value('age_rating', "R18+")
|
||||
comic_item.project_name(self.name)
|
||||
comic_item.name(xpath='//div[@class="col"]/h5/text()')
|
||||
comic_item.icon(xpath='//img[@class="img-thumbnail"]/@src')
|
||||
comic_item.author(xpath='//div[contains(@class,"bookid_bookInfo")]/p[1]/text()', index=1)
|
||||
comic_item.tags(xpath='//div[contains(@class,"bookid_bookInfo")]/p[3]/b/text()')
|
||||
comic_item.dep(xpath='//div[contains(@class,"bookid_bookInfo")]/p[4]/text()', index=1)
|
||||
comic_item.date(xpath='//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()', index=1)
|
||||
comic_item.genre(value="韩漫")
|
||||
comic_item.age_rating(value="R18+")
|
||||
chapter_href = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]'
|
||||
'//div[contains(@class,"bookid_chapter")]/a/@href')
|
||||
chapters = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]'
|
||||
'//div[contains(@class,"bookid_chapter")]/a/text()')
|
||||
for chapter, link in zip(chapters, chapter_href):
|
||||
comic_item.add_value('chapter', chapter)
|
||||
comic_item.add_value('chapters', chapters)
|
||||
comic_item.chapters(value=chapters)
|
||||
comic_item.chapter(value=chapter)
|
||||
item = comic_item.load_item()
|
||||
cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True)
|
||||
if os.path.exists(cbz_path):
|
||||
@ -59,11 +59,11 @@ class RmComicSpider(scrapy.Spider):
|
||||
comic_item = ComicLoader(item=response.meta['item'], response=response)
|
||||
data = comic_item.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
|
||||
str_exec = "props.pageProps."
|
||||
comic_item.add_exec('name', data, str_exec=str_exec+"bookName")
|
||||
comic_item.add_exec('dep', data, str_exec=str_exec+"description")
|
||||
comic_item.add_exec('chapter', data, str_exec=str_exec + "chapterName")
|
||||
comic_item.add_exec('image_urls', data, str_exec+"images")
|
||||
comic_item.add_exec('images', data, str_exec+"images")
|
||||
comic_item.name(value=data, sexec=str_exec+"bookName")
|
||||
comic_item.dep(value=data, sexec=str_exec+"description")
|
||||
comic_item.chapter(value=data, sexec=str_exec+"chapterName")
|
||||
comic_item.image_urls(value=data, sexec=str_exec+"images")
|
||||
comic_item.images(value=data, sexec=str_exec+"images")
|
||||
comic = comic_item.load_item()
|
||||
chapter_api_url = comic_item.get_exec(data, str_exec+"chapterAPIPath")
|
||||
if chapter_api_url is not None:
|
||||
@ -74,9 +74,9 @@ class RmComicSpider(scrapy.Spider):
|
||||
# 加密数据API处理
|
||||
def parse_chapter_api(self, response):
|
||||
comic_item = ComicLoader(item=response.meta['item'], response=response)
|
||||
comic_item.add_exec('chapter', response.text, str_exec='chapter.name')
|
||||
comic_item.add_exec('image_urls', response.text, str_exec='chapter.images')
|
||||
comic_item.add_exec('images', response.text, str_exec='chapter.images')
|
||||
comic_item.chapter(value=response.text, sexec='chapter.name')
|
||||
comic_item.image_urls(value=response.text, sexec='chapter.images')
|
||||
comic_item.images(value=response.text, sexec='chapter.images')
|
||||
yield comic_item.load_item()
|
||||
|
||||
|
||||
|
||||
69
Comics/spiders/yh_comic.py
Normal file
69
Comics/spiders/yh_comic.py
Normal file
@ -0,0 +1,69 @@
|
||||
import scrapy,logging,time,os
|
||||
from Comics.items import ComicItem
|
||||
from Comics.loader import ComicLoader
|
||||
from Comics.items import ListComicItem
|
||||
from Comics.utils.Constant import ComicPath
|
||||
from Comics.settings import PROJECT_KEY
|
||||
import skip
|
||||
|
||||
class RmComicSpider(scrapy.Spider):
|
||||
name = 'yh_comic'
|
||||
allowed_domains = ['www.shuanglilock.com.cn']
|
||||
main_url = 'https://'+allowed_domains[0]
|
||||
start_urls = main_url+'/info'
|
||||
|
||||
def start_requests(self):
|
||||
# for x in range(0,60):
|
||||
yield scrapy.Request("https://www.shuanglilock.com.cn/info/27145/", callback=self.parse_comic)
|
||||
|
||||
# 获取多个漫画信息
|
||||
# def books_comic(self, response):
|
||||
# comics = ComicLoader(item=ComicItem(), response=response)
|
||||
# data = comics.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
|
||||
# for book in comics.get_exec(data, str_exec="props.pageProps.books"):
|
||||
# comics.add_value('link', self.start_urls+"/"+book['id'])
|
||||
# if book['name'] not in skip.skip_comic:
|
||||
# yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic)
|
||||
|
||||
# 获取某个漫画的相关数据
|
||||
# 获取到多个章节链接后进入下个流程
|
||||
def parse_comic(self, response):
|
||||
comic_item = ComicLoader(item=ComicItem(), response=response)
|
||||
comic_item.project_name(self.name)
|
||||
comic_item.name(xpath='//div[@class="comics-detail__info"]/h1[@class="comics-detail__title"]/text()')
|
||||
comic_item.icon(xpath='//div[@class="pure-u-1-1 pure-u-sm-1-3 pure-u-md-1-6"]/img/@src')
|
||||
comic_item.author(xpath='//div[@class="comics-detail__info"]/h2[@class="comics-detail__author"]/text()')
|
||||
comic_item.tags(xpath='//div[@class="tag-list"]/a[@class="tag"]/text()')
|
||||
comic_item.dep(xpath='//p[contains(@class,"comics-detail__desc")]/text()')
|
||||
#comic_item.date(xpath='//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()', index=1)
|
||||
comic_item.genre(value="樱花漫画")
|
||||
#comic_item.age_rating(value="R18+")
|
||||
chapter_href = comic_item.get_xpath('//div[contains(@id,"chapter-items")]'
|
||||
'//a[@class="comics-chapters__item"]/@href')
|
||||
chapters = comic_item.get_xpath('//div[contains(@id,"chapter-items")]'
|
||||
'//a[@class="comics-chapters__item"]//span/text()')
|
||||
for chapter, link in zip(chapters, chapter_href):
|
||||
comic_item.chapters(value=chapters)
|
||||
comic_item.chapter(value=chapter)
|
||||
item = comic_item.load_item()
|
||||
cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True)
|
||||
if os.path.exists(cbz_path):
|
||||
logging.info(f"漫画 {cbz_path} 已存在, 跳过中...")
|
||||
yield item
|
||||
else:
|
||||
yield scrapy.Request(self.main_url+link, meta={'item': item}, callback=self.parse_chapter)
|
||||
|
||||
|
||||
# 读取某章节下的所有图片
|
||||
def parse_chapter(self, response):
|
||||
comic_item = ComicLoader(item=response.meta['item'], response=response)
|
||||
comic_item.image_urls(xpath='//div[@class="comiclist"]/div[@class="comicpage"]/div/img/@data-original')
|
||||
comic_item.images(xpath='//div[@class="comiclist"]/div[@class="comicpage"]/div/img/@data-original')
|
||||
comic = comic_item.load_item()
|
||||
yield comic
|
||||
|
||||
def parse(self, response):
|
||||
raise NotImplementedError
|
||||
|
||||
def error_parse(self, response):
|
||||
raise NotImplementedError
|
||||
@ -79,6 +79,11 @@ class ComicPath:
|
||||
file = os.path.join(settings.IMAGES_STORE, PROJECT, "images", name, chapter)
|
||||
return file
|
||||
|
||||
@classmethod
|
||||
def CBZ(cls, item):
|
||||
return cls.get_file_path(item, result_type="cbz", convert=True)
|
||||
|
||||
|
||||
class ntfy:
|
||||
@classmethod
|
||||
def sendMsg(cls, msg,alert=False,sleep=None,error=None):
|
||||
|
||||
Loading…
Reference in New Issue
Block a user