This commit is contained in:
cwx committed 2024-03-31 15:55:14 +08:00
1 parent 9b3ac8878f
commit c0c77b3776
7 files changed
+343 -148

No files matched your search

+14 -4
View File
@@ -2,8 +2,8 @@ import scrapy,logging,time,os,skip
from Comics.items import ComicItem
from Comics.loader import ComicLoader
from Comics.utils import ComicPath
from Comics.utils import checkUtils
from Comics.utils import Conf
from Comics.exporters import ItemExporter
class RmComicSpider(scrapy.Spider):
name = 'rm_comic'
@@ -19,18 +19,27 @@ class RmComicSpider(scrapy.Spider):
# 获取多个漫画信息
def books_comic(self, response):
comics = ComicLoader(item=ComicItem(), response=response)
# 获取漫画网站//script[@id]内的json数据并获取props.pageProps.books数据并作偱环解析
for book in comics.get_exec(comics.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0], str_exec="props.pageProps.books"):
# 排除指定的漫画名
if book['name'] not in skip.skip_comic:
yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic)
# 获取某个漫画的相关数据
# 获取到多个章节链接后进入下个流程
def parse_comic(self, response):
# 初始化Comic数据并根据工程名称读取配置文件并自动解析
comic_item = Conf().comic(self.name, ComicLoader(ComicItem(), response))
# 循环遍历根据配置文件自动解析并注入的章节名和章节链接
for chapter, link in zip(comic_item.get_chapters(), comic_item.get_chapter_href()):
item = comic_item.load_item()
cbz_path = ComicPath().get_file_path(item=item, result_type="cbz", convert=True, chapter=chapter)
if not checkUtils().is_error(item) and os.path.exists(cbz_path):
# 打包导出item数据
item = comic_item.load_item(chapter=chapter)
# 获取最终存放CBZ的路径
#cbz_path = ComicPath(item=item).file_path(ComicPath.PATH_CBZ, convert=True, chapter=chapter)
cbz_path = ComicPath(item=item).PATH_CBZ()
# 校验繁体和简体中文CBZ路径是否存在
# if not checkUtils().is_error(item) and os.path.exists(cbz_path):
if os.path.exists(cbz_path):
logging.info(f"漫画 {cbz_path} 已存在, 跳过中...")
yield item
else:
@@ -40,6 +49,7 @@ class RmComicSpider(scrapy.Spider):
# 读取某章节下的所有图片
def parse_chapter(self, response):
# 获取传入的漫画item数据
comic_item = ComicLoader(item=response.meta['item'], response=response)
data = comic_item.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
item: ComicLoader = Conf().parse_chapter(item=comic_item, value=data)
+1 -1
View File
@@ -45,7 +45,7 @@ class RmComicSpider(scrapy.Spider):
comic_item.chapters(value=chapters)
comic_item.chapter(value=chapter)
item = comic_item.load_item()
cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True)
cbz_path = ComicPath(item).get_file_path(result_type="cbz", convert=True)
if os.path.exists(cbz_path):
logging.info(f"漫画 {cbz_path} 已存在, 跳过中...")
yield item