update new Rouman5
This commit is contained in:
cwx committed
1 parent
6638254416
commit
5f5b6cb6ec
10 files changed
+142
-169
No files matched your search
+26
-23
@@ -1,6 +1,8 @@
|
||||
import scrapy,logging,os,skip
|
||||
from Comics.items import ComicItem
|
||||
import scrapy,logging,os,skip,json,re
|
||||
from Comics.items import ComicItem,Image
|
||||
from Comics.items import BooksItem
|
||||
from Comics.loader import ComicLoader
|
||||
from Comics.loader import BooksLoader
|
||||
from Comics.utils import ComicPath
|
||||
from Comics.utils import Conf
|
||||
from Comics.utils import oldUtils
|
||||
@@ -9,7 +11,7 @@ class RmComicSpider(scrapy.Spider):
|
||||
name = 'rm_comic'
|
||||
allowed_domains = ['rouman5.com']
|
||||
main_url = 'https://'+allowed_domains[0]
|
||||
start_urls = main_url+'/books'
|
||||
start_urls = main_url+"/books"
|
||||
|
||||
# 遍历网站页数数据
|
||||
def start_requests(self):
|
||||
@@ -18,18 +20,18 @@ class RmComicSpider(scrapy.Spider):
|
||||
|
||||
# 获取多个漫画信息
|
||||
def books_comic(self, response):
|
||||
comics = ComicLoader(item=ComicItem(), response=response)
|
||||
books_item = Conf().books(self.name, BooksLoader(BooksItem(), response))
|
||||
# 获取漫画网站//script[@id]内的json数据并获取props.pageProps.books数据并作偱环解析
|
||||
for book in comics.get_exec(comics.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0], str_exec="props.pageProps.books"):
|
||||
for book,url in zip(books_item.get_names(), books_item.get_urls()):
|
||||
# 排除指定的漫画名
|
||||
if book['name'] not in skip.skip_comic:
|
||||
yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic)
|
||||
if book not in skip.skip_comic: yield scrapy.Request(url=self.main_url+"/"+url, callback=self.parse_comic)
|
||||
|
||||
# 获取某个漫画的相关数据
|
||||
# 获取到多个章节链接后进入下个流程
|
||||
def parse_comic(self, response):
|
||||
# 初始化Comic数据并根据工程名称读取配置文件并自动解析
|
||||
comic_item = Conf().comic(self.name, ComicLoader(ComicItem(), response))
|
||||
comic_item.set_domain(self.main_url)
|
||||
path_comic = comic_item.load_item()
|
||||
cbz_dir = ComicPath(path_comic).file_path(result_type=ComicPath.MAPPING_CBZ_DIR)
|
||||
move_folder = ComicPath(path_comic).file_path(result_type=ComicPath.MAPPING_OLD_CBZ_DIR)
|
||||
@@ -44,7 +46,6 @@ class RmComicSpider(scrapy.Spider):
|
||||
# 获取最终存放CBZ的路径
|
||||
cbz_path = ComicPath(item=item).PATH_CBZ()
|
||||
# 校验繁体和简体中文CBZ路径是否存在
|
||||
# if not checkUtils().is_error(item) and os.path.exists(cbz_path):
|
||||
if cbz_path !=None and os.path.exists(cbz_path):
|
||||
logging.info(f"漫画 {cbz_path} 已存在, 跳过中...")
|
||||
yield item
|
||||
@@ -56,22 +57,24 @@ class RmComicSpider(scrapy.Spider):
|
||||
def parse_chapter(self, response):
|
||||
# 获取传入的漫画item数据
|
||||
ci = ComicLoader(item=response.meta['item'], response=response)
|
||||
reuslt_json = None
|
||||
for data_json in ci.get_xpath('//script/text()'):
|
||||
if data_json.startswith('self.__next_f.push([1,"5') : reuslt_json = data_json
|
||||
# 再次通过获取的XPATH数据解析并保存到ci(ComicItem)中
|
||||
item: ComicLoader = Conf().parse_chapter(item=ci, value=ci.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0])
|
||||
comic, chapter_api_url = [ item.load_item() ,item.get_chapter_api() ]
|
||||
if chapter_api_url is not None and len(chapter_api_url) != 0 :
|
||||
try:
|
||||
yield scrapy.Request(self.main_url + chapter_api_url, meta={'item': comic}, callback=self.parse_chapter_api)
|
||||
except:
|
||||
logging.warning(f"yield scrapy.Request({self.main_url} + {chapter_api_url}, meta={comic}, callback=self.parse_chapter_api)")
|
||||
else:
|
||||
yield comic
|
||||
|
||||
# 加密数据API处理
|
||||
def parse_chapter_api(self, response):
|
||||
comic_item = ComicLoader(item=response.meta['item'], response=response)
|
||||
return Conf().parse_chapter_api(item=comic_item, value=response.text).load_item()
|
||||
|
||||
# 正则表达式匹配 .jpg 链接
|
||||
jpg_links = re.findall(r'(https?://\S+\.jpg)', reuslt_json)
|
||||
images_urls = []
|
||||
# 打印提取的 .jpg 链接
|
||||
for link in jpg_links:
|
||||
sr_value = re.search(r'sr:(\d+)', link)
|
||||
# 打印提取到的 sr: 的值
|
||||
if sr_value:
|
||||
sr = sr_value.group(1) # group(1) 返回第一个捕获组，即数字部分
|
||||
else:
|
||||
print("No match found")
|
||||
images_urls.append(Image().setImage(url=link, scramble=sr.replace("0", "False").replace("1", "True")))
|
||||
ci.image_urls(value=images_urls)
|
||||
yield ci.load_item()
|
||||
|
||||
def parse(self, response):
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -1,22 +1,29 @@
|
||||
books:
|
||||
names: '//div[@class="truncate"]/text()'
|
||||
urls: '//div[@class="grid grid-cols-1 sm:grid-cols-4 md:grid-cols-6 gap-2 sm:gap-4"]//a/@href'
|
||||
|
||||
|
||||
data:
|
||||
name: '//div[@class="col"]/h5/text()'
|
||||
icon: '//img[@class="img-thumbnail"]/@src'
|
||||
name: '//div[@class="basis-3/5 text-sm sm:text-base"]//div[@class="text-xl text-gray-900"]/text()'
|
||||
icon: '//div[@class="flex flex-row gap-3 sm:gap-4"]//div[@class="basis-2/5"]/img[@class="rounded"]/@src'
|
||||
author:
|
||||
xpath: '//div[contains(@class,"bookid_bookInfo")]/p[1]/text()'
|
||||
index: 1
|
||||
tags: '//div[contains(@class,"bookid_bookInfo")]/p[3]/b/text()'
|
||||
xpath: '//div[@class="flex flex-row gap-3 sm:gap-4"]//span[@class="text-gray-800"]/text()'
|
||||
index: 0
|
||||
tags:
|
||||
xpath: '//div[@class="flex flex-row gap-3 sm:gap-4"]//span[@class="text-gray-800"]/text()'
|
||||
index: 3
|
||||
dep:
|
||||
xpath: '//div[contains(@class,"bookid_bookInfo")]/p[4]/text()'
|
||||
xpath: '//div[@class="my-2 text-gray-800 text-sm sm:text-base"]/p/text()'
|
||||
index: 1
|
||||
date:
|
||||
xpath: '//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()'
|
||||
xpath: '//div[@class="text-gray-500 text-sm mt-2"]/div/text()'
|
||||
index: 1
|
||||
genre:
|
||||
value: "韩漫"
|
||||
age_rating:
|
||||
value: "R18+"
|
||||
chapter_href: '//div[contains(@class,"bookid_chapterBox")]//div[contains(@class,"bookid_chapter")]/a/@href'
|
||||
chapters: '//div[contains(@class,"bookid_chapterBox")]//div[contains(@class,"bookid_chapter")]/a/text()'
|
||||
chapter_href: '//div[@class="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 gap-2 px-2 py-4"]//a/@href'
|
||||
chapters: '//div[@class="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 gap-2 px-2 py-4"]//div[@class="text truncate bg-slate-300 p-2 hover:bg-rose-100"]/text()'
|
||||
|
||||
parse_chapter:
|
||||
name:
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
import scrapy,logging,time,os
|
||||
from Comics.items import ComicItem
|
||||
from Comics.loader import ComicLoader
|
||||
from Comics.utils import ComicPath
|
||||
from Comics.settings import PROJECT_KEY
|
||||
import skip
|
||||
|
||||
class RmComicSpider(scrapy.Spider):
|
||||
name = 'yh_comic'
|
||||
allowed_domains = ['www.shuanglilock.com.cn']
|
||||
main_url = 'https://'+allowed_domains[0]
|
||||
start_urls = main_url+'/info'
|
||||
|
||||
def start_requests(self):
|
||||
# for x in range(0,60):
|
||||
yield scrapy.Request("https://www.shuanglilock.com.cn/info/27145/", callback=self.parse_comic)
|
||||
|
||||
# 获取多个漫画信息
|
||||
# def books_comic(self, response):
|
||||
# comics = ComicLoader(item=ComicItem(), response=response)
|
||||
# data = comics.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
|
||||
# for book in comics.get_exec(data, str_exec="props.pageProps.books"):
|
||||
# comics.add_value('link', self.start_urls+"/"+book['id'])
|
||||
# if book['name'] not in skip.skip_comic:
|
||||
# yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic)
|
||||
|
||||
# 获取某个漫画的相关数据
|
||||
# 获取到多个章节链接后进入下个流程
|
||||
def parse_comic(self, response):
|
||||
comic_item = ComicLoader(item=ComicItem(), response=response)
|
||||
comic_item.project_name(self.name)
|
||||
comic_item.name(xpath='//div[@class="comics-detail__info"]/h1[@class="comics-detail__title"]/text()')
|
||||
comic_item.icon(xpath='//div[@class="pure-u-1-1 pure-u-sm-1-3 pure-u-md-1-6"]/img/@src')
|
||||
comic_item.author(xpath='//div[@class="comics-detail__info"]/h2[@class="comics-detail__author"]/text()')
|
||||
comic_item.tags(xpath='//div[@class="tag-list"]/a[@class="tag"]/text()')
|
||||
comic_item.dep(xpath='//p[contains(@class,"comics-detail__desc")]/text()')
|
||||
#comic_item.date(xpath='//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()', index=1)
|
||||
comic_item.genre(value="樱花漫画")
|
||||
#comic_item.age_rating(value="R18+")
|
||||
chapter_href = comic_item.get_xpath('//div[contains(@id,"chapter-items")]'
|
||||
'//a[@class="comics-chapters__item"]/@href')
|
||||
chapters = comic_item.get_xpath('//div[contains(@id,"chapter-items")]'
|
||||
'//a[@class="comics-chapters__item"]//span/text()')
|
||||
for chapter, link in zip(chapters, chapter_href):
|
||||
comic_item.chapters(value=chapters)
|
||||
comic_item.chapter(value=chapter)
|
||||
item = comic_item.load_item()
|
||||
cbz_path = ComicPath(item).get_file_path(result_type="cbz", convert=True)
|
||||
if os.path.exists(cbz_path):
|
||||
logging.info(f"漫画 {cbz_path} 已存在, 跳过中...")
|
||||
yield item
|
||||
else:
|
||||
yield scrapy.Request(self.main_url+link, meta={'item': item}, callback=self.parse_chapter)
|
||||
|
||||
|
||||
# 读取某章节下的所有图片
|
||||
def parse_chapter(self, response):
|
||||
comic_item = ComicLoader(item=response.meta['item'], response=response)
|
||||
comic_item.image_urls(xpath='//div[@class="comiclist"]/div[@class="comicpage"]/div/img/@data-original')
|
||||
comic_item.images(xpath='//div[@class="comiclist"]/div[@class="comicpage"]/div/img/@data-original')
|
||||
comic = comic_item.load_item()
|
||||
yield comic
|
||||
|
||||
def parse(self, response):
|
||||
raise NotImplementedError
|
||||
|
||||
def error_parse(self, response):
|
||||
raise NotImplementedError
|
||||
Reference in new issue
Block a user