cwx/ComicScrapy
1
0 Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
cwx committed 2023-10-03 16:54:02 +08:00
1 parent e64ba840bf
commit 76c7aa69a8
6 files changed
+178 -34
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Comics/spiders/rm_comic.py
+20 -20
View File
@@ -22,29 +22,29 @@ class RmComicSpider(scrapy.Spider):
data = comics.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
for book in comics.get_exec(data, str_exec="props.pageProps.books"):
comics.add_value('link', self.start_urls+"/"+book['id'])
if not book['name'] in skip.skip_comic:
if book['name'] not in skip.skip_comic:
yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic)
# 获取某个漫画的相关数据
# 获取到多个章节链接后进入下个流程
def parse_comic(self, response):
comic_item = ComicLoader(item=ComicItem(), response=response)
comic_item.add_value(PROJECT_KEY, self.name)
comic_item.add_xpath('name', '//div[@class="col"]/h5/text()')
comic_item.add_xpath('icon', '//img[@class="img-thumbnail"]/@src')
comic_item.add_xpath('author', '//div[contains(@class,"bookid_bookInfo")]/p[1]/text()', index=1)
comic_item.add_xpath('tags', '//div[contains(@class,"bookid_bookInfo")]/p[3]/b/text()')
comic_item.add_xpath('dep', '//div[contains(@class,"bookid_bookInfo")]/p[4]/text()', index=1)
comic_item.add_xpath('date', '//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()', index=1)
comic_item.add_value('genre', "韩漫")
comic_item.add_value('age_rating', "R18+")
comic_item.project_name(self.name)
comic_item.name(xpath='//div[@class="col"]/h5/text()')
comic_item.icon(xpath='//img[@class="img-thumbnail"]/@src')
comic_item.author(xpath='//div[contains(@class,"bookid_bookInfo")]/p[1]/text()', index=1)
comic_item.tags(xpath='//div[contains(@class,"bookid_bookInfo")]/p[3]/b/text()')
comic_item.dep(xpath='//div[contains(@class,"bookid_bookInfo")]/p[4]/text()', index=1)
comic_item.date(xpath='//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()', index=1)
comic_item.genre(value="韩漫")
comic_item.age_rating(value="R18+")
chapter_href = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]'
'//div[contains(@class,"bookid_chapter")]/a/@href')
chapters = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]'
'//div[contains(@class,"bookid_chapter")]/a/text()')
for chapter, link in zip(chapters, chapter_href):
comic_item.add_value('chapter', chapter)
comic_item.add_value('chapters', chapters)
comic_item.chapters(value=chapters)
comic_item.chapter(value=chapter)
item = comic_item.load_item()
cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True)
if os.path.exists(cbz_path):
@@ -59,11 +59,11 @@ class RmComicSpider(scrapy.Spider):
comic_item = ComicLoader(item=response.meta['item'], response=response)
data = comic_item.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
str_exec = "props.pageProps."
comic_item.add_exec('name', data, str_exec=str_exec+"bookName")
comic_item.add_exec('dep', data, str_exec=str_exec+"description")
comic_item.add_exec('chapter', data, str_exec=str_exec + "chapterName")
comic_item.add_exec('image_urls', data, str_exec+"images")
comic_item.add_exec('images', data, str_exec+"images")
comic_item.name(value=data, sexec=str_exec+"bookName")
comic_item.dep(value=data, sexec=str_exec+"description")
comic_item.chapter(value=data, sexec=str_exec+"chapterName")
comic_item.image_urls(value=data, sexec=str_exec+"images")
comic_item.images(value=data, sexec=str_exec+"images")
comic = comic_item.load_item()
chapter_api_url = comic_item.get_exec(data, str_exec+"chapterAPIPath")
if chapter_api_url is not None:
@@ -74,9 +74,9 @@ class RmComicSpider(scrapy.Spider):
# 加密数据API处理
def parse_chapter_api(self, response):
comic_item = ComicLoader(item=response.meta['item'], response=response)
comic_item.add_exec('chapter', response.text, str_exec='chapter.name')
comic_item.add_exec('image_urls', response.text, str_exec='chapter.images')
comic_item.add_exec('images', response.text, str_exec='chapter.images')
comic_item.chapter(value=response.text, sexec='chapter.name')
comic_item.image_urls(value=response.text, sexec='chapter.images')
comic_item.images(value=response.text, sexec='chapter.images')
yield comic_item.load_item()
Comics/spiders/yh_comic.py
+69
View File
@@ -0,0 +1,69 @@
import scrapy,logging,time,os
from Comics.items import ComicItem
from Comics.loader import ComicLoader
from Comics.items import ListComicItem
from Comics.utils.Constant import ComicPath
from Comics.settings import PROJECT_KEY
import skip
class RmComicSpider(scrapy.Spider):
name = 'yh_comic'
allowed_domains = ['www.shuanglilock.com.cn']
main_url = 'https://'+allowed_domains[0]
start_urls = main_url+'/info'
def start_requests(self):
# for x in range(0,60):
yield scrapy.Request("https://www.shuanglilock.com.cn/info/27145/", callback=self.parse_comic)
# 获取多个漫画信息
# def books_comic(self, response):
# comics = ComicLoader(item=ComicItem(), response=response)
# data = comics.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
# for book in comics.get_exec(data, str_exec="props.pageProps.books"):
# comics.add_value('link', self.start_urls+"/"+book['id'])
# if book['name'] not in skip.skip_comic:
# yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic)
# 获取某个漫画的相关数据
# 获取到多个章节链接后进入下个流程
def parse_comic(self, response):
comic_item = ComicLoader(item=ComicItem(), response=response)
comic_item.project_name(self.name)
comic_item.name(xpath='//div[@class="comics-detail__info"]/h1[@class="comics-detail__title"]/text()')
comic_item.icon(xpath='//div[@class="pure-u-1-1 pure-u-sm-1-3 pure-u-md-1-6"]/img/@src')
comic_item.author(xpath='//div[@class="comics-detail__info"]/h2[@class="comics-detail__author"]/text()')
comic_item.tags(xpath='//div[@class="tag-list"]/a[@class="tag"]/text()')
comic_item.dep(xpath='//p[contains(@class,"comics-detail__desc")]/text()')
#comic_item.date(xpath='//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()', index=1)
comic_item.genre(value="樱花漫画")
#comic_item.age_rating(value="R18+")
chapter_href = comic_item.get_xpath('//div[contains(@id,"chapter-items")]'
'//a[@class="comics-chapters__item"]/@href')
chapters = comic_item.get_xpath('//div[contains(@id,"chapter-items")]'
'//a[@class="comics-chapters__item"]//span/text()')
for chapter, link in zip(chapters, chapter_href):
comic_item.chapters(value=chapters)
comic_item.chapter(value=chapter)
item = comic_item.load_item()
cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True)
if os.path.exists(cbz_path):
logging.info(f"漫画 {cbz_path} 已存在, 跳过中...")
yield item
else:
yield scrapy.Request(self.main_url+link, meta={'item': item}, callback=self.parse_chapter)
# 读取某章节下的所有图片
def parse_chapter(self, response):
comic_item = ComicLoader(item=response.meta['item'], response=response)
comic_item.image_urls(xpath='//div[@class="comiclist"]/div[@class="comicpage"]/div/img/@data-original')
comic_item.images(xpath='//div[@class="comiclist"]/div[@class="comicpage"]/div/img/@data-original')
comic = comic_item.load_item()
yield comic
def parse(self, response):
raise NotImplementedError
def error_parse(self, response):
raise NotImplementedError