update
This commit is contained in:
cwx committed
1 parent
8f55a51140
commit
00f77967a3
13 files changed
+255
-312
No files matched your search
+39
-39
@@ -1,9 +1,8 @@
|
||||
import urllib.parse
|
||||
|
||||
import scrapy,json,requests
|
||||
import scrapy
|
||||
from Comics.items import ComicItem
|
||||
from Comics.utils.FileUtils import CommonUtils
|
||||
from scrapy.loader import ItemLoader
|
||||
from Comics.loader import ComicLoader
|
||||
from itemadapter import ItemAdapter
|
||||
from Comics.items import ComicInfoItem
|
||||
|
||||
class RmComicSpider(scrapy.Spider):
|
||||
name = 'rm_comic'
|
||||
@@ -14,49 +13,50 @@ class RmComicSpider(scrapy.Spider):
|
||||
def start_requests(self):
|
||||
yield scrapy.Request('https://rm01.xyz'
|
||||
'/books/306ec1e2-f701-4fda-bb78-041ad6ec4020', callback=self.parse_comic)
|
||||
|
||||
|
||||
# 获取某个漫画的相关数据
|
||||
# 获取到多个章节链接后进入下个流程
|
||||
def parse_comic(self, response):
|
||||
comic = ComicItem()
|
||||
# comic_item = ItemLoader(item=ComicItem(), response=response)
|
||||
comic['name'] = response.xpath('//div[@class="col"]/h5/text()').extract_first()
|
||||
comic['icon'] = response.xpath('//img[@class="img-thumbnail"]/@src').extract_first()
|
||||
comic['author'] = response.xpath('//div[contains(@class,"bookid_bookInfo")]/p[1]/text()').extract()[1]
|
||||
comic['tags'] = response.xpath('//div[contains(@class,"bookid_bookInfo")]/p[3]/b/text()').extract_first()
|
||||
comic['dep'] = response.xpath('//div[contains(@class,"bookid_bookInfo")]/p[4]/text()').extract()[1]
|
||||
comic['date'] = response.xpath('//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()').extract()[1]
|
||||
comic['genre'] = "韩漫"
|
||||
comic['age_rating'] = "R18+"
|
||||
chapters = response.xpath('//div[contains(@class,"bookid_chapterBox")]'
|
||||
'//div[contains(@class,"bookid_chapter")]/a/text()').extract()
|
||||
chapter_href = response.xpath('//div[contains(@class,"bookid_chapterBox")]'
|
||||
'//div[contains(@class,"bookid_chapter")]/a/@href').extract()
|
||||
comic_item = ComicLoader(item=ComicItem(), response=response)
|
||||
comic_item.add_xpath('name', '//div[@class="col"]/h5/text()')
|
||||
comic_item.add_xpath('icon', '//img[@class="img-thumbnail"]/@src')
|
||||
comic_item.add_xpath('author', '//div[contains(@class,"bookid_bookInfo")]/p[1]/text()', index=1)
|
||||
comic_item.add_xpath('tags', '//div[contains(@class,"bookid_bookInfo")]/p[3]/b/text()')
|
||||
comic_item.add_xpath('dep', '//div[contains(@class,"bookid_bookInfo")]/p[4]/text()', index=1)
|
||||
comic_item.add_xpath('date', '//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()', index=1)
|
||||
comic_item.add_value('genre', "韩漫")
|
||||
comic_item.add_value('age_rating', "R18+")
|
||||
chapter_href = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]'
|
||||
'//div[contains(@class,"bookid_chapter")]/a/@href')
|
||||
#chapters = response.xpath('//div[contains(@class,"bookid_chapterBox")]'
|
||||
# '//div[contains(@class,"bookid_chapter")]/a/text()').extract()
|
||||
#for chapter, link in zip(chapters, chapter_href):
|
||||
for i, link in enumerate(chapter_href, start=1):
|
||||
yield scrapy.Request(self.main_url+link, meta={'item' : comic, 'number': i}, callback=self.parse_chapter)
|
||||
yield scrapy.Request(self.main_url+link, meta={'item': comic_item.load_item(), 'num': i}, callback=self.parse_chapter)
|
||||
|
||||
# 读取某章节下的所有图片
|
||||
def parse_chapter(self, response):
|
||||
item = response.meta['item']
|
||||
number = response.meta['number']
|
||||
data = response.xpath('//script[@id="__NEXT_DATA__"]/text()').extract_first()
|
||||
comic_item = ComicLoader(item=response.meta['item'], response=response)
|
||||
data = comic_item.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
|
||||
str_exec = "props.pageProps."
|
||||
comic_name = CommonUtils.parseExec(data, str_exec+"bookName")
|
||||
chapterName = CommonUtils.parseExec(data, str_exec+"chapterName")
|
||||
description = CommonUtils.parseExec(data, str_exec+"description")
|
||||
images = CommonUtils.parseExec(data, str_exec+"images")
|
||||
chapter_api_url = CommonUtils.parseExec(data, str_exec+"chapterAPIPath")
|
||||
item['chapter'] = chapterName
|
||||
item['list_img'] = images
|
||||
item['index'] = number
|
||||
if chapter_api_url != None:
|
||||
yield scrapy.Request(self.main_url+ chapter_api_url,meta={'item' : item}, callback= self.parse_chapter_api)
|
||||
#comic_item.add_exec('name', data, str_exec=str_exec+"bookName")
|
||||
#comic_item.add_exec('dep', data, str_exec=str_exec+"description")
|
||||
comic_item.add_value('index', response.meta['num'])
|
||||
comic_item.add_exec('chapter', data, str_exec=str_exec + "chapterName")
|
||||
comic_item.add_exec('list_img', data, str_exec+"images")
|
||||
comic = comic_item.load_item()
|
||||
chapter_api_url = comic_item.get_exec(data, str_exec+"chapterAPIPath")
|
||||
if chapter_api_url is not None:
|
||||
yield scrapy.Request(self.main_url + chapter_api_url, meta={'item': comic}, callback=self.parse_chapter_api)
|
||||
else:
|
||||
yield item
|
||||
yield comic
|
||||
|
||||
# 加密数据API处理
|
||||
def parse_chapter_api(self, response):
|
||||
item = response.meta['item']
|
||||
item['chapter'] = CommonUtils.parseExec(response.text, "chapter.name")
|
||||
item['list_img'] = CommonUtils.parseExec(response.text, "chapter.images")
|
||||
yield item
|
||||
comic_item = ComicLoader(item=response.meta['item'], response=response)
|
||||
comic_item.add_exec('chapter', response.text, str_exec='chapter.name')
|
||||
comic_item.add_exec('list_img', response.text, str_exec='chapter.images')
|
||||
yield comic_item.load_item()
|
||||
|
||||
def parse(self, response):
|
||||
raise NotImplementedError
|
||||
Reference in new issue
Block a user