73 lines
3.2 KiB
Python
73 lines
3.2 KiB
Python
import scrapy,json,requests
|
|
from Comics.items import ComicItem
|
|
from Comics.utils.FileUtils import CommonUtils
|
|
import threading
|
|
import toml
|
|
|
|
class ErrorLog:
|
|
def __init__(self) -> None:
|
|
self.lock = threading.Lock()
|
|
|
|
def err_ls(self, dic):
|
|
self.lock.acquire()
|
|
with open('error.toml', 'r+t') as f:
|
|
data = toml.load('error.toml')
|
|
f.seek(0, 0)
|
|
f.truncate()
|
|
dic_name = f'err_{len(data)}'
|
|
data[dic_name] = dic
|
|
_ = toml.dump(data, f)
|
|
self.lock.release()
|
|
|
|
|
|
error_logger = ErrorLog()
|
|
|
|
class RmComicSpider(scrapy.Spider):
|
|
name = 'rm_comic'
|
|
allowed_domains = ['rm01.xyz']
|
|
main_url = 'https://rm01.xyz'
|
|
#start_urls = ['https://rm01.xyz/books/63b65185-f798-4c8f-a0b0-8811615908fd/0']
|
|
|
|
def start_requests(self):
|
|
yield scrapy.Request(self.main_url + '/books/0a7e8bd1-4cfa-481a-b067-1df663fb2017', callback=self.parse_comic)
|
|
|
|
def parse_comic(self, response):
|
|
comic = ComicItem()
|
|
comic['name'] = response.xpath('//div[@class="col"]/h5/text()').extract_first()
|
|
comic['icon'] = response.xpath('//img[@class="img-thumbnail"]/@src').extract_first()
|
|
comic['author'] = response.xpath('//div[contains(@class,"bookid_bookInfo")]/p[1]/text()').extract()[1]
|
|
comic['tags'] = response.xpath('//div[contains(@class,"bookid_bookInfo")]/p[3]/b/text()').extract_first()
|
|
comic['dep'] = response.xpath('//div[contains(@class,"bookid_bookInfo")]/p[4]/text()').extract()[1]
|
|
comic['date'] = response.xpath('//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()').extract()[1]
|
|
comic['chapters'] = response.xpath('//div[contains(@class,"bookid_chapterBox")]//div[contains(@class,"bookid_chapter")]/a/text()').extract()
|
|
comic['chapter_href'] = response.xpath('//div[contains(@class,"bookid_chapterBox")]//div[contains(@class,"bookid_chapter")]/a/@href').extract()
|
|
for link in comic['chapter_href']:
|
|
yield scrapy.Request(self.main_url+link,meta={'item' : comic} , callback=self.parse_chapter,errback=self.err)
|
|
|
|
def err(self):
|
|
print("Error=====")
|
|
|
|
def parse_chapter(self, response):
|
|
item = response.meta['item']
|
|
data = response.xpath('//script[@id="__NEXT_DATA__"]/text()').extract_first()
|
|
str_exec="props.pageProps."
|
|
comic_name = CommonUtils.parseExec(data,str_exec+"bookName")
|
|
chapterName = CommonUtils.parseExec(data,str_exec+"chapterName")
|
|
description = CommonUtils.parseExec(data,str_exec+"description")
|
|
images = CommonUtils.parseExec(data,str_exec+"images")
|
|
chapter_api_url = CommonUtils.parseExec(data,str_exec+"chapterAPIPath")
|
|
item['chapter'] = chapterName
|
|
item['list_img'] = images
|
|
if chapter_api_url != None:
|
|
yield scrapy.Request(url=self.main_url+chapter_api_url,meta={'item' : item}, callback=self.parse_chapter_api, errback=self.err)
|
|
else:
|
|
item['list_img'] = images
|
|
yield item
|
|
|
|
def parse_chapter_api(self,response,item):
|
|
data = response.meta['item']
|
|
print(item)
|
|
return response
|
|
|
|
def parse(self, response):
|
|
raise NotImplementedError |