update 20230806
This commit is contained in:
parent
40d484ee4c
commit
aeff24daa4
@ -57,6 +57,8 @@ class ListComicItem(Item):
|
||||
class ComicItem(Item):
|
||||
# 工程
|
||||
current_project = Field()
|
||||
# link
|
||||
link = Field()
|
||||
# 编号
|
||||
index = Field(output_processor=TakeFirst())
|
||||
# 漫画名
|
||||
|
||||
@ -10,7 +10,7 @@ class ComicLoader(ItemLoader):
|
||||
data = data.get(dot)
|
||||
return data
|
||||
|
||||
def add_xpath(self, field_name, xpath, *processors, index=None, exec=None, re=None, **kw):
|
||||
def add_xpath(self, field_name, xpath, *processors, index=None, exec=None, re=None, is_null=None, **kw):
|
||||
"""
|
||||
Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a
|
||||
value, which is used to extract a list of strings from the
|
||||
@ -33,11 +33,15 @@ class ComicLoader(ItemLoader):
|
||||
if exec is not None:
|
||||
values = self.parseExec(values, exec)
|
||||
if index is not None:
|
||||
try:
|
||||
values = values[index]
|
||||
except Exception as e:
|
||||
logging.error(f"values [{values}] error index [{index}]")
|
||||
logging.error(e)
|
||||
if len(values) - 1 >= index:
|
||||
try:
|
||||
values = values[index]
|
||||
except Exception as e:
|
||||
logging.error(f"values [{values}] error index [{index}]")
|
||||
logging.error(e)
|
||||
else:
|
||||
if is_null is not None: values = is_null
|
||||
else: values=""
|
||||
self.add_value(field_name, values, *processors, re=re, **kw)
|
||||
|
||||
def add_exec(self, field_name, value, str_exec=None, *processors, re=None, **kw):
|
||||
|
||||
@ -22,8 +22,13 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class ProxyMiddleware(object):
|
||||
def process_request(self, request, spider):
|
||||
if len(PROXY_LIST) != 0:
|
||||
request.meta["proxy"] = random.choice(PROXY_LIST)
|
||||
url = request.url
|
||||
logging.debug(f"proxy url=== {url} {str(url).split('.')[-1]}")
|
||||
if str(url).split('.')[-1] not in HTTPCACHE_ALLOW_PREFIXS:
|
||||
if len(PROXY_LIST) != 0:
|
||||
request.meta["proxy"] = random.choice(PROXY_LIST)
|
||||
else:
|
||||
logging.info(f"skip proxy... {url}")
|
||||
|
||||
################################################################
|
||||
# HTTP/HTTPS文件缓存中间件
|
||||
|
||||
@ -17,37 +17,23 @@ from Comics.utils.FileUtils import CBZUtils
|
||||
class ComicsPipeline:
|
||||
def open_spider(self, spider):
|
||||
pass
|
||||
|
||||
# item就是yield后面的对象
|
||||
def process_item(self, item, spider):
|
||||
if isinstance(item, ComicItem):
|
||||
file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter'])
|
||||
item['count'] = len(item['images'])
|
||||
data = JsonExport(file=file).export_json(item, if_return=True)
|
||||
data[PROJECT_KEY] = spider.name
|
||||
#data[PROJECT_KEY] = spider.name
|
||||
return data
|
||||
# image解析
|
||||
|
||||
def close_spider(self,spider):
|
||||
def close_spider(self, spider):
|
||||
pass
|
||||
|
||||
class ImgDownloadPipeline(ImagesPipeline):
|
||||
def get_file_path(self, item, file=None, result_type="image"):
|
||||
if result_type == "image":
|
||||
if os.path.sep not in file:
|
||||
file = os.path.join(item[settings.PROJECT_KEY], "images", item['name'], item['chapter'], file)
|
||||
elif result_type == "comic_info":
|
||||
file = os.path.join(item[settings.PROJECT_KEY], "images", item['name'], item['chapter'])
|
||||
elif result_type == "cbz_icon":
|
||||
file = os.path.join(CBZ_EXPORT_PATH, item[settings.PROJECT_KEY], item['name'], item['chapter']+".jpg")
|
||||
elif result_type == "down_icon":
|
||||
file = self.download_icon(item,result_type='fullpath')
|
||||
elif result_type == "icon":
|
||||
file = os.path.join(item[settings.PROJECT_KEY], "icons", item['name'], item['chapter']+".jpg")
|
||||
elif result_type == "cbz":
|
||||
file = os.path.join(CBZ_EXPORT_PATH, item[settings.PROJECT_KEY], item['name'], item['chapter']+".CBZ")
|
||||
elif result_type == "images_dir":
|
||||
file = os.path.join(settings.IMAGES_STORE, item[settings.PROJECT_KEY], "images", item['name'], item['chapter'])
|
||||
return file
|
||||
return ComicPath.get_file_path(item=item, file=file, result_type= result_type)
|
||||
|
||||
def image_scramble_exits(self, item,image_path):
|
||||
en_image_path = ComicPath().getFileScrambleImageSave(image_path, relative="fullpath")
|
||||
|
||||
@ -18,6 +18,7 @@ OUTPUT_DIR = "output"
|
||||
# Crawl responsibly by identifying yourself (and your website) on the user-agent
|
||||
#USER_AGENT = 'Comics (+http://www.yourdomain.com)'
|
||||
USER_AGENT = UserAgent().random
|
||||
# Override the default request headers:
|
||||
# Obey robots.txt rules
|
||||
ROBOTSTXT_OBEY = False
|
||||
|
||||
|
||||
@ -1,32 +1,34 @@
|
||||
import scrapy,logging,time
|
||||
import scrapy,logging,time,os
|
||||
from Comics.items import ComicItem
|
||||
from Comics.loader import ComicLoader
|
||||
from Comics.items import ListComicItem
|
||||
from Comics.utils.Constant import ComicPath
|
||||
from Comics.settings import PROJECT_KEY
|
||||
|
||||
class RmComicSpider(scrapy.Spider):
|
||||
name = 'rm_comic'
|
||||
allowed_domains = ['rm01.xyz']
|
||||
main_url = 'https://rm01.xyz'
|
||||
start_urls = 'https://rm01.xyz/books'
|
||||
allowed_domains = ['rouman5.com']
|
||||
main_url = 'https://'+allowed_domains[0]
|
||||
start_urls = main_url+'/books'
|
||||
|
||||
def start_requests(self):
|
||||
yield scrapy.Request(self.start_urls, callback=self.books_comic)
|
||||
|
||||
def books_comic(self, response):
|
||||
books_comic = ComicLoader(item=ListComicItem(), response=response)
|
||||
#books_comic = ComicLoader(item=ListComicItem(), response=response)
|
||||
books_comic = ComicLoader(item=ComicItem(), response=response)
|
||||
data = books_comic.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
|
||||
str_exec = "props.pageProps.books"
|
||||
books = books_comic.get_exec(data, str_exec=str_exec)
|
||||
for book in books:
|
||||
books_comic.add_value('link', book['id'])
|
||||
time.sleep(3)
|
||||
books_comic.add_value('link', self.start_urls+"/"+book['id'])
|
||||
yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic)
|
||||
|
||||
|
||||
# 获取某个漫画的相关数据
|
||||
# 获取到多个章节链接后进入下个流程
|
||||
def parse_comic(self, response):
|
||||
comic_item = ComicLoader(item=ComicItem(), response=response)
|
||||
comic_item.add_value(PROJECT_KEY, self.name)
|
||||
comic_item.add_xpath('name', '//div[@class="col"]/h5/text()')
|
||||
comic_item.add_xpath('icon', '//img[@class="img-thumbnail"]/@src')
|
||||
comic_item.add_xpath('author', '//div[contains(@class,"bookid_bookInfo")]/p[1]/text()', index=1)
|
||||
@ -39,9 +41,21 @@ class RmComicSpider(scrapy.Spider):
|
||||
'//div[contains(@class,"bookid_chapter")]/a/@href')
|
||||
#chapters = response.xpath('//div[contains(@class,"bookid_chapterBox")]'
|
||||
# '//div[contains(@class,"bookid_chapter")]/a/text()').extract()
|
||||
|
||||
chapters = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]'
|
||||
'//div[contains(@class,"bookid_chapter")]/a/text()')
|
||||
#for chapter, link in zip(chapters, chapter_href):
|
||||
for i, link in enumerate(chapter_href, start=1):
|
||||
yield scrapy.Request(self.main_url+link, meta={'item': comic_item.load_item(), 'num': i}, callback=self.parse_chapter)
|
||||
# yield scrapy.Request(self.main_url+link, meta={'item': comic_item.load_item(), 'num': i}, callback=self.parse_chapter)
|
||||
comic_item.add_value('chapter', chapters[i-1])
|
||||
item = comic_item.load_item()
|
||||
cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True)
|
||||
if os.path.exists(cbz_path):
|
||||
logging.info(f"漫画 {cbz_path} 已存在, 跳过中...")
|
||||
return
|
||||
else:
|
||||
yield scrapy.Request(self.main_url+link, meta={'item': item, 'num': i}, callback=self.parse_chapter)
|
||||
|
||||
|
||||
# 读取某章节下的所有图片
|
||||
def parse_chapter(self, response):
|
||||
@ -72,3 +86,6 @@ class RmComicSpider(scrapy.Spider):
|
||||
|
||||
def parse(self, response):
|
||||
raise NotImplementedError
|
||||
|
||||
def error_parse(self, response):
|
||||
raise NotImplementedError
|
||||
@ -1,5 +1,6 @@
|
||||
import os.path
|
||||
import re
|
||||
from Comics import settings
|
||||
from opencc import OpenCC
|
||||
class ComicPath:
|
||||
PREFIX_SCRAMBLE = "scramble="
|
||||
@ -46,3 +47,30 @@ class ComicPath:
|
||||
filename = filename[0:len(filename)+1-count]
|
||||
break
|
||||
return filename
|
||||
|
||||
@classmethod
|
||||
def get_file_path(cls, item, result_type="image", file=None, convert=False):
|
||||
PROJECT = item[settings.PROJECT_KEY][0]
|
||||
if not convert:
|
||||
name = item['name']
|
||||
chapter = item['chapter']
|
||||
else:
|
||||
name = cls.fix_file_name(cls.chinese_convert(item['name']))
|
||||
chapter = cls.fix_file_name(cls.chinese_convert(item['chapter']))
|
||||
|
||||
if result_type == "image":
|
||||
if os.path.sep not in file:
|
||||
file = os.path.join(PROJECT, "images", name, chapter, file)
|
||||
elif result_type == "comic_info":
|
||||
file = os.path.join(PROJECT, "images", name, chapter)
|
||||
elif result_type == "cbz_icon":
|
||||
file = os.path.join(settings.CBZ_EXPORT_PATH, PROJECT, name, chapter+".jpg")
|
||||
elif result_type == "down_icon":
|
||||
file = os.path.join(settings.IMAGES_STORE, cls.get_file_path(item=item, result_type="icon"))
|
||||
elif result_type == "icon":
|
||||
file = os.path.join(PROJECT, "icons", name+".jpg")
|
||||
elif result_type == "cbz":
|
||||
file = os.path.join(settings.CBZ_EXPORT_PATH, PROJECT, name, chapter+".CBZ")
|
||||
elif result_type == "images_dir":
|
||||
file = os.path.join(settings.IMAGES_STORE, PROJECT, "images", name, chapter)
|
||||
return file
|
||||
Loading…
Reference in New Issue
Block a user