cwx/ComicScrapy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
caiwx86 2023-08-06 22:19:30 +08:00
parent 57c62c88c0
commit 312bdad626
4 changed files with 10 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Comics/items.py
View File

@ -59,6 +59,8 @@ class ComicItem(Item):
current_project = Field() current_project = Field()
# link # link
link = Field() link = Field()
# 全部章节
chapters = Field()
# 编号 # 编号
index = Field(output_processor=TakeFirst()) index = Field(output_processor=TakeFirst())
# 漫画名 # 漫画名

1
Comics/pipelines.py
View File

@ -23,6 +23,7 @@ class ComicsPipeline:
if isinstance(item, ComicItem): if isinstance(item, ComicItem):
file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter']) file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter'])
item['count'] = len(item['images']) item['count'] = len(item['images'])
item['index'] = item['chapters'].index(item['chapter']) + 1
data = JsonExport(file=file).export_json(item, if_return=True) data = JsonExport(file=file).export_json(item, if_return=True)
#data[PROJECT_KEY] = spider.name #data[PROJECT_KEY] = spider.name
return data return data

2
Comics/settings.py
View File

@ -14,7 +14,7 @@ BOT_NAME = 'Comics'
SPIDER_MODULES = ['Comics.spiders'] SPIDER_MODULES = ['Comics.spiders']
NEWSPIDER_MODULE = 'Comics.spiders' NEWSPIDER_MODULE = 'Comics.spiders'
BASE_OUTPUT = "/mnt/Comics" BASE_OUTPUT = ""
OUTPUT_DIR = os.path.join(BASE_OUTPUT, "output") OUTPUT_DIR = os.path.join(BASE_OUTPUT, "output")
# Crawl responsibly by identifying yourself (and your website) on the user-agent # Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'Comics (+http://www.yourdomain.com)' #USER_AGENT = 'Comics (+http://www.yourdomain.com)'

12
Comics/spiders/rm_comic.py
View File

@ -23,7 +23,7 @@ class RmComicSpider(scrapy.Spider):
for book in books: for book in books:
books_comic.add_value('link', self.start_urls+"/"+book['id']) books_comic.add_value('link', self.start_urls+"/"+book['id'])
yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic) yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic)
# 获取某个漫画的相关数据 # 获取某个漫画的相关数据
# 获取到多个章节链接后进入下个流程 # 获取到多个章节链接后进入下个流程
def parse_comic(self, response): def parse_comic(self, response):
@ -44,17 +44,18 @@ class RmComicSpider(scrapy.Spider):
chapters = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]' chapters = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]'
'//div[contains(@class,"bookid_chapter")]/a/text()') '//div[contains(@class,"bookid_chapter")]/a/text()')
#for chapter, link in zip(chapters, chapter_href): for chapter, link in zip(chapters, chapter_href):
for i, link in enumerate(chapter_href, start=1): #for i, link in enumerate(chapter_href, start=1):
# yield scrapy.Request(self.main_url+link, meta={'item': comic_item.load_item(), 'num': i}, callback=self.parse_chapter) # yield scrapy.Request(self.main_url+link, meta={'item': comic_item.load_item(), 'num': i}, callback=self.parse_chapter)
comic_item.add_value('chapter', chapters[i-1]) comic_item.add_value('chapter', chapter)
comic_item.add_value('chapters', chapters)
item = comic_item.load_item() item = comic_item.load_item()
cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True) cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True)
if os.path.exists(cbz_path): if os.path.exists(cbz_path):
logging.info(f"漫画 {cbz_path} 已存在, 跳过中...") logging.info(f"漫画 {cbz_path} 已存在, 跳过中...")
return return
else: else:
yield scrapy.Request(self.main_url+link, meta={'item': item, 'num': i}, callback=self.parse_chapter) yield scrapy.Request(self.main_url+link, meta={'item': item}, callback=self.parse_chapter)
# 读取某章节下的所有图片 # 读取某章节下的所有图片
@ -64,7 +65,6 @@ class RmComicSpider(scrapy.Spider):
str_exec = "props.pageProps." str_exec = "props.pageProps."
comic_item.add_exec('name', data, str_exec=str_exec+"bookName") comic_item.add_exec('name', data, str_exec=str_exec+"bookName")
comic_item.add_exec('dep', data, str_exec=str_exec+"description") comic_item.add_exec('dep', data, str_exec=str_exec+"description")
comic_item.add_value('index', response.meta['num'])
comic_item.add_exec('chapter', data, str_exec=str_exec + "chapterName") comic_item.add_exec('chapter', data, str_exec=str_exec + "chapterName")
comic_item.add_exec('image_urls', data, str_exec+"images") comic_item.add_exec('image_urls', data, str_exec+"images")
comic_item.add_exec('images', data, str_exec+"images") comic_item.add_exec('images', data, str_exec+"images")