diff --git a/Comics/items.py b/Comics/items.py index 6ac3e90..d06bec1 100644 --- a/Comics/items.py +++ b/Comics/items.py @@ -59,6 +59,8 @@ class ComicItem(Item): current_project = Field() # link link = Field() + # 全部章节 + chapters = Field() # 编号 index = Field(output_processor=TakeFirst()) # 漫画名 diff --git a/Comics/pipelines.py b/Comics/pipelines.py index 2156388..065494f 100644 --- a/Comics/pipelines.py +++ b/Comics/pipelines.py @@ -23,6 +23,7 @@ class ComicsPipeline: if isinstance(item, ComicItem): file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter']) item['count'] = len(item['images']) + item['index'] = item['chapters'].index(item['chapter']) + 1 data = JsonExport(file=file).export_json(item, if_return=True) #data[PROJECT_KEY] = spider.name return data diff --git a/Comics/settings.py b/Comics/settings.py index f8e5a59..78d30ba 100644 --- a/Comics/settings.py +++ b/Comics/settings.py @@ -14,7 +14,7 @@ BOT_NAME = 'Comics' SPIDER_MODULES = ['Comics.spiders'] NEWSPIDER_MODULE = 'Comics.spiders' -BASE_OUTPUT = "/mnt/Comics" +BASE_OUTPUT = "" OUTPUT_DIR = os.path.join(BASE_OUTPUT, "output") # Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = 'Comics (+http://www.yourdomain.com)' diff --git a/Comics/spiders/rm_comic.py b/Comics/spiders/rm_comic.py index 878a021..0e0402a 100644 --- a/Comics/spiders/rm_comic.py +++ b/Comics/spiders/rm_comic.py @@ -23,7 +23,7 @@ class RmComicSpider(scrapy.Spider): for book in books: books_comic.add_value('link', self.start_urls+"/"+book['id']) yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic) - + # 获取某个漫画的相关数据 # 获取到多个章节链接后进入下个流程 def parse_comic(self, response): @@ -44,17 +44,18 @@ class RmComicSpider(scrapy.Spider): chapters = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]' '//div[contains(@class,"bookid_chapter")]/a/text()') - #for chapter, link in zip(chapters, chapter_href): - for i, link in enumerate(chapter_href, start=1): + for chapter, link in zip(chapters, chapter_href): + #for i, link in enumerate(chapter_href, start=1): # yield scrapy.Request(self.main_url+link, meta={'item': comic_item.load_item(), 'num': i}, callback=self.parse_chapter) - comic_item.add_value('chapter', chapters[i-1]) + comic_item.add_value('chapter', chapter) + comic_item.add_value('chapters', chapters) item = comic_item.load_item() cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True) if os.path.exists(cbz_path): logging.info(f"漫画 {cbz_path} 已存在, 跳过中...") return else: - yield scrapy.Request(self.main_url+link, meta={'item': item, 'num': i}, callback=self.parse_chapter) + yield scrapy.Request(self.main_url+link, meta={'item': item}, callback=self.parse_chapter) # 读取某章节下的所有图片 @@ -64,7 +65,6 @@ class RmComicSpider(scrapy.Spider): str_exec = "props.pageProps." comic_item.add_exec('name', data, str_exec=str_exec+"bookName") comic_item.add_exec('dep', data, str_exec=str_exec+"description") - comic_item.add_value('index', response.meta['num']) comic_item.add_exec('chapter', data, str_exec=str_exec + "chapterName") comic_item.add_exec('image_urls', data, str_exec+"images") comic_item.add_exec('images', data, str_exec+"images")