diff --git a/Comics/items.py b/Comics/items.py index 173f28b..6d0c271 100644 --- a/Comics/items.py +++ b/Comics/items.py @@ -57,6 +57,8 @@ class ListComicItem(Item): class ComicItem(Item): # 工程 current_project = Field() + # link + link = Field() # 编号 index = Field(output_processor=TakeFirst()) # 漫画名 diff --git a/Comics/loader.py b/Comics/loader.py index 0be9359..2ba54ec 100644 --- a/Comics/loader.py +++ b/Comics/loader.py @@ -10,7 +10,7 @@ class ComicLoader(ItemLoader): data = data.get(dot) return data - def add_xpath(self, field_name, xpath, *processors, index=None, exec=None, re=None, **kw): + def add_xpath(self, field_name, xpath, *processors, index=None, exec=None, re=None, is_null=None, **kw): """ Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a value, which is used to extract a list of strings from the @@ -33,11 +33,15 @@ class ComicLoader(ItemLoader): if exec is not None: values = self.parseExec(values, exec) if index is not None: - try: - values = values[index] - except Exception as e: - logging.error(f"values [{values}] error index [{index}]") - logging.error(e) + if len(values) - 1 >= index: + try: + values = values[index] + except Exception as e: + logging.error(f"values [{values}] error index [{index}]") + logging.error(e) + else: + if is_null is not None: values = is_null + else: values="" self.add_value(field_name, values, *processors, re=re, **kw) def add_exec(self, field_name, value, str_exec=None, *processors, re=None, **kw): diff --git a/Comics/middlewares.py b/Comics/middlewares.py index 5c015a5..0c44af0 100644 --- a/Comics/middlewares.py +++ b/Comics/middlewares.py @@ -22,8 +22,13 @@ logger = logging.getLogger(__name__) class ProxyMiddleware(object): def process_request(self, request, spider): - if len(PROXY_LIST) != 0: - request.meta["proxy"] = random.choice(PROXY_LIST) + url = request.url + logging.debug(f"proxy url=== {url} {str(url).split('.')[-1]}") + if str(url).split('.')[-1] not in HTTPCACHE_ALLOW_PREFIXS: + if len(PROXY_LIST) != 0: + request.meta["proxy"] = random.choice(PROXY_LIST) + else: + logging.info(f"skip proxy... {url}") ################################################################ # HTTP/HTTPS文件缓存中间件 diff --git a/Comics/pipelines.py b/Comics/pipelines.py index 3e2c56f..653bd7e 100644 --- a/Comics/pipelines.py +++ b/Comics/pipelines.py @@ -17,38 +17,24 @@ from Comics.utils.FileUtils import CBZUtils class ComicsPipeline: def open_spider(self, spider): pass + # item就是yield后面的对象 def process_item(self, item, spider): if isinstance(item, ComicItem): file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter']) item['count'] = len(item['images']) data = JsonExport(file=file).export_json(item, if_return=True) - data[PROJECT_KEY] = spider.name + #data[PROJECT_KEY] = spider.name return data # image解析 - def close_spider(self,spider): + def close_spider(self, spider): pass class ImgDownloadPipeline(ImagesPipeline): def get_file_path(self, item, file=None, result_type="image"): - if result_type == "image": - if os.path.sep not in file: - file = os.path.join(item[settings.PROJECT_KEY], "images", item['name'], item['chapter'], file) - elif result_type == "comic_info": - file = os.path.join(item[settings.PROJECT_KEY], "images", item['name'], item['chapter']) - elif result_type == "cbz_icon": - file = os.path.join(CBZ_EXPORT_PATH, item[settings.PROJECT_KEY], item['name'], item['chapter']+".jpg") - elif result_type == "down_icon": - file = self.download_icon(item,result_type='fullpath') - elif result_type == "icon": - file = os.path.join(item[settings.PROJECT_KEY], "icons", item['name'], item['chapter']+".jpg") - elif result_type == "cbz": - file = os.path.join(CBZ_EXPORT_PATH, item[settings.PROJECT_KEY], item['name'], item['chapter']+".CBZ") - elif result_type == "images_dir": - file = os.path.join(settings.IMAGES_STORE, item[settings.PROJECT_KEY], "images", item['name'], item['chapter']) - return file - + return ComicPath.get_file_path(item=item, file=file, result_type= result_type) + def image_scramble_exits(self, item,image_path): en_image_path = ComicPath().getFileScrambleImageSave(image_path, relative="fullpath") return os.path.exists(os.path.join(settings.IMAGES_STORE, self.get_file_path(item, en_image_path))) diff --git a/Comics/settings.py b/Comics/settings.py index 03a70ae..86d1bfd 100644 --- a/Comics/settings.py +++ b/Comics/settings.py @@ -18,6 +18,7 @@ OUTPUT_DIR = "output" # Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = 'Comics (+http://www.yourdomain.com)' USER_AGENT = UserAgent().random +# Override the default request headers: # Obey robots.txt rules ROBOTSTXT_OBEY = False diff --git a/Comics/spiders/rm_comic.py b/Comics/spiders/rm_comic.py index 2f10ad7..0494f2a 100644 --- a/Comics/spiders/rm_comic.py +++ b/Comics/spiders/rm_comic.py @@ -1,32 +1,34 @@ -import scrapy,logging,time +import scrapy,logging,time,os from Comics.items import ComicItem from Comics.loader import ComicLoader from Comics.items import ListComicItem +from Comics.utils.Constant import ComicPath +from Comics.settings import PROJECT_KEY class RmComicSpider(scrapy.Spider): name = 'rm_comic' - allowed_domains = ['rm01.xyz'] - main_url = 'https://rm01.xyz' - start_urls = 'https://rm01.xyz/books' + allowed_domains = ['rouman5.com'] + main_url = 'https://'+allowed_domains[0] + start_urls = main_url+'/books' def start_requests(self): yield scrapy.Request(self.start_urls, callback=self.books_comic) def books_comic(self, response): - books_comic = ComicLoader(item=ListComicItem(), response=response) + #books_comic = ComicLoader(item=ListComicItem(), response=response) + books_comic = ComicLoader(item=ComicItem(), response=response) data = books_comic.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0] str_exec = "props.pageProps.books" books = books_comic.get_exec(data, str_exec=str_exec) for book in books: - books_comic.add_value('link', book['id']) - time.sleep(3) - yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic) - + books_comic.add_value('link', self.start_urls+"/"+book['id']) + yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic) # 获取某个漫画的相关数据 # 获取到多个章节链接后进入下个流程 def parse_comic(self, response): comic_item = ComicLoader(item=ComicItem(), response=response) + comic_item.add_value(PROJECT_KEY, self.name) comic_item.add_xpath('name', '//div[@class="col"]/h5/text()') comic_item.add_xpath('icon', '//img[@class="img-thumbnail"]/@src') comic_item.add_xpath('author', '//div[contains(@class,"bookid_bookInfo")]/p[1]/text()', index=1) @@ -39,9 +41,21 @@ class RmComicSpider(scrapy.Spider): '//div[contains(@class,"bookid_chapter")]/a/@href') #chapters = response.xpath('//div[contains(@class,"bookid_chapterBox")]' # '//div[contains(@class,"bookid_chapter")]/a/text()').extract() + + chapters = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]' + '//div[contains(@class,"bookid_chapter")]/a/text()') #for chapter, link in zip(chapters, chapter_href): for i, link in enumerate(chapter_href, start=1): - yield scrapy.Request(self.main_url+link, meta={'item': comic_item.load_item(), 'num': i}, callback=self.parse_chapter) + # yield scrapy.Request(self.main_url+link, meta={'item': comic_item.load_item(), 'num': i}, callback=self.parse_chapter) + comic_item.add_value('chapter', chapters[i-1]) + item = comic_item.load_item() + cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True) + if os.path.exists(cbz_path): + logging.info(f"漫画 {cbz_path} 已存在, 跳过中...") + return + else: + yield scrapy.Request(self.main_url+link, meta={'item': item, 'num': i}, callback=self.parse_chapter) + # 读取某章节下的所有图片 def parse_chapter(self, response): @@ -71,4 +85,7 @@ class RmComicSpider(scrapy.Spider): def parse(self, response): + raise NotImplementedError + + def error_parse(self, response): raise NotImplementedError \ No newline at end of file diff --git a/Comics/utils/Constant.py b/Comics/utils/Constant.py index 18d0489..abf447e 100644 --- a/Comics/utils/Constant.py +++ b/Comics/utils/Constant.py @@ -1,5 +1,6 @@ import os.path import re +from Comics import settings from opencc import OpenCC class ComicPath: PREFIX_SCRAMBLE = "scramble=" @@ -45,4 +46,31 @@ class ComicPath: else: filename = filename[0:len(filename)+1-count] break - return filename \ No newline at end of file + return filename + + @classmethod + def get_file_path(cls, item, result_type="image", file=None, convert=False): + PROJECT = item[settings.PROJECT_KEY][0] + if not convert: + name = item['name'] + chapter = item['chapter'] + else: + name = cls.fix_file_name(cls.chinese_convert(item['name'])) + chapter = cls.fix_file_name(cls.chinese_convert(item['chapter'])) + + if result_type == "image": + if os.path.sep not in file: + file = os.path.join(PROJECT, "images", name, chapter, file) + elif result_type == "comic_info": + file = os.path.join(PROJECT, "images", name, chapter) + elif result_type == "cbz_icon": + file = os.path.join(settings.CBZ_EXPORT_PATH, PROJECT, name, chapter+".jpg") + elif result_type == "down_icon": + file = os.path.join(settings.IMAGES_STORE, cls.get_file_path(item=item, result_type="icon")) + elif result_type == "icon": + file = os.path.join(PROJECT, "icons", name+".jpg") + elif result_type == "cbz": + file = os.path.join(settings.CBZ_EXPORT_PATH, PROJECT, name, chapter+".CBZ") + elif result_type == "images_dir": + file = os.path.join(settings.IMAGES_STORE, PROJECT, "images", name, chapter) + return file \ No newline at end of file