cwx/ComicScrapy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
caiwx86 2024-02-21 19:01:06 +08:00
parent 9b6fb12c8d
commit 9b3ac8878f
2 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Comics/settings.py
View File

@ -45,7 +45,7 @@ RETRY_HTTP_CODES = [408, 401]
CONCURRENT_REQUESTS_PER_DOMAIN = 16 CONCURRENT_REQUESTS_PER_DOMAIN = 16
CONCURRENT_REQUESTS_PER_IP = 16 CONCURRENT_REQUESTS_PER_IP = 16
PROXY_LIST = [ PROXY_LIST = [
"http://127.0.0.1:7890", # "http://127.0.0.1:7890",
# "http://10.0.10.117:8123", # "http://10.0.10.117:8123",
] ]
# Disable cookies (enabled by default) # Disable cookies (enabled by default)

5
Comics/spiders/rm_comic.py
View File

@ -11,7 +11,7 @@ class RmComicSpider(scrapy.Spider):
main_url = 'https://'+allowed_domains[0] main_url = 'https://'+allowed_domains[0]
start_urls = main_url+'/books' start_urls = main_url+'/books'
# 遍历网站页数 # 遍历网站页数数据
def start_requests(self): def start_requests(self):
for x in range(0,60): for x in range(0,60):
yield scrapy.Request(self.start_urls+"?&page="+str(x), callback=self.books_comic) yield scrapy.Request(self.start_urls+"?&page="+str(x), callback=self.books_comic)
@ -30,8 +30,7 @@ class RmComicSpider(scrapy.Spider):
for chapter, link in zip(comic_item.get_chapters(), comic_item.get_chapter_href()): for chapter, link in zip(comic_item.get_chapters(), comic_item.get_chapter_href()):
item = comic_item.load_item() item = comic_item.load_item()
cbz_path = ComicPath().get_file_path(item=item, result_type="cbz", convert=True, chapter=chapter) cbz_path = ComicPath().get_file_path(item=item, result_type="cbz", convert=True, chapter=chapter)
#if not checkUtils().is_error(item) and os.path.exists(cbz_path): if not checkUtils().is_error(item) and os.path.exists(cbz_path):
if os.path.exists(cbz_path):
logging.info(f"漫画 {cbz_path} 已存在, 跳过中...") logging.info(f"漫画 {cbz_path} 已存在, 跳过中...")
yield item yield item
else: else: