cwx/ComicScrapy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
caiwx86 2024-10-29 00:13:42 +08:00
parent d5ee8173e4
commit eb19dd5cf9
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Comics/settings.py
View File

@ -24,7 +24,7 @@ USER_AGENT = UserAgent().random
# Obey robots.txt rules # Obey robots.txt rules
ROBOTSTXT_OBEY = False ROBOTSTXT_OBEY = False
HTTPERROR_ALLOWED_CODES = [ 200 , 403] HTTPERROR_ALLOWED_CODES = [ 200 , 403, 301, 302]
# Configure maximum concurrent requests performed by Scrapy (default: 16) # Configure maximum concurrent requests performed by Scrapy (default: 16)
CONCURRENT_REQUESTS = 16 CONCURRENT_REQUESTS = 16
@ -39,6 +39,7 @@ COMIC_INFO_XML_STORE = IMAGES_STORE
# 随机化下载延迟 # 随机化下载延迟
# RANDOMIZE_DOWNLOAD_DELAY = True # RANDOMIZE_DOWNLOAD_DELAY = True
#重试 #重试
DOWNLOAD_DELAY = 1 # 每次请求间隔2秒
# 增加下载超时时间 # 增加下载超时时间
DOWNLOAD_TIMEOUT = 30 # 30秒 DOWNLOAD_TIMEOUT = 30 # 30秒
# 增加重试之间的延迟 # 增加重试之间的延迟