cwx/ComicScrapy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
caiwx86 2024-02-21 13:05:18 +08:00
parent c182bbacd4
commit 9b6fb12c8d
2 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Comics/settings.py
View File

@ -26,7 +26,7 @@ ROBOTSTXT_OBEY = False
HTTPERROR_ALLOWED_CODES = [ 200 , 403] HTTPERROR_ALLOWED_CODES = [ 200 , 403]
# Configure maximum concurrent requests performed by Scrapy (default: 16) # Configure maximum concurrent requests performed by Scrapy (default: 16)
CONCURRENT_REQUESTS = 8 CONCURRENT_REQUESTS = 3
# Configure a delay for requests for the same website (default: 0) # Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
@ -45,7 +45,7 @@ RETRY_HTTP_CODES = [408, 401]
CONCURRENT_REQUESTS_PER_DOMAIN = 16 CONCURRENT_REQUESTS_PER_DOMAIN = 16
CONCURRENT_REQUESTS_PER_IP = 16 CONCURRENT_REQUESTS_PER_IP = 16
PROXY_LIST = [ PROXY_LIST = [
# "http://127.0.0.1:7890", "http://127.0.0.1:7890",
# "http://10.0.10.117:8123", # "http://10.0.10.117:8123",
] ]
# Disable cookies (enabled by default) # Disable cookies (enabled by default)

5
Comics/utils.py
View File

@ -135,7 +135,10 @@ class fileUtils:
# 提取重复并需删除的文件名 # 提取重复并需删除的文件名
diff_names = {value["name"] for value in files_size.values()} diff_names = {value["name"] for value in files_size.values()}
# 不存在则返回原文件名 # 不存在则返回原文件名
if len(diff_names) == 0: return file if len(diff_names) == 0:
if result == "size": return diff_names
else:
return file
for file_name in files_name: for file_name in files_name:
if file_name not in diff_names: if file_name not in diff_names:
logging.info(f"删除文件：{file_name}") logging.info(f"删除文件：{file_name}")