cwx/ComicScrapy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
caiwx86 2024-07-15 05:10:42 +08:00
parent cf5bde7e08
commit d6cb8e4251
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Comics/middlewares.py
View File

@ -24,7 +24,8 @@ class ProxyMiddleware(object):
def process_request(self, request, spider): def process_request(self, request, spider):
url = request.url url = request.url
logging.debug(f"proxy url=== {url} {str(url).split('.')[-1]}") logging.debug(f"proxy url=== {url} {str(url).split('.')[-1]}")
if str(url).split('.')[-1] not in HTTPCACHE_ALLOW_PREFIXS or str(url).replace("https://", "").replace("http://", "").split("/") in HTTPCACHE_PROXY_DOMAINS: domain = str(url).replace("https://", "").replace("http://", "").split("/")[0]
if str(url).split('.')[-1] not in HTTPCACHE_ALLOW_PREFIXS or domain in HTTPCACHE_PROXY_DOMAINS:
if len(PROXY_LIST) != 0: if len(PROXY_LIST) != 0:
request.meta["proxy"] = random.choice(PROXY_LIST) request.meta["proxy"] = random.choice(PROXY_LIST)
else: else: