diff --git a/utils/HtmlUtils.py b/utils/HtmlUtils.py
index f17f602..cc93759 100644
--- a/utils/HtmlUtils.py
+++ b/utils/HtmlUtils.py
@@ -1,8 +1,10 @@
from fake_useragent import UserAgent
import requests
from lxml import html
-from utils.comic.ComicStr import comicStr
-import os
+import traceback
+import time
+from urllib3.util.retry import Retry
+from requests.adapters import HTTPAdapter
class htmlUtils:
headers = {'User-Agent': UserAgent().random}
@@ -14,22 +16,40 @@ class htmlUtils:
#file_url = re.sub(rstr, "_", curl)
keys = cls.url_data.keys()
url_text = cls.url_data.get(curl)
+ retries = Retry(total=3,
+ backoff_factor=0.1,
+ status_forcelist=[ 500, 502, 503, 504 ])
+ s = requests.Session()
+ s.mount('http://', HTTPAdapter(max_retries=retries))
+ s.mount('https://', HTTPAdapter(max_retries=retries))
+ count = 1
#数据为空则获取数据
- if url_text == None:
- print("请求地址:",curl)
- res = requests.get(curl, headers=cls.headers, timeout=(10,15))
- url_text = html.fromstring(res.text)
+ while url_text == None:
+ if count < 5:
+ try:
+ print("请求地址:",curl)
+ res = s.get(curl,stream=True, headers=cls.headers, timeout=20)
+ url_text = html.fromstring(res.text)
+ except:
+ print(f'Retry! 第{count}次')
+ time.sleep(3)
+ traceback.print_exc()
+ count += 1
+ continue
+ else:
+ print(f"fail 请求失败:{curl}")
+ break
data = { curl : url_text}
cls.url_data.update(data)
return url_text
@classmethod
def getBytes(cls, url):
- return requests.get(url, headers=cls.headers,stream=True,timeout=(10,15))
+ return requests.get(url, headers=cls.headers,stream=True,timeout=30)
@classmethod
def getJSON(cls,curl):
- res = requests.get(curl, headers=cls.headers,timeout=(10,15))
+ res = requests.get(curl, headers=cls.headers,timeout=30)
data_json = res.json()
return data_json
diff --git a/utils/downloader.py b/utils/downloader.py
index fa1ad66..fb454df 100644
--- a/utils/downloader.py
+++ b/utils/downloader.py
@@ -10,6 +10,7 @@ import imghdr
import os
import concurrent.futures
import requests
+import time
headers = {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
@@ -49,9 +50,12 @@ def download_image(image_url, dst_dir, file_name, timeout=20, proxy_type=None, p
if response:
response.close()
if repair_count < 5:
+ time.sleep(1)
+ print("## Repeat: {} {}".format(image_url))
download_image(image_url,dst_dir,file_name)
repair_count += 1
- print("## Fail: {} {}".format(image_url, e.args))
+ else:
+ print("## Fail: {} {}".format(image_url, e.args))
def download_images(image_urls, dst_dir, file_prefix="img", concurrency=50, timeout=20, proxy_type=None, proxy=None,filesName=None):