fix download

utils/HtmlUtils.py

@@ -1,8 +1,10 @@
 from fake_useragent import UserAgent
 import requests
 from lxml import html
-from utils.comic.ComicStr import comicStr
+import traceback
-import os
+import time
+from urllib3.util.retry import Retry
+from requests.adapters import HTTPAdapter
 
 class htmlUtils:
     headers = {'User-Agent': UserAgent().random}
@@ -14,22 +16,40 @@ class htmlUtils:
         #file_url = re.sub(rstr, "_", curl)
         keys = cls.url_data.keys()
         url_text = cls.url_data.get(curl)
+        retries = Retry(total=3,
+                        backoff_factor=0.1,
+                        status_forcelist=[ 500, 502, 503, 504 ])
+        s = requests.Session()
+        s.mount('http://', HTTPAdapter(max_retries=retries))
+        s.mount('https://', HTTPAdapter(max_retries=retries))
+        count = 1
         #数据为空则获取数据
-        if url_text == None:
+        while url_text == None:
-            print("请求地址：",curl)
+            if count < 5:
-            res = requests.get(curl, headers=cls.headers, timeout=(10,15))
+                try:
-            url_text = html.fromstring(res.text)
+                    print("请求地址：",curl)
+                    res = s.get(curl,stream=True, headers=cls.headers, timeout=20)
+                    url_text = html.fromstring(res.text)
+                except:
+                    print(f'Retry! 第{count}次')
+                    time.sleep(3)
+                    traceback.print_exc()
+                    count += 1
+                    continue 
+            else:
+                print(f"fail 请求失败：{curl}")
+                break
         data = { curl : url_text}
         cls.url_data.update(data)
         return url_text
     
     @classmethod
     def getBytes(cls, url):
-        return requests.get(url, headers=cls.headers,stream=True,timeout=(10,15))
+        return requests.get(url, headers=cls.headers,stream=True,timeout=30)
      
     @classmethod
     def getJSON(cls,curl):
-        res = requests.get(curl, headers=cls.headers,timeout=(10,15))
+        res = requests.get(curl, headers=cls.headers,timeout=30)
         data_json = res.json()
         return data_json
      

utils/downloader.py

@@ -10,6 +10,7 @@ import imghdr
 import os
 import concurrent.futures
 import requests
+import time
 
 headers = {
     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
@@ -49,9 +50,12 @@ def download_image(image_url, dst_dir, file_name, timeout=20, proxy_type=None, p
         if response:
            response.close()
         if repair_count < 5:
+            time.sleep(1)
+            print("## Repeat: {}  {}".format(image_url))
             download_image(image_url,dst_dir,file_name)
             repair_count += 1
-        print("## Fail:  {}  {}".format(image_url, e.args))
+        else:
+            print("## Fail:  {}  {}".format(image_url, e.args))
 
 
 def download_images(image_urls, dst_dir, file_prefix="img", concurrency=50, timeout=20, proxy_type=None, proxy=None,filesName=None):