add download queue

utils/base/BaseComicEntity.py

@@ -124,11 +124,11 @@ class baseComic:
         #不存在ComicInfo.xml则生成
         if is_next and fu.notExists(ci.getPathComicInfoXML()): ci.writeComicInfoXML(chapter_name)    
         repeat = 1 
-        while is_next and repeat <= 10 and not is_old:
+        #while is_next and repeat <= 10 and not is_old:
-            ntfy.sendMsg(f"{book_name} {chapter_name} 下载中")
+        #    ntfy.sendMsg(f"{book_name} {chapter_name} 下载中")
-            download_images(list_img,ci.getDirComicChapter(), files_name=files_name,concurrency=None,timeout=8)
+        download_images(list_img,ci.getDirComicChapter(), files_name=files_name,concurrency=None,timeout=8)
-            ntfy.sendMsg("等待数据检验中...",sleep=0.5)
+        #    ntfy.sendMsg("等待数据检验中...",sleep=0.5)
-            is_next = fu.equImages(ci.getDirComicChapter(),list_img)
+        is_next = fu.equImages(ci.getDirComicChapter(),list_img)
-            if not is_next: ntfy.sendMsg(msg=f"下载数据(不完整，{int(repeat*2)}秒钟后尝试第{repeat}次",sleep=int(repeat*2))
+        #    if not is_next: ntfy.sendMsg(msg=f"下载数据(不完整，{int(repeat*2)}秒钟后尝试第{repeat}次",sleep=int(repeat*2))
-            repeat += 1
+        #    repeat += 1
         return is_next

utils/downloader.py

@@ -5,6 +5,7 @@
 
 from __future__ import print_function
 
+from queue import Queue
 import shutil
 import imghdr
 import os
@@ -25,15 +26,14 @@ headers = {
     # 'Connection': 'close',
 }
 
+down_queue = Queue()
 
-def download_image(image_url, dst_dir, file_name, timeout=20, proxy_type=None, proxy=None,type="image"):
+def common_download(file_name,image_url,dst_dir,timeout=10,proxy=None,proxy_type=None):
     proxies = None
     if proxy_type is not None:
         proxies = {
             "http": proxy_type + "://" + proxy,
-            "https": proxy_type + "://" + proxy
+            "https": proxy_type + "://" + proxy }
-        }
-
     response = None
     file_path = os.path.join(dst_dir, file_name)
     if os.path.exists(file_path):
@@ -41,31 +41,38 @@ def download_image(image_url, dst_dir, file_name, timeout=20, proxy_type=None, p
         return None
     temp_path = os.path.join(dst_dir, file_name+".downloads")
     repair_count = 1
-    try:
+    response = requests.get(
-        response = requests.get(
             image_url, headers=headers, timeout=timeout, proxies=proxies)
-        while response.status_code != 200 and repair_count <= 5:
+    while response.status_code != 200 and repair_count <= 5:
-            time.sleep(0.7)
+        time.sleep(0.7)
-            download_image(image_url,dst_dir,file_name)
+        download_image(image_url,dst_dir,file_name)
-            ntfy.sendMsg(f'重试：第{repair_count}次 {image_url}')
+        ntfy.sendMsg(f'重试：第{repair_count}次 {image_url}')
-            repair_count += 1
+        repair_count += 1
-        with open(temp_path, 'wb') as f:
+    with open(temp_path, 'wb') as f:
-            f.write(response.content)
+        f.write(response.content)
-        response.close()
+    response.close()
-        #验证是否是图像
+    #验证是否是图像
-        if fu.ver_file(temp_path,type=type):
+    if fu.ver_file(temp_path,type="image"):
-            shutil.move(temp_path, file_path)
+        shutil.move(temp_path, file_path)
-            print("## OK:  {}  {}".format(file_path, image_url))
+        print("## OK:  {}  {}".format(file_path, image_url))
-    except Exception as e:
+    else:
-        if response:
+        print("## Fail:  {}  {}".format(image_url, "图像损坏"))
-           response.close()
+        down_queue.put([file_name,image_url,dst_dir])
-        if repair_count < 5:
+
-            time.sleep(1)
+def download_image(timeout=20, proxy_type=None, proxy=None,type="image"):
-            ntfy.sendMsg("## Repeat: {}  {}".format(image_url))
+    repeat = 0
-            download_image(image_url,dst_dir,file_name)
+    while not down_queue.empty() and repeat <= 10:
-            repair_count += 1
+        repeat += 1
-        else:
+        if repeat > 1:
-            print("## Fail:  {}  {}".format(image_url, e.args))
+            ntfy.sendMsg(f"第{repeat}次下载数据中...")
+        data = down_queue.get(False)
+        (file_name,image_url,dst_dir) = [data[0],data[1],data[2]]
+        try:
+            common_download(file_name,image_url,dst_dir)
+        except:
+            ntfy.sendMsg(f"下载重试中 {file_name}={image_url}")
+            down_queue.put([file_name,image_url,dst_dir])
+
 
 
 def download_images(image_urls, dst_dir,concurrency=None,timeout=20,proxy_type=None, proxy=None,files_name=None):
@@ -89,8 +96,9 @@ def download_images(image_urls, dst_dir,concurrency=None,timeout=20,proxy_type=N
             os.makedirs(dst_dir)
         for image_url in image_urls:
             file_name = files_name[count]
+            down_queue.put([file_name,image_url,dst_dir])
             future_list.append(executor.submit(
-                download_image, image_url, dst_dir, file_name, timeout, proxy_type, proxy))
+                download_image,timeout, proxy_type, proxy))
             count += 1
         concurrent.futures.wait(future_list, timeout)