diff --git a/utils/base/BaseComicEntity.py b/utils/base/BaseComicEntity.py index 8f71398..9176848 100644 --- a/utils/base/BaseComicEntity.py +++ b/utils/base/BaseComicEntity.py @@ -124,11 +124,11 @@ class baseComic: #不存在ComicInfo.xml则生成 if is_next and fu.notExists(ci.getPathComicInfoXML()): ci.writeComicInfoXML(chapter_name) repeat = 1 - while is_next and repeat <= 10 and not is_old: - ntfy.sendMsg(f"{book_name} {chapter_name} 下载中") - download_images(list_img,ci.getDirComicChapter(), files_name=files_name,concurrency=None,timeout=8) - ntfy.sendMsg("等待数据检验中...",sleep=0.5) - is_next = fu.equImages(ci.getDirComicChapter(),list_img) - if not is_next: ntfy.sendMsg(msg=f"下载数据(不完整,{int(repeat*2)}秒钟后尝试第{repeat}次",sleep=int(repeat*2)) - repeat += 1 + #while is_next and repeat <= 10 and not is_old: + # ntfy.sendMsg(f"{book_name} {chapter_name} 下载中") + download_images(list_img,ci.getDirComicChapter(), files_name=files_name,concurrency=None,timeout=8) + # ntfy.sendMsg("等待数据检验中...",sleep=0.5) + is_next = fu.equImages(ci.getDirComicChapter(),list_img) + # if not is_next: ntfy.sendMsg(msg=f"下载数据(不完整,{int(repeat*2)}秒钟后尝试第{repeat}次",sleep=int(repeat*2)) + # repeat += 1 return is_next \ No newline at end of file diff --git a/utils/downloader.py b/utils/downloader.py index 4d5d0d4..0aa4019 100644 --- a/utils/downloader.py +++ b/utils/downloader.py @@ -5,6 +5,7 @@ from __future__ import print_function +from queue import Queue import shutil import imghdr import os @@ -25,15 +26,14 @@ headers = { # 'Connection': 'close', } +down_queue = Queue() -def download_image(image_url, dst_dir, file_name, timeout=20, proxy_type=None, proxy=None,type="image"): +def common_download(file_name,image_url,dst_dir,timeout=10,proxy=None,proxy_type=None): proxies = None if proxy_type is not None: proxies = { "http": proxy_type + "://" + proxy, - "https": proxy_type + "://" + proxy - } - + "https": proxy_type + "://" + proxy } response = None file_path = os.path.join(dst_dir, file_name) if os.path.exists(file_path): @@ -41,31 +41,38 @@ def download_image(image_url, dst_dir, file_name, timeout=20, proxy_type=None, p return None temp_path = os.path.join(dst_dir, file_name+".downloads") repair_count = 1 - try: - response = requests.get( + response = requests.get( image_url, headers=headers, timeout=timeout, proxies=proxies) - while response.status_code != 200 and repair_count <= 5: - time.sleep(0.7) - download_image(image_url,dst_dir,file_name) - ntfy.sendMsg(f'重试:第{repair_count}次 {image_url}') - repair_count += 1 - with open(temp_path, 'wb') as f: - f.write(response.content) - response.close() - #验证是否是图像 - if fu.ver_file(temp_path,type=type): - shutil.move(temp_path, file_path) - print("## OK: {} {}".format(file_path, image_url)) - except Exception as e: - if response: - response.close() - if repair_count < 5: - time.sleep(1) - ntfy.sendMsg("## Repeat: {} {}".format(image_url)) - download_image(image_url,dst_dir,file_name) - repair_count += 1 - else: - print("## Fail: {} {}".format(image_url, e.args)) + while response.status_code != 200 and repair_count <= 5: + time.sleep(0.7) + download_image(image_url,dst_dir,file_name) + ntfy.sendMsg(f'重试:第{repair_count}次 {image_url}') + repair_count += 1 + with open(temp_path, 'wb') as f: + f.write(response.content) + response.close() + #验证是否是图像 + if fu.ver_file(temp_path,type="image"): + shutil.move(temp_path, file_path) + print("## OK: {} {}".format(file_path, image_url)) + else: + print("## Fail: {} {}".format(image_url, "图像损坏")) + down_queue.put([file_name,image_url,dst_dir]) + +def download_image(timeout=20, proxy_type=None, proxy=None,type="image"): + repeat = 0 + while not down_queue.empty() and repeat <= 10: + repeat += 1 + if repeat > 1: + ntfy.sendMsg(f"第{repeat}次下载数据中...") + data = down_queue.get(False) + (file_name,image_url,dst_dir) = [data[0],data[1],data[2]] + try: + common_download(file_name,image_url,dst_dir) + except: + ntfy.sendMsg(f"下载重试中 {file_name}={image_url}") + down_queue.put([file_name,image_url,dst_dir]) + def download_images(image_urls, dst_dir,concurrency=None,timeout=20,proxy_type=None, proxy=None,files_name=None): @@ -89,8 +96,9 @@ def download_images(image_urls, dst_dir,concurrency=None,timeout=20,proxy_type=N os.makedirs(dst_dir) for image_url in image_urls: file_name = files_name[count] + down_queue.put([file_name,image_url,dst_dir]) future_list.append(executor.submit( - download_image, image_url, dst_dir, file_name, timeout, proxy_type, proxy)) + download_image,timeout, proxy_type, proxy)) count += 1 concurrent.futures.wait(future_list, timeout)