cwx/PyComicPackRouMan
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add download queue

Browse Source
This commit is contained in:
caiwx86 2023-04-01 19:50:39 +08:00
parent f907258d78
commit badb0529c0
2 changed files with 44 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
utils/base/BaseComicEntity.py
View File

@ -124,11 +124,11 @@ class baseComic:
#不存在ComicInfo.xml则生成
if is_next and fu.notExists(ci.getPathComicInfoXML()): ci.writeComicInfoXML(chapter_name)
repeat = 1
while is_next and repeat <= 10 and not is_old:
ntfy.sendMsg(f"{book_name} {chapter_name} 下载中")
download_images(list_img,ci.getDirComicChapter(), files_name=files_name,concurrency=None,timeout=8)
ntfy.sendMsg("等待数据检验中...",sleep=0.5)
is_next = fu.equImages(ci.getDirComicChapter(),list_img)
if not is_next: ntfy.sendMsg(msg=f"下载数据(不完整，{int(repeat*2)}秒钟后尝试第{repeat}次",sleep=int(repeat*2))
repeat += 1
#while is_next and repeat <= 10 and not is_old:
# ntfy.sendMsg(f"{book_name} {chapter_name} 下载中")
download_images(list_img,ci.getDirComicChapter(), files_name=files_name,concurrency=None,timeout=8)
# ntfy.sendMsg("等待数据检验中...",sleep=0.5)
is_next = fu.equImages(ci.getDirComicChapter(),list_img)
# if not is_next: ntfy.sendMsg(msg=f"下载数据(不完整，{int(repeat*2)}秒钟后尝试第{repeat}次",sleep=int(repeat*2))
# repeat += 1
return is_next

66
utils/downloader.py
View File

@ -5,6 +5,7 @@
from __future__ import print_function
from queue import Queue
import shutil
import imghdr
import os
@ -25,15 +26,14 @@ headers = {
# 'Connection': 'close',
}
down_queue = Queue()
def download_image(image_url, dst_dir, file_name, timeout=20, proxy_type=None, proxy=None,type="image"):
def common_download(file_name,image_url,dst_dir,timeout=10,proxy=None,proxy_type=None):
proxies = None
if proxy_type is not None:
proxies = {
"http": proxy_type + "://" + proxy,
"https": proxy_type + "://" + proxy
}
"https": proxy_type + "://" + proxy }
response = None
file_path = os.path.join(dst_dir, file_name)
if os.path.exists(file_path):
@ -41,31 +41,38 @@ def download_image(image_url, dst_dir, file_name, timeout=20, proxy_type=None, p
return None
temp_path = os.path.join(dst_dir, file_name+".downloads")
repair_count = 1
try:
response = requests.get(
response = requests.get(
image_url, headers=headers, timeout=timeout, proxies=proxies)
while response.status_code != 200 and repair_count <= 5:
time.sleep(0.7)
download_image(image_url,dst_dir,file_name)
ntfy.sendMsg(f'重试：第{repair_count}次 {image_url}')
repair_count += 1
with open(temp_path, 'wb') as f:
f.write(response.content)
response.close()
#验证是否是图像
if fu.ver_file(temp_path,type=type):
shutil.move(temp_path, file_path)
print("## OK: {} {}".format(file_path, image_url))
except Exception as e:
if response:
response.close()
if repair_count < 5:
time.sleep(1)
ntfy.sendMsg("## Repeat: {} {}".format(image_url))
download_image(image_url,dst_dir,file_name)
repair_count += 1
else:
print("## Fail: {} {}".format(image_url, e.args))
while response.status_code != 200 and repair_count <= 5:
time.sleep(0.7)
download_image(image_url,dst_dir,file_name)
ntfy.sendMsg(f'重试：第{repair_count}次 {image_url}')
repair_count += 1
with open(temp_path, 'wb') as f:
f.write(response.content)
response.close()
#验证是否是图像
if fu.ver_file(temp_path,type="image"):
shutil.move(temp_path, file_path)
print("## OK: {} {}".format(file_path, image_url))
else:
print("## Fail: {} {}".format(image_url, "图像损坏"))
down_queue.put([file_name,image_url,dst_dir])
def download_image(timeout=20, proxy_type=None, proxy=None,type="image"):
repeat = 0
while not down_queue.empty() and repeat <= 10:
repeat += 1
if repeat > 1:
ntfy.sendMsg(f"第{repeat}次下载数据中...")
data = down_queue.get(False)
(file_name,image_url,dst_dir) = [data[0],data[1],data[2]]
try:
common_download(file_name,image_url,dst_dir)
except:
ntfy.sendMsg(f"下载重试中 {file_name}={image_url}")
down_queue.put([file_name,image_url,dst_dir])
def download_images(image_urls, dst_dir,concurrency=None,timeout=20,proxy_type=None, proxy=None,files_name=None):
@ -89,8 +96,9 @@ def download_images(image_urls, dst_dir,concurrency=None,timeout=20,proxy_type=N
os.makedirs(dst_dir)
for image_url in image_urls:
file_name = files_name[count]
down_queue.put([file_name,image_url,dst_dir])
future_list.append(executor.submit(
download_image, image_url, dst_dir, file_name, timeout, proxy_type, proxy))
download_image,timeout, proxy_type, proxy))
count += 1
concurrent.futures.wait(future_list, timeout)