cwx/PyComicPackRouMan
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
caiwx86 2023-01-20 21:44:10 +08:00
parent 3ff44f4226
commit 5398f1f0e4
1 changed files with 2 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
utils/downloader.py
View File

@ -80,15 +80,7 @@ def download_images(image_urls, dst_dir, file_prefix="img", concurrency=50, time
:param concurrency: number of requests process simultaneously :param concurrency: number of requests process simultaneously
:return: none :return: none
""" """
file_count = 0 concurrency = int(image_urls)
count = 0
for image_url in image_urls:
file_name = filesName[count]
file_path = os.path.join(dst_dir, file_name)
if not os.path.exists(file_path):
file_count += 1
count += 1
concurrency = int(file_count)
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor:
future_list = list() future_list = list()
count = 0 count = 0
@ -96,11 +88,7 @@ def download_images(image_urls, dst_dir, file_prefix="img", concurrency=50, time
os.makedirs(dst_dir) os.makedirs(dst_dir)
for image_url in image_urls: for image_url in image_urls:
file_name = filesName[count] file_name = filesName[count]
file_path = os.path.join(dst_dir, file_name) future_list.append(executor.submit(
if os.path.exists(file_path):
print("文件已存在，已跳过=",file_path)
else:
future_list.append(executor.submit(
download_image, image_url, dst_dir, file_name, timeout, proxy_type, proxy)) download_image, image_url, dst_dir, file_name, timeout, proxy_type, proxy))
count += 1 count += 1
concurrent.futures.wait(future_list, timeout) concurrent.futures.wait(future_list, timeout)