diff --git a/utils/downloader.py b/utils/downloader.py index ee2bc67..e2d8e5f 100644 --- a/utils/downloader.py +++ b/utils/downloader.py @@ -68,7 +68,7 @@ def download_image(image_url, dst_dir, file_name, timeout=20, proxy_type=None, p print("## Fail: {} {}".format(image_url, e.args)) -def download_images(image_urls, dst_dir,timeout=20, proxy_type=None, proxy=None,files_name=None): +def download_images(image_urls, dst_dir,timeout=20,concurrency=5, proxy_type=None, proxy=None,files_name=None): """ Download image according to given urls and automatically rename them in order. :param timeout: @@ -80,7 +80,6 @@ def download_images(image_urls, dst_dir,timeout=20, proxy_type=None, proxy=None, :param concurrency: number of requests process simultaneously :return: none """ - concurrency = len(image_urls) with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: future_list = list() count = 0 @@ -91,6 +90,7 @@ def download_images(image_urls, dst_dir,timeout=20, proxy_type=None, proxy=None, future_list.append(executor.submit( download_image, image_url, dst_dir, file_name, timeout, proxy_type, proxy)) count += 1 + time.sleep(0.3) concurrent.futures.wait(future_list, timeout) def download_comic_icon():