diff --git a/utils/downloader.py b/utils/downloader.py index 6d026ce..b40b778 100644 --- a/utils/downloader.py +++ b/utils/downloader.py @@ -80,7 +80,15 @@ def download_images(image_urls, dst_dir, file_prefix="img", concurrency=50, time :param concurrency: number of requests process simultaneously :return: none """ - concurrency = len(image_urls) + file_count = 0 + count = 0 + for image_url in image_urls: + file_name = filesName[count] + file_path = os.path.join(dst_dir, file_name) + if not os.path.exists(file_path): + file_count += 1 + count += 1 + concurrency = int(file_count) with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: future_list = list() count = 0