77 lines
2.8 KiB
Python
77 lines
2.8 KiB
Python
""" Download image according to given urls and automatically rename them in order. """
|
|
# -*- coding: utf-8 -*-
|
|
# author: Yabin Zheng
|
|
# Email: sczhengyabin@hotmail.com
|
|
|
|
from __future__ import print_function
|
|
|
|
import shutil
|
|
import imghdr
|
|
import os
|
|
import concurrent.futures
|
|
import requests
|
|
|
|
headers = {
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
"Proxy-Connection": "keep-alive",
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36",
|
|
"Accept-Encoding": "gzip, deflate, sdch",
|
|
# 'Connection': 'close',
|
|
}
|
|
|
|
|
|
def download_image(image_url, dst_dir, file_name, timeout=20, proxy_type=None, proxy=None):
|
|
proxies = None
|
|
if proxy_type is not None:
|
|
proxies = {
|
|
"http": proxy_type + "://" + proxy,
|
|
"https": proxy_type + "://" + proxy
|
|
}
|
|
|
|
response = None
|
|
file_path = os.path.join(dst_dir, file_name)
|
|
temp_path = os.path.join(dst_dir, file_name+".downloads")
|
|
repair_count = 1
|
|
try:
|
|
response = requests.get(
|
|
image_url, headers=headers, timeout=timeout, proxies=proxies)
|
|
while response.status_code != 200 and repair_count <= 5:
|
|
download_image(image_url,dst_dir,file_name)
|
|
print(f'重试：第{repair_count}次 {image_url}')
|
|
repair_count += 1
|
|
with open(temp_path, 'wb') as f:
|
|
f.write(response.content)
|
|
response.close()
|
|
shutil.move(temp_path, file_path)
|
|
print("## OK: {} {}".format(file_path, image_url))
|
|
except Exception as e:
|
|
if response:
|
|
response.close()
|
|
print("## Fail: {} {}".format(image_url, e.args))
|
|
|
|
|
|
def download_images(image_urls, dst_dir, file_prefix="img", concurrency=50, timeout=20, proxy_type=None, proxy=None,filesName=None):
|
|
"""
|
|
Download image according to given urls and automatically rename them in order.
|
|
:param timeout:
|
|
:param proxy:
|
|
:param proxy_type:
|
|
:param image_urls: list of image urls
|
|
:param dst_dir: output the downloaded images to dst_dir
|
|
:param file_prefix: if set to "img", files will be in format "img_xxx.jpg"
|
|
:param concurrency: number of requests process simultaneously
|
|
:return: none
|
|
"""
|
|
concurrency = len(image_urls)
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor:
|
|
future_list = list()
|
|
count = 0
|
|
if not os.path.exists(dst_dir):
|
|
os.makedirs(dst_dir)
|
|
for image_url in image_urls:
|
|
file_name = filesName[count]
|
|
future_list.append(executor.submit(
|
|
download_image, image_url, dst_dir, file_name, timeout, proxy_type, proxy))
|
|
count += 1
|
|
concurrent.futures.wait(future_list, timeout=180) |