""" Download image according to given urls and automatically rename them in order. """ # -*- coding: utf-8 -*- # author: Yabin Zheng # Email: sczhengyabin@hotmail.com from __future__ import print_function import shutil import imghdr import os import concurrent.futures import requests import time from utils.Ntfy import ntfy from utils.comic.ComicInfo import comicInfo from utils.HtmlUtils import htmlUtils from utils.FileUtils import fileUtils headers = { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Proxy-Connection": "keep-alive", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36", "Accept-Encoding": "gzip, deflate, sdch", # 'Connection': 'close', } def download_image(image_url, dst_dir, file_name, timeout=20, proxy_type=None, proxy=None,type="image"): proxies = None if proxy_type is not None: proxies = { "http": proxy_type + "://" + proxy, "https": proxy_type + "://" + proxy } response = None file_path = os.path.join(dst_dir, file_name) if os.path.exists(file_path): print("文件已存在,已跳过=",file_path) return None temp_path = os.path.join(dst_dir, file_name+".downloads") repair_count = 1 try: response = requests.get( image_url, headers=headers, timeout=timeout, proxies=proxies) while response.status_code != 200 and repair_count <= 5: time.sleep(0.7) download_image(image_url,dst_dir,file_name) ntfy.sendMsg(f'重试:第{repair_count}次 {image_url}') repair_count += 1 with open(temp_path, 'wb') as f: f.write(response.content) response.close() #验证是否是图像 if fileUtils.ver_file(temp_path,type=type): shutil.move(temp_path, file_path) print("## OK: {} {}".format(file_path, image_url)) except Exception as e: if response: response.close() if repair_count < 5: time.sleep(1) ntfy.sendMsg("## Repeat: {} {}".format(image_url)) download_image(image_url,dst_dir,file_name) repair_count += 1 else: print("## Fail: {} {}".format(image_url, e.args)) def download_images(image_urls, dst_dir, file_prefix="img", concurrency=50, timeout=20, proxy_type=None, proxy=None,filesName=None): """ Download image according to given urls and automatically rename them in order. :param timeout: :param proxy: :param proxy_type: :param image_urls: list of image urls :param dst_dir: output the downloaded images to dst_dir :param file_prefix: if set to "img", files will be in format "img_xxx.jpg" :param concurrency: number of requests process simultaneously :return: none """ concurrency = len(image_urls) with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: future_list = list() count = 0 if not os.path.exists(dst_dir): os.makedirs(dst_dir) for image_url in image_urls: file_name = filesName[count] future_list.append(executor.submit( download_image, image_url, dst_dir, file_name, timeout, proxy_type, proxy)) count += 1 concurrent.futures.wait(future_list, timeout) def download_comic_icon(): icon_url = comicInfo.getIcon() if icon_url == None: print("icon 不存在,已跳过") return None save_name = "cover" icon_su = "."+str(icon_url).split(".")[-1] icon_su = icon_su.split("?")[0] #判断漫画名路径是否已存在comicname/cover.jpg, 存在跳过 pathComicIcon = os.path.join(comicInfo.getDirConfComic(),save_name+icon_su) if not os.path.exists(pathComicIcon): download(icon_url, pathComicIcon) pathCBZComic = comicInfo.getDirCBZComic() if not os.path.exists(pathCBZComic): os.makedirs(pathCBZComic) save_path = os.path.join(pathCBZComic,comicInfo.getChapter()+icon_su) shutil.copy(pathComicIcon, save_path) print(f"{pathComicIcon} 已复制至: {save_path}") comicInfo.nextDownloadToCBZChapter() # 定义下载函数 def download(url,path,fileType=None): if os.path.exists(path): if imghdr.what(path): msg = "已存在同路径文件,已跳过:"+path print(msg) return msg else: print("文件已损坏,已重试:"+path) path = os.path.join(os.path.dirname(path),str(os.path.basename(path)).split("?")[0]) tmp_file = path+".downloads" if os.path.exists(tmp_file): os.remove(tmp_file) print("存在缓存文件,已删除:",tmp_file) repair_count = 1 res = htmlUtils.getBytes(url) while res.status_code != 200 and repair_count <= 5: res = htmlUtils.getBytes(url) print(f'重试:第{repair_count}次 {url}') repair_count += 1 #判断是否为图片 if fileType == "image": if 'image' not in res.headers.get("content-type",""): print(f"url= {url} Error: URL doesnot appear to be an image") basedir= os.path.dirname(path) if not os.path.exists(basedir): os.makedirs(basedir) #expected_length = res.headers.get('Content-Length') #actual_length = res.raw.tell() with open(tmp_file, 'wb') as f: for ch in res: f.write(ch) f.close() shutil.move(tmp_file, path) print(f"url={url} 保存至:{path}") return path