""" Download image according to given urls and automatically rename them in order. """ # -*- coding: utf-8 -*- # author: Yabin Zheng # Email: sczhengyabin@hotmail.com from __future__ import print_function import shutil import imghdr import os import concurrent.futures import requests import time from utils.Ntfy import ntfy from utils.comic.ComicInfo import comicInfo from utils.HtmlUtils import htmlUtils from utils.FileUtils import fileUtils headers = { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Proxy-Connection": "keep-alive", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36", "Accept-Encoding": "gzip, deflate, sdch", # 'Connection': 'close', } def download_image(image_url, dst_dir, file_name, timeout=20, proxy_type=None, proxy=None,type="image"): proxies = None if proxy_type is not None: proxies = { "http": proxy_type + "://" + proxy, "https": proxy_type + "://" + proxy } response = None file_path = os.path.join(dst_dir, file_name) if os.path.exists(file_path): print("download_image 文件已存在,已跳过=",file_path) return None temp_path = os.path.join(dst_dir, file_name+".downloads") repair_count = 1 try: response = requests.get( image_url, headers=headers, timeout=timeout, proxies=proxies) while response.status_code != 200 and repair_count <= 5: time.sleep(0.7) download_image(image_url,dst_dir,file_name) ntfy.sendMsg(f'重试:第{repair_count}次 {image_url}') repair_count += 1 with open(temp_path, 'wb') as f: f.write(response.content) response.close() #验证是否是图像 if fileUtils.ver_file(temp_path,type=type): shutil.move(temp_path, file_path) print("## OK: {} {}".format(file_path, image_url)) except Exception as e: if response: response.close() if repair_count < 5: time.sleep(1) ntfy.sendMsg("## Repeat: {} {}".format(image_url)) download_image(image_url,dst_dir,file_name) repair_count += 1 else: print("## Fail: {} {}".format(image_url, e.args)) def download_images(image_urls, dst_dir,timeout=30,concurrency=None, proxy_type=None, proxy=None,files_name=None): """ Download image according to given urls and automatically rename them in order. :param timeout: :param proxy: :param proxy_type: :param image_urls: list of image urls :param dst_dir: output the downloaded images to dst_dir :param file_prefix: if set to "img", files will be in format "img_xxx.jpg" :param concurrency: number of requests process simultaneously :return: none """ if concurrency == None: concurrency = len(image_urls) with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: future_list = list() count = 0 if not os.path.exists(dst_dir): os.makedirs(dst_dir) for image_url in image_urls: file_name = files_name[count] future_list.append(executor.submit( download_image, image_url, dst_dir, file_name, timeout, proxy_type, proxy)) count += 1 concurrent.futures.wait(future_list, timeout) def download_comic_icon(): icon_url = comicInfo.getIcon() if icon_url == None: print("icon 不存在,已跳过") return None save_name = comicInfo.COMIC_ICON_NAME icon_prefix = "."+str(icon_url).split(".")[-1] icon_prefix = icon_prefix.split("?")[0] #判断漫画名路径是否已存在comicname/cover.jpg, 存在跳过 path_comic_icon = os.path.join(comicInfo.getDirConfComic(),save_name+icon_prefix) if not comicInfo.equIcon() and os.path.exists(path_comic_icon): os.remove(path_comic_icon) if not os.path.exists(path_comic_icon): download(icon_url, path_comic_icon) #if not os.path.exists(path_cbz_comic): # os.makedirs(path_cbz_comic) save_path = os.path.join(comicInfo.getDirCBZComic(),comicInfo.getChapter()+icon_prefix) #历史版本ICON if os.path.exists(save_path): os.remove(save_path) #shutil.copy(path_comic_icon, save_path) #print(f"{path_comic_icon} 已复制至: {save_path}") if os.path.exists(path_comic_icon): base_dir = comicInfo.getDirComicChapter() if not os.path.exists(base_dir): os.makedirs(base_dir) shutil.copy(path_comic_icon,os.path.join(base_dir,save_name+icon_prefix)) #保存icon信息 comicInfo.iconDB() comicInfo.nextDownloadToCBZChapter() comicInfo.setProgress(comicInfo.PROGRESS_CBZ) # 定义下载函数 def download(url,path,file_type=None): if os.path.exists(path): if imghdr.what(path): msg = "已存在同路径文件,已跳过:"+path print(msg) return msg else: print("文件已损坏,已重试:"+path) path = os.path.join(os.path.dirname(path),str(os.path.basename(path)).split("?")[0]) tmp_file = path+".downloads" if os.path.exists(tmp_file): os.remove(tmp_file) print("存在缓存文件,已删除:",tmp_file) repair_count = 1 res = htmlUtils.getBytes(url) while res.status_code != 200 and repair_count <= 5: res = htmlUtils.getBytes(url) print(f'重试:第{repair_count}次 {url}') repair_count += 1 #判断是否为图片 if file_type == "image": if 'image' not in res.headers.get("content-type",""): print(f"url= {url} Error: URL doesnot appear to be an image") basedir= os.path.dirname(path) if not os.path.exists(basedir): os.makedirs(basedir) #expected_length = res.headers.get('Content-Length') #actual_length = res.raw.tell() with open(tmp_file, 'wb') as f: for ch in res: f.write(ch) f.close() shutil.move(tmp_file, path) print(f"url={url} 保存至:{path}") return path