PyComicPackRouMan/utils/downloader.py

""" Download image according to given urls and automatically rename them in order. """
# -*- coding: utf-8 -*-
# author: Yabin Zheng
# Email: sczhengyabin@hotmail.com

from __future__ import print_function

import shutil
import imghdr
import os
import concurrent.futures
import requests
import time
from utils.Ntfy import ntfy
from utils.comic.ComicInfo import comicInfo
from utils.HtmlUtils import htmlUtils

headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Proxy-Connection": "keep-alive",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36",
    "Accept-Encoding": "gzip, deflate, sdch",
    # 'Connection': 'close',
}


def download_image(image_url, dst_dir, file_name, timeout=20, proxy_type=None, proxy=None):
    proxies = None
    if proxy_type is not None:
        proxies = {
            "http": proxy_type + "://" + proxy,
            "https": proxy_type + "://" + proxy
        }

    response = None
    file_path = os.path.join(dst_dir, file_name)
    if os.path.exists(file_path):
        print("文件已存在，已跳过=",file_path)
        return None
    temp_path = os.path.join(dst_dir, file_name+".downloads")
    repair_count = 1
    try:
        response = requests.get(
            image_url, headers=headers, timeout=timeout, proxies=proxies)
        while response.status_code != 200 and repair_count <= 5:
            time.sleep(0.7)
            download_image(image_url,dst_dir,file_name)
            ntfy.sendMsg(f'重试：第{repair_count}次 {image_url}')
            repair_count += 1
        with open(temp_path, 'wb') as f:
            f.write(response.content)
        response.close()
        shutil.move(temp_path, file_path)
        print("## OK:  {}  {}".format(file_path, image_url))
    except Exception as e:
        if response:
           response.close()
        if repair_count < 5:
            time.sleep(1)
            ntfy.sendMsg("## Repeat: {}  {}".format(image_url))
            download_image(image_url,dst_dir,file_name)
            repair_count += 1
        else:
            print("## Fail:  {}  {}".format(image_url, e.args))


def download_images(image_urls, dst_dir, file_prefix="img", concurrency=50, timeout=20, proxy_type=None, proxy=None,filesName=None):
    """
    Download image according to given urls and automatically rename them in order.
    :param timeout:
    :param proxy:
    :param proxy_type:
    :param image_urls: list of image urls
    :param dst_dir: output the downloaded images to dst_dir
    :param file_prefix: if set to "img", files will be in format "img_xxx.jpg"
    :param concurrency: number of requests process simultaneously
    :return: none
    """
    concurrency = len(image_urls) 
    with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor:
        future_list = list()
        count = 0 
        if not os.path.exists(dst_dir):
            os.makedirs(dst_dir)
        for image_url in image_urls:
            file_name = filesName[count] 
            future_list.append(executor.submit(
                download_image, image_url, dst_dir, file_name, timeout, proxy_type, proxy))
            count += 1
        concurrent.futures.wait(future_list, timeout)

def download_comic_icon():
        icon_url = comicInfo.getIcon()
        if icon_url == None:
            print("icon 不存在，已跳过")
            return None
        save_name = "cover"
        icon_su = "."+str(icon_url).split(".")[-1]
        icon_su = icon_su.split("?")[0]
        #判断漫画名路径是否已存在comicname/cover.jpg, 存在跳过
        pathComicIcon = os.path.join(comicInfo.getDirConfComic(),save_name+icon_su)
        if not os.path.exists(pathComicIcon):
            download(icon_url, pathComicIcon)
        pathCBZComic = comicInfo.getDirCBZComic()
        if not os.path.exists(pathCBZComic):
              os.makedirs(pathCBZComic)
        save_path = os.path.join(pathCBZComic,comicInfo.getChapter()+icon_su)
        index = comicInfo.getChapterIndex()
        if index != None:
            save_path = os.path.join(pathCBZComic,str(index)+" "+comicInfo.getChapter()+icon_su)
        shutil.copy(pathComicIcon, save_path)
        print(f"{pathComicIcon} 已复制至: {save_path}")
        comicInfo.nextDownloadToCBZChapter()

    # 定义下载函数
def download(url,path,fileType=None):
       if os.path.exists(path):
           if imghdr.what(path):
                msg = "已存在同路径文件,已跳过:"+path
                print(msg)
                return msg
           else:
               print("文件已损坏，已重试："+path)
       path = os.path.join(os.path.dirname(path),str(os.path.basename(path)).split("?")[0])
       tmp_file = path+".downloads"
       if os.path.exists(tmp_file):
           os.remove(tmp_file)
           print("存在缓存文件，已删除：",tmp_file)
       repair_count = 1
       res = htmlUtils.getBytes(url)
       while res.status_code != 200 and repair_count <= 5:
           res = htmlUtils.getBytes(url)
           print(f'重试：第{repair_count}次 {url}')
           repair_count += 1
       #判断是否为图片
       if fileType == "image":
          if 'image' not in res.headers.get("content-type",""):
              print(f"url= {url} Error: URL doesnot appear to be an image")
       basedir= os.path.dirname(path)
       if not os.path.exists(basedir):
           os.makedirs(basedir)
       #expected_length = res.headers.get('Content-Length')
       #actual_length = res.raw.tell()
       with open(tmp_file, 'wb') as f:
            for ch in res:
                f.write(ch)
            f.close()
       shutil.move(tmp_file, path)
       print(f"url={url} 保存至：{path}")
       return path