diff --git a/Comics/items.py b/Comics/items.py index cb65687..59b3fa9 100644 --- a/Comics/items.py +++ b/Comics/items.py @@ -5,14 +5,16 @@ import os,Comics.settings as settings,logging from scrapy.item import Item, Field from Comics.utils import ComicPath -from Comics.utils import imageUtils +from Comics.utils import imageUtils,fileUtils from itemloaders.processors import TakeFirst # 繁体中文转为简体中文 def serialize_to_chinese(value): return ComicPath.chinese_convert(value) # 将路径字符串转为合法路径 -def serialize_to_fix_file(value): return ComicPath.fix_file_name(ComicPath.chinese_convert(value)) +def serialize_to_fix_file(value): return ComicPath.fix_file_name(value) + +def serialize_name_to_chinese(value): return ComicPath.fix_file_name(ComicPath.chinese_convert(value)) # 图片数据解析(私有方法) def _serialize_to_images(value, result_type=None): @@ -57,9 +59,13 @@ class ComicItem(Item): # 编号 index = Field(output_processor=TakeFirst()) # 漫画名 - name = Field(serializer=serialize_to_fix_file, output_processor=TakeFirst()) + name = Field(serializer=serialize_name_to_chinese, output_processor=TakeFirst()) + # 源漫画名 + s_name = Field(serializer=serialize_to_fix_file, output_processor=TakeFirst()) # 章节名 - chapter = Field(serializer=serialize_to_fix_file, output_processor=TakeFirst()) + chapter = Field(serializer=serialize_name_to_chinese, output_processor=TakeFirst()) + # 源章节名 + s_chapter = Field(serializer=serialize_to_fix_file, output_processor=TakeFirst()) # 图片链接 list_img = Field(serializer=serialize_to_images) # 作者 @@ -134,4 +140,5 @@ class ComicInfoItem(Item): LanguageISO = Field(desc="语言") AgeRating = Field(desc="年龄分级", info='age_rating') Pages = Field(desc="页码", info='images', serializer=serializer_info_images) - # ComicInfo.xml and ComicChapter.json end \ No newline at end of file + # ComicInfo.xml and ComicChapter.json end + \ No newline at end of file diff --git a/Comics/loader.py b/Comics/loader.py index 8dddb37..4eaa77a 100644 --- a/Comics/loader.py +++ b/Comics/loader.py @@ -113,8 +113,12 @@ class ComicLoader(ItemLoader): # 漫画名称 def get_name(self): return self.get_output_value("name") + def get_sname(self): return self.get_output_value("s_name") # 漫画章节 def get_chapter(self): return self.get_output_value("chapter") + def get_schapter(self): return self.get_output_value("s_chapter") + # 漫画封面 + def get_icon(self): return self.get_output_value("icon") # 工程名 def get_project_name(self): return self.get_output_value(PROJECT_KEY) # 章节链接 @@ -124,34 +128,38 @@ class ComicLoader(ItemLoader): def get_chapter_api(self): return self.get_output_value("chapter_api") + def get_images(self): return self.get_output_value("images") + def get_image_urls(self): return self.get_output_value("image_urls") -class ComicEntity: - - ENTITY = None - - def __init__(self, entity): - self.ENTITY = entity - - # 属性获取 - def get_dict(self, key): - try: - return self.ENTITY[key] - except: - return [] - - def set(self, key, value): self.ENTITY[key] = value - def set_count(self, value): self.set('count', value) - def set_index(self, value): self.set('index', value) - def images(self): return self.get_dict("images") - def image_urls(self): return self.get_dict("image_urls") - def chapters(self): return self.get_dict("chapters") - def chapter(self): return self.get_dict("chapter") - def icon(self): return self.get_dict("icon") - def count(self): return self.set_count(len(self.images())) - def index(self): return self.set_index(self.chapters().index(self.chapter()) + 1) - - def item(self): + ### ComicEntity + def set_count(self, value): self.set_properties('count', value=value) + + def set_index(self, value): self.set_properties('index', value=value) + + def set_sname(self, value): self.set_properties('s_name', value=value) + + def set_chapter(self, value): self.set_properties('chapter', value=value) + def set_schapter(self, value): self.set_properties('s_chapter', value=value) + + # 章节页码 + def count(self): + len_images = len(self.get_images()) + if len_images != 0: return self.set_count(len_images) + # 章节编号 + def index(self): + chapters, chapter = [ self.get_chapters(), self.get_chapter() ] + if chapter != None and len(chapters) > 0: return self.set_index(chapters.index(chapter) + 1) + + def save_sname_schapter(self): + chapter = self.get_chapter() + name = self.get_name() + if chapter != None : self.set_schapter(chapter) + if name != None : self.set_sname(name) + + def load_item(self, chapter=None): self.count() self.index() - return self.ENTITY + if chapter != None: self.set_chapter(chapter) + self.save_sname_schapter() + return super().load_item() \ No newline at end of file diff --git a/Comics/pipelines.py b/Comics/pipelines.py index 338abde..e82b2e9 100644 --- a/Comics/pipelines.py +++ b/Comics/pipelines.py @@ -8,8 +8,7 @@ import os,scrapy,logging from Comics import settings from Comics.items import ComicItem -from Comics.settings import OUTPUT_DIR -from Comics.loader import ComicEntity,ComicLoader +from Comics.loader import ComicLoader from Comics.exporters import ComicInfoXmlItemExporter from Comics.utils import CBZUtils,fileUtils as fu from Comics.utils import ComicPath @@ -18,63 +17,83 @@ from Comics.exporters import JsonExport,ItemExporter from scrapy.pipelines.images import ImagesPipeline class ComicsPipeline(): - def open_spider(self, spider): - pass # item就是yield后面的对象 - def process_item(self, item, spider): + def process_item(self, item: ComicItem, spider): if isinstance(item, ComicItem): - # item = ComicEntity(item).item() - # 'output/rm_comic/json/壞X/第1話 壞X' + # 'output/rm_comic/json/壞X/第1話 壞X' # 已存在漫画CBZ文件 调用转换 - if fu.exists(ComicPath.path_cbz(item=item)): return ItemExporter().export_obj(item) - else: - # 不存在漫画CBZ文件 - #file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter']) - return JsonExport(file=ComicPath.getDirJosnComicChapter(item)).export_json(ComicEntity(item).item(), if_return=True) - # image解析 + if fu.exists(ComicPath(item).path_cbz()): return ItemExporter().export_obj(item) + # 不存在漫画CBZ文件 + else: return JsonExport(file=ComicPath(item).getDirJosnComicChapter(item)).export_json(ComicLoader(item).load_item(), if_return=True) - def close_spider(self, spider): - pass - -class ImgDownloadPipeline(ImagesPipeline): - def get_file_path(self, item, file=None, result_type="image"): - return ComicPath().get_file_path(item=item, file=file, result_type= result_type) - - def image_scramble_exits(self, item,image_path): - en_image_path = ComicPath().getFileScrambleImageSave(image_path, relative="fullpath") - return fu.exists(fu.join(settings.IMAGES_STORE, self.get_file_path(item, en_image_path))) +class BaseComicImagesPipeline(ImagesPipeline): - ## Icon Path : CBZ/NAME/CHAPTER.jpg - def download_icon(self, item, result_type="download"): - icon_path = self.get_file_path(item, result_type="icon_cache") - if result_type == "fullpath": - return fu.join(settings.IMAGES_STORE, icon_path) - if fu.exists(icon_path): - return False - else: - self.image_urls.append(item['icon']) - self.images.append(icon_path) - return True + def image_scramble_exits(self, item,image_path): + en_image_path = ComicPath(item).getFileScrambleImageSave(image_path, relative="fullpath") + return fu.exists(fu.join(settings.IMAGES_STORE, self.get_file_path(item, en_image_path))) + def get_file_path(self, item, file=None, result_type="image"): + return ComicPath(item).file_path(file=file, result_type= result_type) + + # 封面路径 def file_path(self, request, response=None, info=None, *, item=None): return request.meta['path'] + + def success_completed(self, item, results): + is_success = False + fail_data = [] + for result in results: + # 失败数据 + if not result[0]: fail_data.append(result[1]) + if len(fail_data) == 0 and len(results) != 0: is_success = True + return is_success +# 封面下载操作类 +class IconDownloadPipeline(BaseComicImagesPipeline): + + # 数据处理 def get_media_requests(self, item, info): - comic = ComicEntity(item) - self.image_urls = comic.image_urls() - self.images = comic.images() - # 下载封面 - self.download_icon(item) + comic = ComicLoader(item=item) + # 获取封面链接和封面保存路径 + icon_url, icon_cache_path = [ comic.get_icon(), self.get_file_path(item, result_type="icon_cache") ] + # 封面已存在 + if fu.exists(icon_cache_path): return False + else: yield scrapy.Request(url=icon_url, meta={'path': icon_cache_path }) + + # 判断是否需要更新封面 + def update_icon(self, item): + # 下载后的缓存封面路径 + image_path = self.get_file_path(item, result_type="down_cache_icon") + # 最终封面保存路径 + save_path = self.get_file_path(item=item, result_type="down_icon") + + fu.update_icon(image_path, save_path) + + def item_completed(self, results, item, info): + if self.success_completed(item, results): + print(" icon download success") + # 更新封面到Icon文件夹内 + self.update_icon(item) + + +class ImgDownloadPipeline(BaseComicImagesPipeline): + + + def get_media_requests(self, item, info): + comic = ComicLoader(item=item) + self.image_urls, self.images = [ comic.get_image_urls(), comic.get_images() ] + # 添加封面下载信息至下载列表中 + # self.add_download_icon(item) for image_url,image in zip(self.image_urls,self.images): - is_down = True - image_path = self.get_file_path(item, image) + if_down, image_path = [ True, self.get_file_path(item, image)] + # 图像（含加密图像）已存在 if self.image_scramble_exits(item, image_path): - if image_path == self.get_file_path(item, result_type="icon_cache"): - logging.info(f"icon file exists: IMAGE_STORE {image_path}") - else: - is_down = False - logging.info(f"file exists: IMAGE_STORE {image_path}") - if is_down: + #if image_path == self.get_file_path(item, result_type="icon_cache"): + # logging.info(f"icon file exists: IMAGE_STORE {image_path}") + #else: + if_down = False + logging.info(f"file exists: IMAGE_STORE {image_path}") + if if_down: logging.info(f"downloading {image_url} --> IMAGE_STORE {image_path}") yield scrapy.Request(url=image_url, meta={'path': image_path}) @@ -89,16 +108,6 @@ class ImgDownloadPipeline(ImagesPipeline): if os.path.isfile(path) and dirname.startswith(name): fu.update_icon(path, cbz_icon) - - - # 判断是否需要更新封面 - def update_icon(self, item): - # 下载后的缓存封面路径 - image_path = self.get_file_path(item, result_type="down_cache_icon") - # 最终封面保存路径 - save_path = self.get_file_path(item=item, result_type="down_icon") - - fu.update_icon(image_path, save_path) def item_completed(self, results, item, info): # return item @@ -107,10 +116,9 @@ class ImgDownloadPipeline(ImagesPipeline): success_data = [] for result in results: if result[0]: success_data.append(result[1]) - image_urls = ComicLoader(item=item).get_image_urls() - if len(success_data) != len(image_urls): return + if len(success_data) != len(ComicLoader(item=item).get_image_urls()): return if fu.exists(cbz_path): - self.update_icon(item) + #self.update_icon(item) self.pack_icon(item) else: # ComicInfoXml 生成 diff --git a/Comics/settings.py b/Comics/settings.py index 079e815..d68eaa6 100644 --- a/Comics/settings.py +++ b/Comics/settings.py @@ -45,7 +45,7 @@ RETRY_HTTP_CODES = [408, 401] CONCURRENT_REQUESTS_PER_DOMAIN = 16 CONCURRENT_REQUESTS_PER_IP = 16 PROXY_LIST = [ -# "http://127.0.0.1:7890", + "http://127.0.0.1:7890", # "http://10.0.10.117:8123", ] # Disable cookies (enabled by default) @@ -89,6 +89,7 @@ ITEM_PIPELINES = { # 'scrapy.pipelines.images.ImagesPipeline' : 1, 'Comics.pipelines.ComicsPipeline': 300, # 'Comics.pipelines.ImageParsePipeline': 400, + 'Comics.pipelines.IconDownloadPipeline': 400, 'Comics.pipelines.ImgDownloadPipeline': 500, } diff --git a/Comics/spiders/rm_comic.py b/Comics/spiders/rm_comic.py index f2ea203..734f5e2 100644 --- a/Comics/spiders/rm_comic.py +++ b/Comics/spiders/rm_comic.py @@ -2,8 +2,8 @@ import scrapy,logging,time,os,skip from Comics.items import ComicItem from Comics.loader import ComicLoader from Comics.utils import ComicPath -from Comics.utils import checkUtils from Comics.utils import Conf +from Comics.exporters import ItemExporter class RmComicSpider(scrapy.Spider): name = 'rm_comic' @@ -19,18 +19,27 @@ class RmComicSpider(scrapy.Spider): # 获取多个漫画信息 def books_comic(self, response): comics = ComicLoader(item=ComicItem(), response=response) + # 获取漫画网站//script[@id]内的json数据并获取props.pageProps.books数据并作偱环解析 for book in comics.get_exec(comics.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0], str_exec="props.pageProps.books"): + # 排除指定的漫画名 if book['name'] not in skip.skip_comic: yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic) # 获取某个漫画的相关数据 # 获取到多个章节链接后进入下个流程 def parse_comic(self, response): + # 初始化Comic数据并根据工程名称读取配置文件并自动解析 comic_item = Conf().comic(self.name, ComicLoader(ComicItem(), response)) + # 循环遍历根据配置文件自动解析并注入的章节名和章节链接 for chapter, link in zip(comic_item.get_chapters(), comic_item.get_chapter_href()): - item = comic_item.load_item() - cbz_path = ComicPath().get_file_path(item=item, result_type="cbz", convert=True, chapter=chapter) - if not checkUtils().is_error(item) and os.path.exists(cbz_path): + # 打包导出item数据 + item = comic_item.load_item(chapter=chapter) + # 获取最终存放CBZ的路径 + #cbz_path = ComicPath(item=item).file_path(ComicPath.PATH_CBZ, convert=True, chapter=chapter) + cbz_path = ComicPath(item=item).PATH_CBZ() + # 校验繁体和简体中文CBZ路径是否存在 + # if not checkUtils().is_error(item) and os.path.exists(cbz_path): + if os.path.exists(cbz_path): logging.info(f"漫画 {cbz_path} 已存在, 跳过中...") yield item else: @@ -40,6 +49,7 @@ class RmComicSpider(scrapy.Spider): # 读取某章节下的所有图片 def parse_chapter(self, response): + # 获取传入的漫画item数据 comic_item = ComicLoader(item=response.meta['item'], response=response) data = comic_item.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0] item: ComicLoader = Conf().parse_chapter(item=comic_item, value=data) diff --git a/Comics/spiders/yh_comic.py b/Comics/spiders/yh_comic.py index 57df971..20a182a 100644 --- a/Comics/spiders/yh_comic.py +++ b/Comics/spiders/yh_comic.py @@ -45,7 +45,7 @@ class RmComicSpider(scrapy.Spider): comic_item.chapters(value=chapters) comic_item.chapter(value=chapter) item = comic_item.load_item() - cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True) + cbz_path = ComicPath(item).get_file_path(result_type="cbz", convert=True) if os.path.exists(cbz_path): logging.info(f"漫画 {cbz_path} 已存在, 跳过中...") yield item diff --git a/Comics/utils.py b/Comics/utils.py index e09af7b..4984cb8 100644 --- a/Comics/utils.py +++ b/Comics/utils.py @@ -10,7 +10,7 @@ from zipfile import ZipFile from Comics.settings import COMIC_INFO_XML_FILE,OUTPUT_DIR,PROJECT_KEY import yaml from Comics.loader import ComicLoader - +from tinydb import TinyDB, Query # 配置类 class Conf(): @@ -81,11 +81,16 @@ class fileUtils: # 保存文件 @classmethod - def save_file(cls,path,data): + def write(cls,path,data,type='file'): root_dir = os.path.dirname(path) if not os.path.exists(root_dir): os.makedirs(root_dir) with open(path,'w',encoding='utf-8') as fs: - fs.write(str(data)) + if type == 'file': + fs.write(data) + elif type == 'json': + fs.write(json.dumps(data,ensure_ascii=False,indent=4)) + elif type == 'yaml': + fs.write(yaml.dump(data, allow_unicode=True, default_flow_style=False, indent=4)) # 返回校验后的文件路径 @classmethod @@ -104,41 +109,67 @@ class fileUtils: # 读取文件 @classmethod - def read(cls, file): + def read(cls, file, type='file'): if os.path.exists(file): - with open(file, "r", encoding="utf-8") as fs: return fs.read() + with open(file, "r", encoding="utf-8") as fs: + if type == "json": + return json.loads(fs.read()) + elif type == 'yaml': + return yaml.safe_load(fs.read()) + elif type == 'file': + return fs.read() else: return [] + @classmethod + def get_file_md5(cls, file_path): + md5 = hashlib.md5() + with open(file_path, 'rb') as f: + for chunk in iter(lambda: f.read(4096), b''): + md5.update(chunk) + return md5.hexdigest() + + + + @classmethod + def write_json(cls, path, data): cls.write(path, data , type="json") + + @classmethod + def read_json(cls, file): return cls.read(file, type="json") + """ 图像编号 image-1.jpg 如：存在image.png 返回 image-1.png 反之 image.png """ @classmethod def file_check(cls, file, result="file", count=0): - temp_file_name, files_size, files_name = [file, {}, []] + # 初始化临时文件名 文件信息 文件名列表 + temp_file_name, files_data, files_name = [file, {}, []] # 默认文件名不存在 if not cls.exists(temp_file_name) and temp_file_name == file: count = 1 while count or count == 0: + # 获取格式化文件名 temp_file_name = ComicPath().images_icon(file=file, count=count) + # 格式化文件名存在 if cls.exists(temp_file_name): # 保存存在的文件名 files_name.append(temp_file_name) file_size = os.path.getsize(temp_file_name) # 保存文件名和大小数据 - files_size[file_size] = {"name": temp_file_name, "size": file_size} + files_data[file_size] = {"name": temp_file_name, "size": file_size} # 格式化文件名 # temp_file_name = ComicPath().images_icon(file=file, count=count) count += 1 else: # 检测是否有重复数据 # 提取重复并需删除的文件名 - diff_names = {value["name"] for value in files_size.values()} + diff_names = {value["name"] for value in files_data.values()} # 不存在则返回原文件名 if len(diff_names) == 0: + # 默认为空 if result == "size": return diff_names - else: - return file + # 原文件名 + else: return file for file_name in files_name: if file_name not in diff_names: logging.info(f"删除文件：{file_name}") @@ -158,7 +189,7 @@ class fileUtils: if len(set(diff_names)) != len(set(files_name)): cls.file_check(file, result=result,count=0) if result == "size": - return {value["size"] for value in files_size.values()} + return {value["size"] for value in files_data.values()} else: return temp_file_name @@ -532,13 +563,14 @@ class CBZUtils: @classmethod def cbz_validate(cls, zip_path, comic_info_images): - if len(cls.zip_info(zip_path)) == len(comic_info_images): + cbz_info = cls.zip_info(zip_path) + if len(cbz_info) == len(comic_info_images): # logging.info(f"validating successfully === {zip_path}") ntfy.sendMsg(f"validating successfully === {zip_path}", alert=True) return True else: os.remove(zip_path) - logging.error(f"validating fail === {zip_path}") + ntfy.sendMsg(f"validating fail === {zip_path}, cbz_info={cbz_info},zip_info={comic_info_images}", alert=True) return False # 检测工具类 @@ -574,8 +606,51 @@ class checkUtils: # Comic路径类 class ComicPath: + ci, project, name, chapter = [ None, "", "", ""] + + def __init__(self, item = None): + if item != None: self.ci : ComicLoader = ComicLoader(item=item) + if self.ci.get_project_name() != None: self.project = self.fix_file_name(self.ci.get_project_name()) + if self.ci.get_name() != None: self.name = self.fix_file_name(self.ci.get_name()) + if self.ci.get_chapter() != None: self.chapter = self.fix_file_name(self.ci.get_chapter()) + PREFIX_SCRAMBLE = "scramble=" + PATH_IMAGE = "image" + PATH_COMIC_INFO = "comic_info" + PATH_CBZ_ICON = "cbz_icon" + PATH_DOWN_ICON = "down_icon" + PATH_DOWN_CACHE_ICON = "down_cache_icon" + PATH_ICON = "icon" + PATH_ICON_CACHE = "icon_cache" + PATH_CBZ = "cbz" + PATH_IMAGES_DIR = "images_dir" + + def PATH_MAPPING(self): + if self.project == None or self.name == None or self.chapter == None: return None + return { + self.PATH_IMAGE: os.path.join(self.project, "images", self.name, self.chapter), + self.PATH_COMIC_INFO: os.path.join(self.project, "images", self.name, self.chapter), + self.PATH_CBZ_ICON: os.path.join(settings.CBZ_EXPORT_PATH, self.project, self.name, self.chapter+".jpg"), + self.PATH_DOWN_ICON: os.path.join(settings.IMAGES_STORE, self.project, "icons", self.name, self.name+".jpg"), + self.PATH_DOWN_CACHE_ICON: os.path.join(settings.IMAGES_STORE, self.project, "icons", ".cache", self.name+".jpg"), + self.PATH_ICON: os.path.join(self.project, "icons", self.name, self.name+".jpg"), + self.PATH_ICON_CACHE: os.path.join(self.project, "icons", ".cache", self.name+".jpg"), + self.PATH_CBZ: os.path.join(settings.CBZ_EXPORT_PATH, self.project, self.name, self.chapter+".CBZ"), + self.PATH_IMAGES_DIR: os.path.join(settings.IMAGES_STORE, self.project, "images", self.name, self.chapter) + } + + def PATH_CBZ(self, result_type=PATH_CBZ): self.file_path(result_type=result_type) + + def file_path(self, result_type=PATH_IMAGE, file=None, convert=True, chapter=None): + if chapter != None: self.chapter = chapter + path = self.PATH_MAPPING().get(result_type, None) + if result_type == self.PATH_IMAGE and os.path.sep not in file: + return os.path.join(path, file) + if convert: + path = self.chinese_convert(path) + return path + @classmethod def getYearMonthDay(cls): today = date.today() @@ -633,42 +708,8 @@ class ComicPath: @classmethod def new_file_name(cls, name): return cls.fix_file_name(cls.chinese_convert(name)) - @classmethod - def get_file_path(cls, item, result_type="image", file=None, convert=False, chapter=None): - PROJECT = ComicLoader(item=item).get_project_name() - if not convert: - name = item['name'] - if chapter == None: chapter = item['chapter'] - else: - name = cls.fix_file_name(cls.chinese_convert(item['name'])) - if chapter == None: chapter = cls.fix_file_name(cls.chinese_convert(item['chapter'])) - - if result_type == "image": - if os.path.sep not in file: - file = os.path.join(PROJECT, "images", name, chapter, file) - elif result_type == "comic_info": - file = os.path.join(PROJECT, "images", name, chapter) - elif result_type == "cbz_icon": - file = os.path.join(settings.CBZ_EXPORT_PATH, PROJECT, name, chapter+".jpg") - elif result_type == "down_icon": - file = os.path.join(settings.IMAGES_STORE, cls.get_file_path(item=item, result_type="icon")) - elif result_type == "down_cache_icon": - file = os.path.join(settings.IMAGES_STORE, cls.get_file_path(item=item, result_type="icon_cache")) - elif result_type == "icon": - file = os.path.join(PROJECT, "icons", name, name+".jpg") - elif result_type == "icon_cache": - file = os.path.join(PROJECT, "icons", ".cache", name+".jpg") - elif result_type == "cbz": - file = os.path.join(settings.CBZ_EXPORT_PATH, PROJECT, name, chapter+".CBZ") - elif result_type == "images_dir": - file = os.path.join(settings.IMAGES_STORE, PROJECT, "images", name, chapter) - else: - raise ValueError(f"Unsupported result_type: {result_type}") - return file - - @classmethod - def path_cbz(cls, item): - return cls.get_file_path(item, result_type="cbz", convert=True) + def path_cbz(self): + return self.file_path(result_type=self.PATH_CBZ, convert=True) @classmethod def images_icon(cls, file, count): @@ -694,4 +735,124 @@ class ntfy: print(f"#ntfy Error: {error}") return False else: - return True \ No newline at end of file + return True + +class logger: + def log_image_download(self, image_path): + if image_path == "": + logging.info(f"icon file exists: IMAGE_STORE {image_path}") + else: + logging.info(f"file exists: IMAGE_STORE {image_path}") + + def log_image_not_downloaded(self, image_path): + logging.info(f"file does not exist: IMAGE_STORE {image_path}") + +class ItemIconUtils: + def __init__(self, path, icon_item=None): + if icon_item != None: self.icon_item = icon_item + else: self.icon_item = [] + self.path = path + if path != None: + self.read_icon_json() + + def add(self, name, path, size, md5, url=None): + self.icon_item.append({ 'name': name, 'path': path, 'size': size, 'md5':md5, 'url': url }) + + def toString(self): + return self.icon_item + + def _get_values(self, key): return [ item.get(key) for item in self.icon_item ] + + def get_names(self): return self._get_values("name") + + def get_file_count_by_md5(self, md5): + md5_list = self._get_values("md5") + md5_count = md5_list.count(md5) + if md5_count > 1: + logging.info(f" {md5_list} == 存在md5重复文件") + return md5_count + + def get_file_by_key_value(self, key, value): + list_item = [] + for item in self.icon_item: + if item.get(key) == value: return list_item.append(item) + return list_item + + def get_file_by_md5(self, md5): + list_item = [] + for item in self.icon_item: + if item.get("md5") == md5 and self.get_file_count_by_md5(md5) == 1: list_item.append(item) + return list_item + + def get_file_by_url(self, url): return self.get_file_by_key_value('url', url) + + def update_file_by_md5_to_url(self, md5, url): + file_item = self.get_file_by_md5(md5) + if len(file_item) > 1 : + logging.info(f" {file_item} == 存在多个结果") + if len(file_item) == 1: + file_item = file_item[0] + self.icon_item.remove(file_item) + file_item['url'] = url + self.icon_item.append(file_item) + return self.write_icon_json() + + def get_files_info(self, directory, filter_suffix=None): + for root, dirs, files in os.walk(directory): + for file in files: + name, suffix = os.path.splitext(file) + if suffix != filter_suffix: + file_path = os.path.join(root, file) + file_size = os.path.getsize(file_path) + self.add(name= name, path=file_path, size=file_size, md5=fileUtils.get_file_md5(file_path)) + return self.toString() + + def write_icon_json(self): + if self.path != None: + self.get_files_info(self.path, filter_suffix='.json') + fileUtils.write_json(os.path.join(self.path,"icons.json"), self.toString()) + + def read_icon_json(self): + if self.path != None: self.icon_item = fileUtils.read_json(os.path.join(self.path, "icons.json")) + return self.icon_item + +class DBUtils: + + def __init__(self, db_name, suffix="json"): + self.db = TinyDB(ComicPath().getDirConfDefault(db_name,suffix=suffix)) + + @classmethod + def init_db(cls,db_name): + return TinyDB(ComicPath.getDirConfDefault(db_name,suffix="json")) + + @classmethod + def set(cls,name,progress,db_name): + if name == None or progress == None or db_name == None: + print("dbUtils set 数据为空") + return False + db = cls.init_db(db_name) + comic = Query() + if len(db.search(comic.name == name)) == 0: db.insert({"name":name,"progress":progress}) + else: db.update({"progress":progress},comic.name== name) + msg = "失败" + if cls.query(name,progress,db_name): msg = "成功" + logger.debug(f"设置{msg}, name={name} value={progress} db={db_name}") + + @classmethod + def query(cls,name,progress=None,db_name=None): + result = False + db = cls.init_db(db_name) + if db == None: return None + data = db.search(Query().name == name) + logger.debug(f"result query= {data}") + if progress != None: + try: + if len(db.search((Query().name == name) & (Query().progress == progress))) != 0: result = True + except Exception as e: + print(e) + return result + + @classmethod + def remove(cls,name,db_name=None): + db = cls.init_db(db_name) + db.remove(Query().name == name) \ No newline at end of file