diff --git a/Comics/pipelines.py b/Comics/pipelines.py index dd74900..b7f66ac 100644 --- a/Comics/pipelines.py +++ b/Comics/pipelines.py @@ -7,42 +7,17 @@ # useful for handling different item types with a single interface import os,requests,re,scrapy,logging from Comics import settings -from Comics.spiders.utils.FileUtils import imageUtils -from Comics.spiders.utils.Constant import ComicPath +from Comics.utils.FileUtils import imageUtils +from Comics.utils.Constant import ComicPath from Comics.items import ComicItem from Comics.items import ImageItem -from scrapy.pipelines.files import FilesPipeline -from scrapy.pipelines.images import ImagesPipeline -from itemadapter import ItemAdapter from scrapy.pipelines.images import ImagesPipeline +from scrapy.exporters import XmlItemExporter class ComicsPipeline: def open_spider(self,spider): self.fp = open('book.json','w',encoding='utf-8') - def comic_process(self,images): - count = 1 - scramble_count = 0 - #(files_name,images_url) = [[],[]] - list_image_item = [] - for image in images: - (image_src,scramble) = [image.get("src"),image.get("scramble")] - count_image = "{:0>3d}".format(count) - image_src_suffix = "."+str(image_src).split(".")[-1] - image_file_name = count_image+image_src_suffix - if scramble: - de_str = str(image_src).split("/")[-1].replace(image_src_suffix,"==") - blocks_num = imageUtils.encodeImage(de_str) - image_file_name = ComicPath.getFileScrambleImageName(count=count_image,block=blocks_num,suffix=image_src_suffix) - scramble_count += 1 - #files_name.append(image_file_name) - #images_url.append(image_src) - #downUtils.putDownImageUrlDirFile(image_src,ComicPath.getDirComicChapter(),image_file_name) - list_image_item.append(ImageItem(image_name=image_file_name,image_url=image_src,image_path=image_file_name)) - yield ImageItem(image_name=image_file_name,image_url=image_src,image_path=image_file_name) - count+=1 - #yield list_image_item - # item就是yield后面的对象 def process_item(self, item, spider): self.fp.write(str(item)) @@ -58,7 +33,6 @@ class ImageParsePipeline: list_img = item['list_img'] count = 1 scramble_count = 0 - #(files_name,images_url) = [[],[]] list_image_item = [] for image in list_img: (image_src,scramble) = [image.get("src"),image.get("scramble")] @@ -70,21 +44,19 @@ class ImageParsePipeline: blocks_num = imageUtils.encodeImage(de_str) scramble_image_file_name = ComicPath.getFileScrambleImageName(count=count_image,block=blocks_num,suffix=image_src_suffix) scramble_count += 1 - #files_name.append(image_file_name) - #images_url.append(image_src) - #downUtils.putDownImageUrlDirFile(image_src,ComicPath.getDirComicChapter(),image_file_name) image_path = os.path.join(item['name'],item['chapter'],scramble_image_file_name) image_path = ComicPath.ChineseConvert(image_path) list_image_item.append(ImageItem(image_name=image_file_name,image_url=image_src,image_path=image_path)) - #ImageItem(image_name=image_file_name,image_url=image_src,image_path=image_file_name) count+=1 return list_image_item class ImgDownloadPipeline(ImagesPipeline): def file_path(self, request, response=None, info=None, *, item=None): image = request.meta['item'] - return image['image_path'] - #return '%s/%s' % (name,chapter) + image_path = image['image_path'] + en_image_path = os.path.join(os.path.dirname(image_path),image['image_name']) + if os.path.exists(en_image_path): return en_image_path + else: return image_path def get_media_requests(self, item, info): for image in item: @@ -99,4 +71,5 @@ class ImgDownloadPipeline(ImagesPipeline): img_path = os.path.join(settings.IMAGES_STORE,img['path']) #解密图片 imageUtils.deScrambleImagesByPath(img_path) + XmlItemExporter() return item diff --git a/Comics/settings.py b/Comics/settings.py index 092ad1c..93011e8 100644 --- a/Comics/settings.py +++ b/Comics/settings.py @@ -66,7 +66,7 @@ COOKIES_ENABLED = False DOWNLOADER_MIDDLEWARES = { # 'Comics.middlewares.ComicsDownloaderMiddleware': 543, # 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500, -# 'Comics.middlewares.ProxyMiddleware' : 100, + 'Comics.middlewares.ProxyMiddleware' : 100, 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 400, } diff --git a/Comics/spiders/rm_comic.py b/Comics/spiders/rm_comic.py index 8f04918..4c7ba47 100644 --- a/Comics/spiders/rm_comic.py +++ b/Comics/spiders/rm_comic.py @@ -1,6 +1,6 @@ import scrapy,json,requests from Comics.items import ComicItem -from Comics.spiders.utils.FileUtils import CommonUtils +from Comics.utils.FileUtils import CommonUtils import threading import toml diff --git a/Comics/spiders/utils/__pycache__/CommonUtils.cpython-37.pyc b/Comics/spiders/utils/__pycache__/CommonUtils.cpython-37.pyc deleted file mode 100644 index 851ef02..0000000 Binary files a/Comics/spiders/utils/__pycache__/CommonUtils.cpython-37.pyc and /dev/null differ diff --git a/Comics/utils/Comic.py b/Comics/utils/Comic.py new file mode 100644 index 0000000..7a11e38 --- /dev/null +++ b/Comics/utils/Comic.py @@ -0,0 +1,248 @@ +import json,re +from opencc import OpenCC +from queue import Queue +from utils.OldUtils import OldUtils + +class Comic: + # ComicInfo.xml and ComicChapter.json bengin + # value origin node dep required + dict_chapter = [None,None,"Title","章节名",True] + dict_comic_name = [None,None,"Series","漫画名",True] + dict_number = [None,None,"Number","编号",True] + dict_comic_names = [None,None,"SeriesGroup","别名",False] + dict_dep = [None,None,"Summary","概述",True] + dict_year = [None,None,"Year","年",False] + dict_month = [None,None,"Month","月",False] + dict_day = [None,None,"Day","日",False] + dict_author = [None,None,"Writer","作者",True] + dict_cbs = [None,None,"Publisher","出版社",False] + dict_genre = [None,None,"Genre","流派",True] + dict_tags = [None,None,"Tags","标签",True] + dict_homepage = [None,None,"Web","主页",False] + dict_page_count = [None,None,"PageCount","总页数",True] + dict_language = [None,None,"LanguageISO","语言",True] + dict_agerating = [None,None,"AgeRating","年龄分级",False] + dict_pages = [None,None,"Pages","页码",True] + CURRENT_DOWN_LINK = None + # ComicInfo.xml and ComicChapter.json end + dict_icon = [None,None,"Icon","图标",True] + dict_chapter_imgs = [None,None,"ChapterImgs","图像",True] + #主页 + dict_list_chapter = [None,None,"ListChapter","全部章节名",True] + (update_at,current_chapter_img,file_chapter_imgs) = [None,None,None] + + + #繁体中文转简体中文 + @classmethod + def ChineseConvert(cls, text,convert='t2s'): return OpenCC(convert).convert(str(text)) + #处理成符合规定的文件名 + @classmethod + def fixFileName(cls,filename,replace=None): + if not isinstance(filename,str): return filename + intab = r'[?*/\|.:><]' + str_replace = "" + if replace != None: str_replace = replace + filename = re.sub(intab, str_replace, filename) + count = 1 + while True: + str_file = filename[0-count] + if str_file == " ": count += 1 + else: + filename = filename[0:len(filename)+1-count] + break + return filename + + @classmethod + def setValue(cls,value): + if value != None: value = cls.ChineseConvert(value) + return value + + @classmethod + def setField(cls,field,value,origin=True,convert=True): + if value != None: + if origin: + field[1] = value + if convert: value = cls.ChineseConvert(value) + field[0] = value + return field + + @classmethod + def getFieldValue(cls,field): + if field == None: return None + return field[0] + @classmethod + def setFieldOrigin(cls,filed,origin): + filed[1] = origin + return filed + @classmethod + def getFieldOrigin(cls,filed): return filed[1] + @classmethod + def getFieldNode(cls,filed): return filed[2] + + @classmethod + def getValue(cls,field,exec=None): + if exec != None: return cls.parseExec(field,exec=exec) + return field + #章节名 + @classmethod + def setChapterName(cls,value,exec=None): + value = cls.fixFileName(cls.parseExec(value,exec=exec)) + OldUtils.setOldChapter(value) + cls.dict_chapter = cls.setField(cls.dict_chapter,value) + + @classmethod + def getChapterName(cls): return cls.getFieldValue(cls.dict_chapter) + @classmethod + def getOriginChapterName(cls): return cls.getFieldOrigin(cls.dict_chapter) + + #漫画名 + @classmethod + def setComicName(cls,value,exec=None): + value = cls.fixFileName(cls.parseExec(value,exec=exec)) + OldUtils.setOldComicName(value) + cls.dict_comic_name = cls.setField(cls.dict_comic_name,value) + + @classmethod + def getComicName(cls): return cls.getFieldValue(cls.dict_comic_name) + @classmethod + def getOriginComicName(cls): return cls.getFieldOrigin(cls.dict_comic_name) + #编号 + @classmethod + def setNumber(cls,value): cls.dict_number = cls.setField(cls.dict_number,value) + @classmethod + def getNumber(cls): return cls.getFieldValue(cls.dict_number) + #概述 + @classmethod + def setDep(cls,value,exec=None): + cls.dict_dep = cls.setField(cls.dict_dep,cls.parseExec(value,exec=exec)) + @classmethod + def getDep(cls): return cls.getFieldValue(cls.dict_dep) + #作者 + @classmethod + def setAuthor(cls,value): cls.dict_author = cls.setField(cls.dict_author,value) + @classmethod + def getAuthor(cls): return cls.getFieldValue(cls.dict_author) + #流派 + @classmethod + def setGenre(cls,value): cls.dict_genre = cls.setField(cls.dict_genre,value) + @classmethod + def getGenre(cls): return cls.getFieldValue(cls.dict_genre) + #语言 + @classmethod + def setLanguage(cls,value): cls.dict_language = cls.setField(cls.dict_language,value) + @classmethod + def getLanguage(cls): return cls.getFieldValue(cls.dict_language) + #年龄分级 + @classmethod + def setAgeRating(cls,value): cls.dict_agerating = cls.setField(cls.dict_agerating,value) + @classmethod + def getAgeRating(cls): return cls.getFieldValue(cls.dict_agerating) + #标签 + @classmethod + def setTags(cls,value): cls.dict_tags = cls.setField(cls.dict_tags,value) + @classmethod + def getTags(cls): return cls.getFieldValue(cls.dict_tags) + #总页数 + @classmethod + def setPageCount(cls,value): cls.dict_page_count = cls.setField(cls.dict_page_count,value) + @classmethod + def getPageCount(cls): return cls.getFieldValue(cls.dict_page_count) + + #------------------------------------------------------------------------ + @classmethod + def parseExec(cls,data,exec,item=True): + if data !=None and exec != None: + dots = str(exec).split(".") + if not isinstance(data,dict): data = json.loads(data) + for dot in dots: + data = data.get(dot) + return data + @classmethod + def setHomePage(cls,value): cls.dict_homepage = cls.setField(cls.dict_homepage,value) + @classmethod + def getHomePage(cls): return cls.getFieldValue(cls.dict_homepage) + @classmethod + def setIcon(cls,value): cls.dict_icon = cls.setField(cls.dict_icon,value,convert=False) + @classmethod + def getIcon(cls): return cls.getFieldValue(cls.dict_icon) + @classmethod + def setListChapter(cls,value): cls.dict_list_chapter = cls.setField(cls.dict_list_chapter,value,convert=False) + @classmethod + def getListChapter(cls): return cls.getFieldValue(cls.dict_list_chapter) + @classmethod + def getLenChapters(cls): return len(cls.getListChapter()) + @classmethod + def setChapterImgs(cls,value,exec=None,item=None): + cls.dict_chapter_imgs = cls.setField(cls.dict_chapter_imgs,cls.parseExec(value,exec=exec,item=item),convert=False) + @classmethod + def getChapterImgs(cls): return cls.getFieldValue(cls.dict_chapter_imgs) + @classmethod + def setUpdateAt(cls,value): cls.update_at = value + @classmethod + def getUpdateAt(cls): return cls.update_at + @classmethod + def setCurrentChapterImg(cls,value): cls.current_chapter_img = value + @classmethod + def getCurrentChapterImg(cls): return cls.current_chapter_img + @classmethod + def setChapterFilesName(cls,value): cls.file_chapter_imgs= value + @classmethod + def getChapterFilesName(cls): return cls.file_chapter_imgs + @classmethod + def setCurrentDownLink(cls,value): cls.CURRENT_DOWN_LINK = value + @classmethod + def getCurrentDownLink(cls): return cls.CURRENT_DOWN_LINK + +class ListComic: + LIST_COMIC_QUEUE = Queue() + (LIST_COMIC_NAME,LIST_COMIC_LINK,LIST_COMIC_UPDATEAT) = [None,None,None] + + @classmethod + def setListComicsLinksUpdateAt(cls,names,links,update_at): + if isinstance(names,list) and isinstance(links,list) and isinstance(update_at,list): + for x in range(0,len(names)): + cls.LIST_COMIC_QUEUE.put([names[x],links[x],update_at[x]]) + @classmethod + def getListComicsLinksUpdateAt(cls): + result = None + if cls.LIST_COMIC_NAME != None and cls.LIST_COMIC_LINK != None: + cls.setListComicsLinksUpdateAt(cls.LIST_COMIC_NAME,cls.LIST_COMIC_LINK,cls.LIST_COMIC_UPDATEAT) + (cls.LIST_COMIC_NAME,cls.LIST_COMIC_LINK,cls.LIST_COMIC_UPDATEAT) = [None,None,None] + if not cls.LIST_COMIC_QUEUE.empty(): result = cls.LIST_COMIC_QUEUE.get(False) + return result + + @classmethod + def addListComicChapterLink(cls,name,link,update_at): + if name != None and link != None: + cls.LIST_COMIC_QUEUE.put(name,link,update_at) + + @classmethod + def getListValue(cls,result,type,start_add=None,result_type="list"): + if result == None: return None + if type == None: return result + if result_type == "list" and type != None: + data = [] + for x in range(0, len(result)): + if start_add != None: + data.append(start_add+result[x].get(type)) + else: + data.append(result[x].get(type)) + return data + return result + + @classmethod + def setListComicName(cls,value,type=None): cls.LIST_COMIC_NAME = cls.getListValue(value,type) + @classmethod + def getListComicName(cls): return cls.LIST_COMIC_NAME + @classmethod + def setListComicChapterLink(cls,value,type=None,start_add=None): cls.LIST_COMIC_LINK = cls.getListValue(value,type,start_add) + @classmethod + def getListComicChapterLink(cls): return cls.LIST_COMIC_LINK + @classmethod + def setListComicUpdateAt(cls,value,type=None): cls.LIST_COMIC_UPDATEAT = cls.getListValue(value,type) + @classmethod + def getListComicUpdateAt(cls): return cls.LIST_COMIC_UPDATEAT + @classmethod + def getListComicChapterLink(cls): return cls.LIST_COMIC_QUEUE.get(False) + + #domain end.... \ No newline at end of file diff --git a/Comics/utils/ComicInfo.py b/Comics/utils/ComicInfo.py new file mode 100644 index 0000000..03c6755 --- /dev/null +++ b/Comics/utils/ComicInfo.py @@ -0,0 +1,156 @@ +import json,os +import logging +from xml.dom.minidom import Document +from Comics.utils.Comic import Comic +from Comics.utils.Constant import ComicPath + +class ComicInfoEntity: + @classmethod + def getNodes(cls): + return [Comic.dict_chapter,Comic.dict_comic_name,Comic.dict_number,Comic.dict_comic_names, + Comic.dict_dep,Comic.dict_year,Comic.dict_month,Comic.dict_day,Comic.dict_author, + Comic.dict_cbs,Comic.dict_genre,Comic.dict_tags,Comic.dict_page_count, + Comic.dict_language,Comic.dict_agerating,Comic.dict_pages] + @classmethod + def getJsonNodes(cls): + return [Comic.dict_chapter,Comic.dict_comic_name,Comic.dict_icon,Comic.dict_number, + Comic.dict_comic_names, + Comic.dict_dep,Comic.dict_year,Comic.dict_month,Comic.dict_day,Comic.dict_author, + Comic.dict_cbs,Comic.dict_genre,Comic.dict_tags,Comic.dict_page_count, + Comic.dict_language,Comic.dict_agerating,Comic.dict_pages, + Comic.dict_list_chapter,Comic.dict_chapter_imgs] + +class ComicInfo: + IS_NEW_ICON = False + document = Document() + path_comic_info = None + + @classmethod + def parseExec(cls,data,exec,start_add=None,item=True): + if data !=None and exec != None: + dots = str(exec).split(".") + if not isinstance(data,dict): data = json.loads(data) + for dot in dots: + data = data.get(dot) + if start_add != None and data != None: + data = start_add+data + return data + + @classmethod + def setNodeAndValue(cls,node,value): + if value != None: + if isinstance(value,str): + c_node = cls.document.createElement(node) + child_node = cls.document.createTextNode(value) + c_node.appendChild(child_node) + return c_node + else: return value + return None + + #页数 + @classmethod + def setPages(cls,values=None): + if values == None: values = Comic.getChapterFilesName() + if values != None and isinstance(values,list): + suffix = "."+str(values[0]).split(".")[-1] + join_list=",".join(values).replace(suffix,"") + values = join_list.split(",") + Comic.setPageCount(len(values)+1 if cls.IS_NEW_ICON else len(values)) + root_node = cls.document.createElement("Pages") + if cls.IS_NEW_ICON: + #添加封面 + icon_node = cls.document.createElement("Page") + icon_node.setAttribute("Image",ComicPath.COMIC_ICON_NAME) + icon_node.setAttribute("Type","FrontCover") + root_node.appendChild(icon_node) + for page in values: + c_node = cls.document.createElement("Page") + page = page.split("_")[-1] + c_node.setAttribute("Image",page) + root_node.appendChild(c_node) + Comic.dict_pages = Comic.setField(Comic.dict_pages,root_node,convert=False) + + @classmethod + def getBaseUrl(cls,url=None): + if url == None: + url = Comic.getHomePage() + (num,index) = [3,0] + for x in range(0, num): + index = str(url).find("/",index)+1 + return url[0:index-1] + + #XML根文档 + @classmethod + def root_node(cls,root_value): return cls.document.createElement(root_value) + + @classmethod + def add_nodes(cls,root,list_value): + if len(list_value) == 0: return list_value + for value in list_value: + #Comic.chapter + if value[0] == None and value[4]: + #数据为空 value[0], 但不允许为空value[4] = False + msg = f"#数据为空 key={value[3]} value[0]={value[0]}, 但不允许为空value[4]={value[4]}" + logger.error(msg) + exit() + if value[0] != None: root.appendChild(cls.setNodeAndValue(value[2],value[0])) + + @classmethod + def initComicInfoXML(cls): + cls.setPages() + + @classmethod + def writeComicInfoXML(cls,overlay=False): + save_path = ComicPath.getPathComicInfoXML() + if os.path.exists(save_path): + if overlay: + os.remove(save_path) + logging.info(f"已存在文件,已删除... {save_path}") + else: + logging.info(f"已存在文件,跳过中... {save_path}") + return True + cls.initComicInfoXML() + root = cls.root_node("ComicInfo") + new_document = Document() + new_document.appendChild(root) + cls.add_nodes(root,ComicInfoEntity.getNodes()) + with open(save_path, "w", encoding="utf-8") as fo: + new_document.writexml(fo, indent='', addindent='\t', newl='\n', encoding="utf-8") + fo.close() + logging.info(f"已生成文件... {save_path}") + + @classmethod + def setComicInfo(cls,comicname=None,homepage=None,alias=None,author=None,icon=None,tags=None, + dep=None,genre=None,lang=None,age_rating=None,chapters=None,current_chapter_img=None): + author = ",".join(set(str(str(author).replace("&",",").replace(" ",",")).split(","))) + Comic.setHomePage(homepage) + Comic.setIcon(icon) + Comic.setListChapter(chapters) + #Comic.setUpdateAt(update_at) + Comic.setComicName(str(comicname)) + #if alias != None: comicInfo.setComicNames(alias) + Comic.setAuthor(author) + Comic.setTags(tags) + Comic.setDep(dep) + #comicInfo.setCBS("韩漫") + if genre != None: Comic.setGenre(genre) + Comic.setLanguage(lang) + Comic.setAgeRating(age_rating) + Comic.setCurrentChapterImg(current_chapter_img) + + @classmethod + def writeJson(cls): + dict_data = {} + nodes = ComicInfoEntity.getJsonNodes() + for node in nodes: + key = Comic.getFieldNode(node) + value = Comic.getFieldOrigin(node) + if isinstance(value,list): + value = ",".join(value) + if key != None and isinstance(value,str): + child_dict = { key : value} + dict_data.update(child_dict) + s = json.dumps(dict_data,ensure_ascii=True) + logging.debug(f"json={s}") + with open(ComicPath.getPathConfComicChapterJson(mkdir=True),"w") as fs: + fs.write(s) \ No newline at end of file diff --git a/Comics/spiders/utils/Constant.py b/Comics/utils/Constant.py similarity index 100% rename from Comics/spiders/utils/Constant.py rename to Comics/utils/Constant.py diff --git a/Comics/spiders/utils/FileUtils.py b/Comics/utils/FileUtils.py similarity index 100% rename from Comics/spiders/utils/FileUtils.py rename to Comics/utils/FileUtils.py diff --git a/Comics/utils/OldUtils.py b/Comics/utils/OldUtils.py new file mode 100644 index 0000000..5bc9938 --- /dev/null +++ b/Comics/utils/OldUtils.py @@ -0,0 +1,16 @@ + +class OldUtils: + old_comic_name=None + old_chapter = None + + @classmethod + def setOldComicName(cls,value): cls.old_comic_name = value + + @classmethod + def setOldChapter(cls,value): cls.old_chapter=value + + @classmethod + def getOldComicName(cls): return cls.old_comic_name + + @classmethod + def getOldChapter(cls): return cls.old_chapter \ No newline at end of file