update

Comics/pipelines.py

@@ -7,42 +7,17 @@
 # useful for handling different item types with a single interface
 import os,requests,re,scrapy,logging
 from Comics import settings
-from Comics.spiders.utils.FileUtils import imageUtils
-from Comics.spiders.utils.Constant import ComicPath
+from Comics.utils.FileUtils import imageUtils
+from Comics.utils.Constant import ComicPath
 from Comics.items import ComicItem
 from Comics.items import ImageItem
-from scrapy.pipelines.files import FilesPipeline
-from scrapy.pipelines.images import ImagesPipeline
-from itemadapter import ItemAdapter
 from scrapy.pipelines.images import ImagesPipeline
+from scrapy.exporters import XmlItemExporter
 
 class ComicsPipeline:
     def open_spider(self,spider):
         self.fp = open('book.json','w',encoding='utf-8')
 
-    def comic_process(self,images):
-        count = 1
-        scramble_count = 0
-        #(files_name,images_url) = [[],[]]
-        list_image_item = []
-        for image in images:
-            (image_src,scramble) = [image.get("src"),image.get("scramble")]
-            count_image = "{:0>3d}".format(count)
-            image_src_suffix = "."+str(image_src).split(".")[-1]
-            image_file_name = count_image+image_src_suffix
-            if scramble:
-                de_str = str(image_src).split("/")[-1].replace(image_src_suffix,"==")
-                blocks_num = imageUtils.encodeImage(de_str)
-                image_file_name = ComicPath.getFileScrambleImageName(count=count_image,block=blocks_num,suffix=image_src_suffix)
-                scramble_count += 1
-            #files_name.append(image_file_name)
-            #images_url.append(image_src)
-            #downUtils.putDownImageUrlDirFile(image_src,ComicPath.getDirComicChapter(),image_file_name)
-            list_image_item.append(ImageItem(image_name=image_file_name,image_url=image_src,image_path=image_file_name))
-            yield ImageItem(image_name=image_file_name,image_url=image_src,image_path=image_file_name)
-            count+=1
-        #yield list_image_item
-            
     # item就是yield后面的对象
     def process_item(self, item, spider):
         self.fp.write(str(item))
@@ -58,7 +33,6 @@ class ImageParsePipeline:
             list_img = item['list_img']
             count = 1
             scramble_count = 0
-            #(files_name,images_url) = [[],[]]
             list_image_item = []
             for image in list_img:
                 (image_src,scramble) = [image.get("src"),image.get("scramble")]
@@ -70,21 +44,19 @@ class ImageParsePipeline:
                     blocks_num = imageUtils.encodeImage(de_str)
                     scramble_image_file_name = ComicPath.getFileScrambleImageName(count=count_image,block=blocks_num,suffix=image_src_suffix)
                     scramble_count += 1
-                #files_name.append(image_file_name)
-                #images_url.append(image_src)
-                #downUtils.putDownImageUrlDirFile(image_src,ComicPath.getDirComicChapter(),image_file_name)
                 image_path = os.path.join(item['name'],item['chapter'],scramble_image_file_name)
                 image_path = ComicPath.ChineseConvert(image_path)
                 list_image_item.append(ImageItem(image_name=image_file_name,image_url=image_src,image_path=image_path))
-                #ImageItem(image_name=image_file_name,image_url=image_src,image_path=image_file_name)
                 count+=1
             return list_image_item
         
 class ImgDownloadPipeline(ImagesPipeline):
     def file_path(self, request, response=None, info=None, *, item=None):
         image = request.meta['item']
-        return image['image_path']
-        #return '%s/%s' % (name,chapter)
+        image_path = image['image_path']
+        en_image_path = os.path.join(os.path.dirname(image_path),image['image_name'])
+        if os.path.exists(en_image_path): return en_image_path
+        else: return image_path
 
     def get_media_requests(self, item, info):
         for image in item:
@@ -99,4 +71,5 @@ class ImgDownloadPipeline(ImagesPipeline):
                 img_path = os.path.join(settings.IMAGES_STORE,img['path'])
                 #解密图片
                 imageUtils.deScrambleImagesByPath(img_path)
+        XmlItemExporter()
         return item

Comics/settings.py

@@ -66,7 +66,7 @@ COOKIES_ENABLED = False
 DOWNLOADER_MIDDLEWARES = {
 #    'Comics.middlewares.ComicsDownloaderMiddleware': 543,
 #    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
-#    'Comics.middlewares.ProxyMiddleware' : 100,
+    'Comics.middlewares.ProxyMiddleware' : 100,
     'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 400,
 }
 

Comics/spiders/rm_comic.py

@@ -1,6 +1,6 @@
 import scrapy,json,requests
 from Comics.items import ComicItem
-from Comics.spiders.utils.FileUtils import CommonUtils
+from Comics.utils.FileUtils import CommonUtils
 import threading
 import toml
 

Comics/utils/Comic.py

@@ -0,0 +1,248 @@
+import json,re
+from opencc import OpenCC
+from queue import Queue
+from utils.OldUtils import OldUtils
+
+class Comic:
+    # ComicInfo.xml and ComicChapter.json bengin 
+    #            value   origin  node dep required
+    dict_chapter = [None,None,"Title","章节名",True]
+    dict_comic_name = [None,None,"Series","漫画名",True]
+    dict_number = [None,None,"Number","编号",True]
+    dict_comic_names = [None,None,"SeriesGroup","别名",False]
+    dict_dep = [None,None,"Summary","概述",True]
+    dict_year = [None,None,"Year","年",False]
+    dict_month = [None,None,"Month","月",False]
+    dict_day = [None,None,"Day","日",False]
+    dict_author = [None,None,"Writer","作者",True]
+    dict_cbs = [None,None,"Publisher","出版社",False]
+    dict_genre = [None,None,"Genre","流派",True]
+    dict_tags = [None,None,"Tags","标签",True]
+    dict_homepage = [None,None,"Web","主页",False]
+    dict_page_count = [None,None,"PageCount","总页数",True]
+    dict_language = [None,None,"LanguageISO","语言",True]
+    dict_agerating = [None,None,"AgeRating","年龄分级",False]
+    dict_pages = [None,None,"Pages","页码",True]
+    CURRENT_DOWN_LINK = None
+    # ComicInfo.xml and ComicChapter.json end 
+    dict_icon = [None,None,"Icon","图标",True]
+    dict_chapter_imgs = [None,None,"ChapterImgs","图像",True]
+    #主页
+    dict_list_chapter = [None,None,"ListChapter","全部章节名",True]
+    (update_at,current_chapter_img,file_chapter_imgs) = [None,None,None]
+    
+    
+    #繁体中文转简体中文
+    @classmethod
+    def ChineseConvert(cls, text,convert='t2s'): return OpenCC(convert).convert(str(text))
+    #处理成符合规定的文件名
+    @classmethod
+    def fixFileName(cls,filename,replace=None):
+        if not isinstance(filename,str): return filename
+        intab = r'[?*/\|.:><]'
+        str_replace = ""
+        if replace != None: str_replace = replace
+        filename = re.sub(intab, str_replace, filename)
+        count = 1
+        while True:
+            str_file = filename[0-count]
+            if str_file == " ": count += 1
+            else:
+                filename = filename[0:len(filename)+1-count]
+                break
+        return filename
+    
+    @classmethod
+    def setValue(cls,value):
+       if value != None: value = cls.ChineseConvert(value)
+       return value
+    
+    @classmethod
+    def setField(cls,field,value,origin=True,convert=True):
+       if value != None:
+           if origin:
+              field[1] = value
+           if convert: value = cls.ChineseConvert(value)
+           field[0] = value
+       return field
+   
+    @classmethod
+    def getFieldValue(cls,field): 
+        if field == None: return None
+        return field[0]
+    @classmethod
+    def setFieldOrigin(cls,filed,origin):
+        filed[1] = origin
+        return filed
+    @classmethod
+    def getFieldOrigin(cls,filed): return filed[1]
+    @classmethod
+    def getFieldNode(cls,filed): return filed[2]
+    
+    @classmethod
+    def getValue(cls,field,exec=None):
+        if exec != None: return cls.parseExec(field,exec=exec)
+        return field
+    #章节名
+    @classmethod
+    def setChapterName(cls,value,exec=None):
+        value = cls.fixFileName(cls.parseExec(value,exec=exec))
+        OldUtils.setOldChapter(value)
+        cls.dict_chapter = cls.setField(cls.dict_chapter,value)
+        
+    @classmethod
+    def getChapterName(cls): return cls.getFieldValue(cls.dict_chapter)
+    @classmethod
+    def getOriginChapterName(cls): return cls.getFieldOrigin(cls.dict_chapter)
+
+    #漫画名
+    @classmethod
+    def setComicName(cls,value,exec=None):
+        value = cls.fixFileName(cls.parseExec(value,exec=exec))
+        OldUtils.setOldComicName(value)
+        cls.dict_comic_name = cls.setField(cls.dict_comic_name,value)
+ 
+    @classmethod
+    def getComicName(cls): return cls.getFieldValue(cls.dict_comic_name)
+    @classmethod
+    def getOriginComicName(cls): return cls.getFieldOrigin(cls.dict_comic_name)
+    #编号
+    @classmethod
+    def setNumber(cls,value): cls.dict_number = cls.setField(cls.dict_number,value)
+    @classmethod
+    def getNumber(cls): return cls.getFieldValue(cls.dict_number)
+    #概述
+    @classmethod
+    def setDep(cls,value,exec=None): 
+        cls.dict_dep = cls.setField(cls.dict_dep,cls.parseExec(value,exec=exec))
+    @classmethod
+    def getDep(cls): return cls.getFieldValue(cls.dict_dep)
+    #作者
+    @classmethod
+    def setAuthor(cls,value): cls.dict_author = cls.setField(cls.dict_author,value) 
+    @classmethod
+    def getAuthor(cls): return cls.getFieldValue(cls.dict_author)
+    #流派
+    @classmethod
+    def setGenre(cls,value): cls.dict_genre = cls.setField(cls.dict_genre,value) 
+    @classmethod
+    def getGenre(cls): return cls.getFieldValue(cls.dict_genre) 
+    #语言
+    @classmethod
+    def setLanguage(cls,value): cls.dict_language = cls.setField(cls.dict_language,value) 
+    @classmethod
+    def getLanguage(cls): return cls.getFieldValue(cls.dict_language) 
+    #年龄分级
+    @classmethod
+    def setAgeRating(cls,value): cls.dict_agerating = cls.setField(cls.dict_agerating,value) 
+    @classmethod
+    def getAgeRating(cls): return cls.getFieldValue(cls.dict_agerating) 
+    #标签
+    @classmethod
+    def setTags(cls,value): cls.dict_tags = cls.setField(cls.dict_tags,value) 
+    @classmethod
+    def getTags(cls): return cls.getFieldValue(cls.dict_tags)
+    #总页数
+    @classmethod
+    def setPageCount(cls,value): cls.dict_page_count = cls.setField(cls.dict_page_count,value) 
+    @classmethod
+    def getPageCount(cls): return cls.getFieldValue(cls.dict_page_count) 
+
+    #------------------------------------------------------------------------
+    @classmethod
+    def parseExec(cls,data,exec,item=True):
+        if data !=None and exec != None:
+            dots = str(exec).split(".")
+            if not isinstance(data,dict): data = json.loads(data)
+            for dot in dots:
+                data = data.get(dot)
+        return data
+    @classmethod
+    def setHomePage(cls,value): cls.dict_homepage = cls.setField(cls.dict_homepage,value) 
+    @classmethod
+    def getHomePage(cls): return cls.getFieldValue(cls.dict_homepage)
+    @classmethod
+    def setIcon(cls,value): cls.dict_icon = cls.setField(cls.dict_icon,value,convert=False)
+    @classmethod
+    def getIcon(cls): return cls.getFieldValue(cls.dict_icon)
+    @classmethod
+    def setListChapter(cls,value): cls.dict_list_chapter = cls.setField(cls.dict_list_chapter,value,convert=False)
+    @classmethod
+    def getListChapter(cls): return cls.getFieldValue(cls.dict_list_chapter)
+    @classmethod
+    def getLenChapters(cls): return len(cls.getListChapter())
+    @classmethod
+    def setChapterImgs(cls,value,exec=None,item=None): 
+        cls.dict_chapter_imgs = cls.setField(cls.dict_chapter_imgs,cls.parseExec(value,exec=exec,item=item),convert=False)
+    @classmethod
+    def getChapterImgs(cls): return cls.getFieldValue(cls.dict_chapter_imgs)
+    @classmethod
+    def setUpdateAt(cls,value): cls.update_at = value
+    @classmethod
+    def getUpdateAt(cls): return cls.update_at
+    @classmethod
+    def setCurrentChapterImg(cls,value): cls.current_chapter_img = value
+    @classmethod
+    def getCurrentChapterImg(cls): return cls.current_chapter_img
+    @classmethod
+    def setChapterFilesName(cls,value): cls.file_chapter_imgs= value
+    @classmethod
+    def getChapterFilesName(cls): return cls.file_chapter_imgs
+    @classmethod
+    def setCurrentDownLink(cls,value): cls.CURRENT_DOWN_LINK = value
+    @classmethod
+    def getCurrentDownLink(cls): return cls.CURRENT_DOWN_LINK
+    
+class ListComic:
+    LIST_COMIC_QUEUE = Queue()
+    (LIST_COMIC_NAME,LIST_COMIC_LINK,LIST_COMIC_UPDATEAT) = [None,None,None] 
+    
+    @classmethod
+    def setListComicsLinksUpdateAt(cls,names,links,update_at):
+        if isinstance(names,list) and isinstance(links,list) and isinstance(update_at,list):
+            for x in range(0,len(names)):
+                cls.LIST_COMIC_QUEUE.put([names[x],links[x],update_at[x]])
+    @classmethod
+    def getListComicsLinksUpdateAt(cls):
+        result = None
+        if cls.LIST_COMIC_NAME != None and cls.LIST_COMIC_LINK != None:
+            cls.setListComicsLinksUpdateAt(cls.LIST_COMIC_NAME,cls.LIST_COMIC_LINK,cls.LIST_COMIC_UPDATEAT)
+            (cls.LIST_COMIC_NAME,cls.LIST_COMIC_LINK,cls.LIST_COMIC_UPDATEAT) = [None,None,None]
+        if not cls.LIST_COMIC_QUEUE.empty(): result = cls.LIST_COMIC_QUEUE.get(False)
+        return result
+        
+    @classmethod
+    def addListComicChapterLink(cls,name,link,update_at):
+        if name != None and link != None:
+            cls.LIST_COMIC_QUEUE.put(name,link,update_at)
+            
+    @classmethod
+    def getListValue(cls,result,type,start_add=None,result_type="list"):
+        if result == None: return None
+        if type == None: return result
+        if result_type == "list" and type != None:
+            data = []
+            for x in range(0, len(result)):
+                if start_add != None:
+                    data.append(start_add+result[x].get(type))
+                else:
+                    data.append(result[x].get(type))
+            return data
+        return result
+    
+    @classmethod
+    def setListComicName(cls,value,type=None): cls.LIST_COMIC_NAME = cls.getListValue(value,type)
+    @classmethod
+    def getListComicName(cls): return cls.LIST_COMIC_NAME
+    @classmethod
+    def setListComicChapterLink(cls,value,type=None,start_add=None): cls.LIST_COMIC_LINK = cls.getListValue(value,type,start_add) 
+    @classmethod
+    def getListComicChapterLink(cls): return cls.LIST_COMIC_LINK
+    @classmethod
+    def setListComicUpdateAt(cls,value,type=None): cls.LIST_COMIC_UPDATEAT = cls.getListValue(value,type) 
+    @classmethod
+    def getListComicUpdateAt(cls): return cls.LIST_COMIC_UPDATEAT
+    @classmethod
+    def getListComicChapterLink(cls): return cls.LIST_COMIC_QUEUE.get(False)
+
+    #domain end....

Comics/utils/ComicInfo.py

@@ -0,0 +1,156 @@
+import json,os
+import logging
+from xml.dom.minidom import Document
+from Comics.utils.Comic import Comic
+from Comics.utils.Constant import ComicPath
+
+class ComicInfoEntity:
+    @classmethod
+    def getNodes(cls):
+        return [Comic.dict_chapter,Comic.dict_comic_name,Comic.dict_number,Comic.dict_comic_names,
+                Comic.dict_dep,Comic.dict_year,Comic.dict_month,Comic.dict_day,Comic.dict_author,
+                Comic.dict_cbs,Comic.dict_genre,Comic.dict_tags,Comic.dict_page_count,
+                Comic.dict_language,Comic.dict_agerating,Comic.dict_pages]
+    @classmethod
+    def getJsonNodes(cls):
+        return [Comic.dict_chapter,Comic.dict_comic_name,Comic.dict_icon,Comic.dict_number,
+                Comic.dict_comic_names,
+                Comic.dict_dep,Comic.dict_year,Comic.dict_month,Comic.dict_day,Comic.dict_author,
+                Comic.dict_cbs,Comic.dict_genre,Comic.dict_tags,Comic.dict_page_count,
+                Comic.dict_language,Comic.dict_agerating,Comic.dict_pages,
+                Comic.dict_list_chapter,Comic.dict_chapter_imgs]
+        
+class ComicInfo:
+    IS_NEW_ICON = False 
+    document = Document()
+    path_comic_info = None
+    
+    @classmethod
+    def parseExec(cls,data,exec,start_add=None,item=True):
+        if data !=None and exec != None:
+            dots = str(exec).split(".")
+            if not isinstance(data,dict): data = json.loads(data)
+            for dot in dots:
+                data = data.get(dot)
+        if start_add != None and data != None:
+            data = start_add+data
+        return data
+    
+    @classmethod
+    def setNodeAndValue(cls,node,value):
+       if value != None:
+            if isinstance(value,str):
+                c_node =  cls.document.createElement(node)
+                child_node =  cls.document.createTextNode(value)
+                c_node.appendChild(child_node)
+                return c_node
+            else: return value 
+       return None
+    
+    #页数
+    @classmethod
+    def setPages(cls,values=None):
+       if values == None: values = Comic.getChapterFilesName()
+       if values != None and isinstance(values,list):
+            suffix = "."+str(values[0]).split(".")[-1]
+            join_list=",".join(values).replace(suffix,"")
+            values = join_list.split(",")
+            Comic.setPageCount(len(values)+1 if cls.IS_NEW_ICON else len(values))
+            root_node = cls.document.createElement("Pages")
+            if cls.IS_NEW_ICON:
+                #添加封面
+                icon_node = cls.document.createElement("Page")
+                icon_node.setAttribute("Image",ComicPath.COMIC_ICON_NAME)
+                icon_node.setAttribute("Type","FrontCover")
+                root_node.appendChild(icon_node)
+            for page in values:
+                c_node =  cls.document.createElement("Page")
+                page = page.split("_")[-1]
+                c_node.setAttribute("Image",page)
+                root_node.appendChild(c_node)
+            Comic.dict_pages = Comic.setField(Comic.dict_pages,root_node,convert=False)
+
+    @classmethod
+    def getBaseUrl(cls,url=None):
+        if url == None:
+            url = Comic.getHomePage()
+        (num,index) = [3,0]
+        for x in range(0, num):
+            index = str(url).find("/",index)+1
+        return url[0:index-1]
+    
+    #XML根文档
+    @classmethod
+    def root_node(cls,root_value): return cls.document.createElement(root_value)
+    
+    @classmethod
+    def add_nodes(cls,root,list_value):
+        if len(list_value) == 0: return list_value
+        for value in list_value:
+            #Comic.chapter
+            if value[0] == None and value[4]: 
+                #数据为空 value[0]， 但不允许为空value[4] = False
+                msg = f"#数据为空 key={value[3]} value[0]={value[0]}， 但不允许为空value[4]={value[4]}"
+                logger.error(msg)
+                exit()
+            if value[0] != None: root.appendChild(cls.setNodeAndValue(value[2],value[0]))
+            
+    @classmethod
+    def initComicInfoXML(cls):
+        cls.setPages()
+    
+    @classmethod
+    def writeComicInfoXML(cls,overlay=False):
+        save_path = ComicPath.getPathComicInfoXML()
+        if os.path.exists(save_path):
+            if overlay: 
+                os.remove(save_path)
+                logging.info(f"已存在文件，已删除... {save_path}") 
+            else:
+                logging.info(f"已存在文件，跳过中... {save_path}") 
+                return True
+        cls.initComicInfoXML()
+        root = cls.root_node("ComicInfo")
+        new_document = Document()
+        new_document.appendChild(root)
+        cls.add_nodes(root,ComicInfoEntity.getNodes())
+        with open(save_path, "w", encoding="utf-8") as fo:
+            new_document.writexml(fo, indent='', addindent='\t', newl='\n', encoding="utf-8")
+            fo.close()
+            logging.info(f"已生成文件... {save_path}")
+ 
+    @classmethod
+    def setComicInfo(cls,comicname=None,homepage=None,alias=None,author=None,icon=None,tags=None,
+        dep=None,genre=None,lang=None,age_rating=None,chapters=None,current_chapter_img=None):
+        author = ",".join(set(str(str(author).replace("&",",").replace(" ",",")).split(",")))
+        Comic.setHomePage(homepage)
+        Comic.setIcon(icon)
+        Comic.setListChapter(chapters)
+        #Comic.setUpdateAt(update_at)
+        Comic.setComicName(str(comicname))
+        #if alias != None: comicInfo.setComicNames(alias)
+        Comic.setAuthor(author)
+        Comic.setTags(tags)
+        Comic.setDep(dep)
+        #comicInfo.setCBS("韩漫")
+        if genre != None: Comic.setGenre(genre)
+        Comic.setLanguage(lang)
+        Comic.setAgeRating(age_rating)
+        Comic.setCurrentChapterImg(current_chapter_img)
+    
+    @classmethod
+    def writeJson(cls):
+        dict_data = {}
+        nodes = ComicInfoEntity.getJsonNodes()
+        for node in nodes:
+            key = Comic.getFieldNode(node)
+            value = Comic.getFieldOrigin(node)
+            if isinstance(value,list):
+                value = ",".join(value)
+            if key != None and isinstance(value,str):
+                child_dict = { key : value}
+                dict_data.update(child_dict)
+        s = json.dumps(dict_data,ensure_ascii=True)
+        logging.debug(f"json={s}")
+        with open(ComicPath.getPathConfComicChapterJson(mkdir=True),"w") as fs:
+            fs.write(s)

Comics/utils/OldUtils.py

@@ -0,0 +1,16 @@
+
+class OldUtils:
+    old_comic_name=None
+    old_chapter = None
+
+    @classmethod
+    def setOldComicName(cls,value): cls.old_comic_name = value
+
+    @classmethod
+    def setOldChapter(cls,value): cls.old_chapter=value
+
+    @classmethod
+    def getOldComicName(cls): return cls.old_comic_name
+
+    @classmethod
+    def getOldChapter(cls): return cls.old_chapter