diff --git a/Comics/items.py b/Comics/items.py index d06bec1..bd88c48 100644 --- a/Comics/items.py +++ b/Comics/items.py @@ -24,9 +24,13 @@ def _serialize_to_images(value, result_type=None): images_item = [] image_urls = [] for image in value: - (image_src, scramble) = [image.get("src"), image.get("scramble")] + try: + (image_src, scramble) = [image.get("src"), image.get("scramble")] + except: + (image_src, scramble) = [image, False] count_image = settings.IMAGES_NAME_FORMAT.format(count) - suffix = "."+str(image_src).split(".")[-1] +# suffix = "."+str(image_src).split(".")[-1] + suffix = ".jpg" image_name = count_image + suffix if scramble: de_str = str(image_src).split("/")[-1].replace(suffix, "==") diff --git a/Comics/loader.py b/Comics/loader.py index 2ba54ec..4eebf5d 100644 --- a/Comics/loader.py +++ b/Comics/loader.py @@ -1,8 +1,9 @@ import json,logging from scrapy.loader import ItemLoader +from Comics.settings import PROJECT_KEY class ComicLoader(ItemLoader): - def parseExec(cls,data,exec): + def parseExec(self,data,exec): if data !=None and exec != None: dots = str(exec).split(".") if not isinstance(data,dict): data = json.loads(data) @@ -61,4 +62,72 @@ class ComicLoader(ItemLoader): if self.get_output_value(field_name) != None: self._replace_value(field_name, value) return False - else: return True \ No newline at end of file + else: return True + + + # 设置漫画属性 + def set_properties(self, name, value=None, xpath=None, index=None, sexec=None): + if value != None and sexec==None: + self.add_value(field_name=name, value=value) + if xpath != None: + self.add_xpath(field_name=name, xpath=xpath, index=index) + if sexec != None: + self.add_exec(field_name=name, value=value, str_exec=sexec) + + # 工程名 + def project_name(self, project_name): self.add_value(PROJECT_KEY, project_name) + # 漫画名 + def name(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('name', value, xpath, index, sexec) + # 漫画封面链接 + def icon(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('icon', value, xpath, index, sexec) + # 作者 + def author(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('author', value, xpath, index, sexec) + # 标签 + def tags(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('tags', value, xpath, index, sexec) + # 概述 + def dep(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('dep', value, xpath, index, sexec) + # 时间 + def date(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('date', value, xpath, index, sexec) + # 流派 + def genre(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('genre', value, xpath, index, sexec) + # 年龄分级 + def age_rating(self, value=None, xpath=None, index=None, sexec=None): + self.set_properties('age_rating', value, xpath, index, sexec) + # 全部章节 + def chapters(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('chapters', value, xpath, index, sexec) + # 单一章节 + def chapter(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('chapter', value, xpath, index, sexec) + # 图像名称 + def images(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('images', value, xpath, index, sexec) + # 图像链接 + def image_urls(self, value=None, xpath=None, index=None, sexec=None): self.set_properties('image_urls', value, xpath, index, sexec) + +class ComicEntity: + + ENTITY = None + + def __init__(self, entity): + self.ENTITY = entity + + # 属性获取 + def get_dict(self, key): + try: + return self.ENTITY[key] + except: + return [] + + def set(self, key, value): self.ENTITY[key] = value + def set_count(self, value): self.set('count', value) + def set_index(self, value): self.set('index', value) + def images(self): return self.get_dict("images") + def image_urls(self): return self.get_dict("image_urls") + def chapters(self): return self.get_dict("chapters") + def chapter(self): return self.get_dict("chapter") + def icon(self): return self.get_dict("icon") + def count(self): return self.set_count(len(self.images())) + def index(self): return self.set_index(self.chapters().index(self.chapter()) + 1) + + def item(self): + self.count() + self.index() + return self.ENTITY diff --git a/Comics/pipelines.py b/Comics/pipelines.py index 0430340..64fcb1f 100644 --- a/Comics/pipelines.py +++ b/Comics/pipelines.py @@ -14,7 +14,7 @@ from scrapy.pipelines.images import ImagesPipeline from Comics.exporters import ComicInfoXmlItemExporter,JsonExport,ItemExporter from Comics.utils.FileUtils import CBZUtils from Comics.utils.FileUtils import fileUtils as fu - +from Comics.loader import ComicEntity class ComicsPipeline: def open_spider(self, spider): @@ -24,15 +24,11 @@ class ComicsPipeline: def process_item(self, item, spider): if isinstance(item, ComicItem): # 'output/rm_comic/json/壞X/第1話 壞X' - cbz_path = ComicPath.get_file_path(item, result_type="cbz", convert=True) - if os.path.exists(cbz_path): - item['image_urls'] = item['images'] = [] + if os.path.exists(ComicPath.CBZ(item=item)): return ItemExporter().export_obj(item) else: file = os.path.join(OUTPUT_DIR, spider.name, "json", item['name'], item['chapter']) - item['count'] = len(item['images']) - item['index'] = item['chapters'].index(item['chapter']) + 1 - return JsonExport(file=file).export_json(item, if_return=True) + return JsonExport(file=file).export_json(ComicEntity(item).item(), if_return=True) # image解析 def close_spider(self, spider): @@ -61,8 +57,9 @@ class ImgDownloadPipeline(ImagesPipeline): def file_path(self, request, response=None, info=None, *, item=None): return request.meta['path'] def get_media_requests(self, item, info): - self.image_urls = item['image_urls'] - self.images = item['images'] + comic = ComicEntity(item) + self.image_urls = comic.image_urls() + self.images = comic.images() # 下载封面 self.download_icon(item) for image_url,image in zip(self.image_urls,self.images): diff --git a/Comics/spiders/rm_comic.py b/Comics/spiders/rm_comic.py index bfb4451..a676fae 100644 --- a/Comics/spiders/rm_comic.py +++ b/Comics/spiders/rm_comic.py @@ -22,29 +22,29 @@ class RmComicSpider(scrapy.Spider): data = comics.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0] for book in comics.get_exec(data, str_exec="props.pageProps.books"): comics.add_value('link', self.start_urls+"/"+book['id']) - if not book['name'] in skip.skip_comic: + if book['name'] not in skip.skip_comic: yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic) # 获取某个漫画的相关数据 # 获取到多个章节链接后进入下个流程 def parse_comic(self, response): comic_item = ComicLoader(item=ComicItem(), response=response) - comic_item.add_value(PROJECT_KEY, self.name) - comic_item.add_xpath('name', '//div[@class="col"]/h5/text()') - comic_item.add_xpath('icon', '//img[@class="img-thumbnail"]/@src') - comic_item.add_xpath('author', '//div[contains(@class,"bookid_bookInfo")]/p[1]/text()', index=1) - comic_item.add_xpath('tags', '//div[contains(@class,"bookid_bookInfo")]/p[3]/b/text()') - comic_item.add_xpath('dep', '//div[contains(@class,"bookid_bookInfo")]/p[4]/text()', index=1) - comic_item.add_xpath('date', '//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()', index=1) - comic_item.add_value('genre', "韩漫") - comic_item.add_value('age_rating', "R18+") + comic_item.project_name(self.name) + comic_item.name(xpath='//div[@class="col"]/h5/text()') + comic_item.icon(xpath='//img[@class="img-thumbnail"]/@src') + comic_item.author(xpath='//div[contains(@class,"bookid_bookInfo")]/p[1]/text()', index=1) + comic_item.tags(xpath='//div[contains(@class,"bookid_bookInfo")]/p[3]/b/text()') + comic_item.dep(xpath='//div[contains(@class,"bookid_bookInfo")]/p[4]/text()', index=1) + comic_item.date(xpath='//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()', index=1) + comic_item.genre(value="韩漫") + comic_item.age_rating(value="R18+") chapter_href = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]' '//div[contains(@class,"bookid_chapter")]/a/@href') chapters = comic_item.get_xpath('//div[contains(@class,"bookid_chapterBox")]' '//div[contains(@class,"bookid_chapter")]/a/text()') for chapter, link in zip(chapters, chapter_href): - comic_item.add_value('chapter', chapter) - comic_item.add_value('chapters', chapters) + comic_item.chapters(value=chapters) + comic_item.chapter(value=chapter) item = comic_item.load_item() cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True) if os.path.exists(cbz_path): @@ -59,11 +59,11 @@ class RmComicSpider(scrapy.Spider): comic_item = ComicLoader(item=response.meta['item'], response=response) data = comic_item.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0] str_exec = "props.pageProps." - comic_item.add_exec('name', data, str_exec=str_exec+"bookName") - comic_item.add_exec('dep', data, str_exec=str_exec+"description") - comic_item.add_exec('chapter', data, str_exec=str_exec + "chapterName") - comic_item.add_exec('image_urls', data, str_exec+"images") - comic_item.add_exec('images', data, str_exec+"images") + comic_item.name(value=data, sexec=str_exec+"bookName") + comic_item.dep(value=data, sexec=str_exec+"description") + comic_item.chapter(value=data, sexec=str_exec+"chapterName") + comic_item.image_urls(value=data, sexec=str_exec+"images") + comic_item.images(value=data, sexec=str_exec+"images") comic = comic_item.load_item() chapter_api_url = comic_item.get_exec(data, str_exec+"chapterAPIPath") if chapter_api_url is not None: @@ -74,9 +74,9 @@ class RmComicSpider(scrapy.Spider): # 加密数据API处理 def parse_chapter_api(self, response): comic_item = ComicLoader(item=response.meta['item'], response=response) - comic_item.add_exec('chapter', response.text, str_exec='chapter.name') - comic_item.add_exec('image_urls', response.text, str_exec='chapter.images') - comic_item.add_exec('images', response.text, str_exec='chapter.images') + comic_item.chapter(value=response.text, sexec='chapter.name') + comic_item.image_urls(value=response.text, sexec='chapter.images') + comic_item.images(value=response.text, sexec='chapter.images') yield comic_item.load_item() diff --git a/Comics/spiders/yh_comic.py b/Comics/spiders/yh_comic.py new file mode 100644 index 0000000..602d9c2 --- /dev/null +++ b/Comics/spiders/yh_comic.py @@ -0,0 +1,69 @@ +import scrapy,logging,time,os +from Comics.items import ComicItem +from Comics.loader import ComicLoader +from Comics.items import ListComicItem +from Comics.utils.Constant import ComicPath +from Comics.settings import PROJECT_KEY +import skip + +class RmComicSpider(scrapy.Spider): + name = 'yh_comic' + allowed_domains = ['www.shuanglilock.com.cn'] + main_url = 'https://'+allowed_domains[0] + start_urls = main_url+'/info' + + def start_requests(self): +# for x in range(0,60): + yield scrapy.Request("https://www.shuanglilock.com.cn/info/27145/", callback=self.parse_comic) + + # 获取多个漫画信息 +# def books_comic(self, response): +# comics = ComicLoader(item=ComicItem(), response=response) +# data = comics.get_xpath('//script[@id="__NEXT_DATA__"]/text()')[0] +# for book in comics.get_exec(data, str_exec="props.pageProps.books"): +# comics.add_value('link', self.start_urls+"/"+book['id']) +# if book['name'] not in skip.skip_comic: +# yield scrapy.Request(url=self.start_urls+"/"+book['id'], callback=self.parse_comic) + + # 获取某个漫画的相关数据 + # 获取到多个章节链接后进入下个流程 + def parse_comic(self, response): + comic_item = ComicLoader(item=ComicItem(), response=response) + comic_item.project_name(self.name) + comic_item.name(xpath='//div[@class="comics-detail__info"]/h1[@class="comics-detail__title"]/text()') + comic_item.icon(xpath='//div[@class="pure-u-1-1 pure-u-sm-1-3 pure-u-md-1-6"]/img/@src') + comic_item.author(xpath='//div[@class="comics-detail__info"]/h2[@class="comics-detail__author"]/text()') + comic_item.tags(xpath='//div[@class="tag-list"]/a[@class="tag"]/text()') + comic_item.dep(xpath='//p[contains(@class,"comics-detail__desc")]/text()') + #comic_item.date(xpath='//div[contains(@class,"bookid_bookInfo")]/p[5]/small/text()', index=1) + comic_item.genre(value="樱花漫画") + #comic_item.age_rating(value="R18+") + chapter_href = comic_item.get_xpath('//div[contains(@id,"chapter-items")]' + '//a[@class="comics-chapters__item"]/@href') + chapters = comic_item.get_xpath('//div[contains(@id,"chapter-items")]' + '//a[@class="comics-chapters__item"]//span/text()') + for chapter, link in zip(chapters, chapter_href): + comic_item.chapters(value=chapters) + comic_item.chapter(value=chapter) + item = comic_item.load_item() + cbz_path = ComicPath.get_file_path(item=item, result_type="cbz", convert=True) + if os.path.exists(cbz_path): + logging.info(f"漫画 {cbz_path} 已存在, 跳过中...") + yield item + else: + yield scrapy.Request(self.main_url+link, meta={'item': item}, callback=self.parse_chapter) + + + # 读取某章节下的所有图片 + def parse_chapter(self, response): + comic_item = ComicLoader(item=response.meta['item'], response=response) + comic_item.image_urls(xpath='//div[@class="comiclist"]/div[@class="comicpage"]/div/img/@data-original') + comic_item.images(xpath='//div[@class="comiclist"]/div[@class="comicpage"]/div/img/@data-original') + comic = comic_item.load_item() + yield comic + + def parse(self, response): + raise NotImplementedError + + def error_parse(self, response): + raise NotImplementedError \ No newline at end of file diff --git a/Comics/utils/Constant.py b/Comics/utils/Constant.py index f177481..7652d47 100644 --- a/Comics/utils/Constant.py +++ b/Comics/utils/Constant.py @@ -78,7 +78,12 @@ class ComicPath: elif result_type == "images_dir": file = os.path.join(settings.IMAGES_STORE, PROJECT, "images", name, chapter) return file - + + @classmethod + def CBZ(cls, item): + return cls.get_file_path(item, result_type="cbz", convert=True) + + class ntfy: @classmethod def sendMsg(cls, msg,alert=False,sleep=None,error=None):