diff --git a/Comics/items.py b/Comics/items.py index cdbb6a8..9a2fe2b 100644 --- a/Comics/items.py +++ b/Comics/items.py @@ -24,4 +24,9 @@ class ComicItem(scrapy.Item): chapters = scrapy.Field() chapter_href= scrapy.Field() genre = scrapy.Field() - age_rating = scrapy.Field() \ No newline at end of file + age_rating = scrapy.Field() + +class DownImagesItem(scrapy.Item): + image_name = scrapy.Field() + image_url = scrapy.Field() + image_path = scrapy.Field() diff --git a/Comics/pipelines.py b/Comics/pipelines.py index 157a572..0541bbe 100644 --- a/Comics/pipelines.py +++ b/Comics/pipelines.py @@ -6,7 +6,7 @@ # useful for handling different item types with a single interface from itemadapter import ItemAdapter - +from scrapy.pipelines.images import ImagesPipeline class ComicsPipeline: def open_spider(self,spider): @@ -38,4 +38,8 @@ class ComicsPipeline: return item def close_spider(self,spider): - self.fp.close() \ No newline at end of file + self.fp.close() + +class ImgDownloadPipeline(ImagesPipeline): + def get_ + \ No newline at end of file diff --git a/Comics/settings.py b/Comics/settings.py index b6c7b01..4a3977f 100644 --- a/Comics/settings.py +++ b/Comics/settings.py @@ -25,7 +25,10 @@ ROBOTSTXT_OBEY = False # Configure a delay for requests for the same website (default: 0) # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs -#DOWNLOAD_DELAY = 3 +IMAGES_URLS_FIELD = "image_url" +IMAGES_RESULT_FIELD = "image_path" +IMAGES_STORE = 'images' +DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: #CONCURRENT_REQUESTS_PER_DOMAIN = 16 #CONCURRENT_REQUESTS_PER_IP = 16 @@ -50,9 +53,9 @@ DEFAULT_REQUEST_HEADERS = { # Enable or disable downloader middlewares # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html -#DOWNLOADER_MIDDLEWARES = { -# 'Comics.middlewares.ComicsDownloaderMiddleware': 543, -#} +DOWNLOADER_MIDDLEWARES = { + 'Comics.middlewares.ComicsDownloaderMiddleware': 543, +} # Enable or disable extensions # See https://docs.scrapy.org/en/latest/topics/extensions.html @@ -81,8 +84,8 @@ ITEM_PIPELINES = { # Enable and configure HTTP caching (disabled by default) # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings -#HTTPCACHE_ENABLED = True -#HTTPCACHE_EXPIRATION_SECS = 0 -#HTTPCACHE_DIR = 'httpcache' -#HTTPCACHE_IGNORE_HTTP_CODES = [] -#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' +HTTPCACHE_ENABLED = True +HTTPCACHE_EXPIRATION_SECS = 0 +HTTPCACHE_DIR = 'httpcache' +HTTPCACHE_IGNORE_HTTP_CODES = [] +HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' diff --git a/Comics/spiders/rm_comic.py b/Comics/spiders/rm_comic.py index 81d99fa..81ed310 100644 --- a/Comics/spiders/rm_comic.py +++ b/Comics/spiders/rm_comic.py @@ -23,14 +23,13 @@ class RmComicSpider(scrapy.Spider): comic['chapter_href'] = response.xpath('//div[contains(@class,"bookid_chapterBox")]//div[contains(@class,"bookid_chapter")]/a/@href').extract() list_img = [] for link in comic['chapter_href']: - list_img.append(scrapy.Request(self.main_url+link, callback=self.parse_chapter,errback=self.err)) - comic['list_img'] = list_img - yield comic + yield list_img.append(scrapy.Request(self.main_url+link,meta={'item' : comic} , callback=self.parse_chapter,errback=self.err)) def err(self): print("Error=====") def parse_chapter(self, response): + item = response.meta['item'] data = response.xpath('//script[@id="__NEXT_DATA__"]/text()').extract_first() str_exec="props.pageProps." comic_name = CommonUtils.parseExec(data,str_exec+"bookName") @@ -38,7 +37,8 @@ class RmComicSpider(scrapy.Spider): description = CommonUtils.parseExec(data,str_exec+"description") images = CommonUtils.parseExec(data,str_exec+"images") chapter_api_url = CommonUtils.parseExec(data,str_exec+"chapterAPIPath") - yield ComicItem(name=comic_name,chapter=chapterName,dep=description,list_img=images) + item['list_img'] = images + yield item def parse(self, response): raise NotImplementedError \ No newline at end of file