This commit is contained in:
cwx committed 2023-05-15 10:45:47 +08:00
1 parent 03f3a5fc7c
commit 43569ed246
4 files changed
+26 -14

No files matched your search

+5
View File
@@ -25,3 +25,8 @@ class ComicItem(scrapy.Item):
chapter_href= scrapy.Field() chapter_href= scrapy.Field()
genre = scrapy.Field() genre = scrapy.Field()
age_rating = scrapy.Field() age_rating = scrapy.Field()
class DownImagesItem(scrapy.Item):
image_name = scrapy.Field()
image_url = scrapy.Field()
image_path = scrapy.Field()
+5 -1
View File
@@ -6,7 +6,7 @@
# useful for handling different item types with a single interface # useful for handling different item types with a single interface
from itemadapter import ItemAdapter from itemadapter import ItemAdapter
from scrapy.pipelines.images import ImagesPipeline
class ComicsPipeline: class ComicsPipeline:
def open_spider(self,spider): def open_spider(self,spider):
@@ -39,3 +39,7 @@ class ComicsPipeline:
def close_spider(self,spider): def close_spider(self,spider):
self.fp.close() self.fp.close()
class ImgDownloadPipeline(ImagesPipeline):
def get_
+12 -9
View File
@@ -25,7 +25,10 @@ ROBOTSTXT_OBEY = False
# Configure a delay for requests for the same website (default: 0) # Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs # See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3 IMAGES_URLS_FIELD = "image_url"
IMAGES_RESULT_FIELD = "image_path"
IMAGES_STORE = 'images'
DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of: # The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16 #CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16 #CONCURRENT_REQUESTS_PER_IP = 16
@@ -50,9 +53,9 @@ DEFAULT_REQUEST_HEADERS = {
# Enable or disable downloader middlewares # Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = { DOWNLOADER_MIDDLEWARES = {
# 'Comics.middlewares.ComicsDownloaderMiddleware': 543, 'Comics.middlewares.ComicsDownloaderMiddleware': 543,
#} }
# Enable or disable extensions # Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html # See https://docs.scrapy.org/en/latest/topics/extensions.html
@@ -81,8 +84,8 @@ ITEM_PIPELINES = {
# Enable and configure HTTP caching (disabled by default) # Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache' HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+4 -4
View File
@@ -23,14 +23,13 @@ class RmComicSpider(scrapy.Spider):
comic['chapter_href'] = response.xpath('//div[contains(@class,"bookid_chapterBox")]//div[contains(@class,"bookid_chapter")]/a/@href').extract() comic['chapter_href'] = response.xpath('//div[contains(@class,"bookid_chapterBox")]//div[contains(@class,"bookid_chapter")]/a/@href').extract()
list_img = [] list_img = []
for link in comic['chapter_href']: for link in comic['chapter_href']:
list_img.append(scrapy.Request(self.main_url+link, callback=self.parse_chapter,errback=self.err)) yield list_img.append(scrapy.Request(self.main_url+link,meta={'item' : comic} , callback=self.parse_chapter,errback=self.err))
comic['list_img'] = list_img
yield comic
def err(self): def err(self):
print("Error=====") print("Error=====")
def parse_chapter(self, response): def parse_chapter(self, response):
item = response.meta['item']
data = response.xpath('//script[@id="__NEXT_DATA__"]/text()').extract_first() data = response.xpath('//script[@id="__NEXT_DATA__"]/text()').extract_first()
str_exec="props.pageProps." str_exec="props.pageProps."
comic_name = CommonUtils.parseExec(data,str_exec+"bookName") comic_name = CommonUtils.parseExec(data,str_exec+"bookName")
@@ -38,7 +37,8 @@ class RmComicSpider(scrapy.Spider):
description = CommonUtils.parseExec(data,str_exec+"description") description = CommonUtils.parseExec(data,str_exec+"description")
images = CommonUtils.parseExec(data,str_exec+"images") images = CommonUtils.parseExec(data,str_exec+"images")
chapter_api_url = CommonUtils.parseExec(data,str_exec+"chapterAPIPath") chapter_api_url = CommonUtils.parseExec(data,str_exec+"chapterAPIPath")
yield ComicItem(name=comic_name,chapter=chapterName,dep=description,list_img=images) item['list_img'] = images
yield item
def parse(self, response): def parse(self, response):
raise NotImplementedError raise NotImplementedError