cwx/ComicScrapy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
caiwx86 2023-08-06 20:22:53 +08:00
parent aeff24daa4
commit ae3826b671
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Comics/settings.py
View File

@ -14,7 +14,8 @@ BOT_NAME = 'Comics'
SPIDER_MODULES = ['Comics.spiders'] SPIDER_MODULES = ['Comics.spiders']
NEWSPIDER_MODULE = 'Comics.spiders' NEWSPIDER_MODULE = 'Comics.spiders'
OUTPUT_DIR = "output" BASE_OUTPUT = "/home/cwx/Dev/Comics"
OUTPUT_DIR = os.path.join(BASE_OUTPUT, "output")
# Crawl responsibly by identifying yourself (and your website) on the user-agent # Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'Comics (+http://www.yourdomain.com)' #USER_AGENT = 'Comics (+http://www.yourdomain.com)'
USER_AGENT = UserAgent().random USER_AGENT = UserAgent().random
@ -29,7 +30,7 @@ CONCURRENT_REQUESTS = 16
# Configure a delay for requests for the same website (default: 0) # Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs # See also autothrottle settings and docs
IMAGES_STORE = "output" IMAGES_STORE = OUTPUT_DIR
IMAGES_NAME_FORMAT = "{:0>3d}" IMAGES_NAME_FORMAT = "{:0>3d}"
COMIC_INFO_XML_STORE = IMAGES_STORE COMIC_INFO_XML_STORE = IMAGES_STORE
DOWNLOAD_DELAY = 0 DOWNLOAD_DELAY = 0
@ -105,7 +106,7 @@ AUTOTHROTTLE_DEBUG = False
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
HTTPCACHE_ENABLED = True HTTPCACHE_ENABLED = True
HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_EXPIRATION_SECS = 0
HTTPCACHE_DIR = 'httpcache' HTTPCACHE_DIR = os.path.join(BASE_OUTPUT,'httpcache')
HTTPCACHE_ALLOW_PREFIXS = [ 'jpg', 'png', 'gif', 'JPG', "PNG", "JPEG"] HTTPCACHE_ALLOW_PREFIXS = [ 'jpg', 'png', 'gif', 'JPG', "PNG", "JPEG"]
HTTPCACHE_STORAGE = 'Comics.middlewares.MyFilesystemCacheStorage' HTTPCACHE_STORAGE = 'Comics.middlewares.MyFilesystemCacheStorage'
@ -113,7 +114,7 @@ HTTPCACHE_STORAGE = 'Comics.middlewares.MyFilesystemCacheStorage'
LOG_LEVEL = "INFO" # 日志等级 LOG_LEVEL = "INFO" # 日志等级
LOG_STDOUT = True # 标准化输出 LOG_STDOUT = True # 标准化输出
CBZ_EXPORT_PATH = "CBZ" CBZ_EXPORT_PATH = os.path.join(BASE_OUTPUT,"CBZ")
#数据导出类 排序 #数据导出类 排序
COMIC_INFO_XML_FILE = "ComicInfo.xml" COMIC_INFO_XML_FILE = "ComicInfo.xml"
COMIC_INFO_FIELDS_TO_EXPORT = [ COMIC_INFO_FIELDS_TO_EXPORT = [