This commit is contained in:
caiwx86 2023-08-06 20:22:53 +08:00
parent aeff24daa4
commit ae3826b671

View File

@ -14,7 +14,8 @@ BOT_NAME = 'Comics'
SPIDER_MODULES = ['Comics.spiders']
NEWSPIDER_MODULE = 'Comics.spiders'
OUTPUT_DIR = "output"
BASE_OUTPUT = "/home/cwx/Dev/Comics"
OUTPUT_DIR = os.path.join(BASE_OUTPUT, "output")
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'Comics (+http://www.yourdomain.com)'
USER_AGENT = UserAgent().random
@ -29,7 +30,7 @@ CONCURRENT_REQUESTS = 16
# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
IMAGES_STORE = "output"
IMAGES_STORE = OUTPUT_DIR
IMAGES_NAME_FORMAT = "{:0>3d}"
COMIC_INFO_XML_STORE = IMAGES_STORE
DOWNLOAD_DELAY = 0
@ -105,7 +106,7 @@ AUTOTHROTTLE_DEBUG = False
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
HTTPCACHE_ENABLED = True
HTTPCACHE_EXPIRATION_SECS = 0
HTTPCACHE_DIR = 'httpcache'
HTTPCACHE_DIR = os.path.join(BASE_OUTPUT,'httpcache')
HTTPCACHE_ALLOW_PREFIXS = [ 'jpg', 'png', 'gif', 'JPG', "PNG", "JPEG"]
HTTPCACHE_STORAGE = 'Comics.middlewares.MyFilesystemCacheStorage'
@ -113,7 +114,7 @@ HTTPCACHE_STORAGE = 'Comics.middlewares.MyFilesystemCacheStorage'
LOG_LEVEL = "INFO" # 日志等级
LOG_STDOUT = True # 标准化输出
CBZ_EXPORT_PATH = "CBZ"
CBZ_EXPORT_PATH = os.path.join(BASE_OUTPUT,"CBZ")
#数据导出类 排序
COMIC_INFO_XML_FILE = "ComicInfo.xml"
COMIC_INFO_FIELDS_TO_EXPORT = [