clear
This commit is contained in:
parent
ca805fe252
commit
5bd0292131
@ -4,8 +4,8 @@
|
||||
# https://docs.org/en/latest/topics/items.html
|
||||
import os,Comics.settings as settings,logging
|
||||
from scrapy.item import Item, Field
|
||||
from Comics.utils import ComicPath
|
||||
from Comics.utils import imageUtils,fileUtils
|
||||
from Comics._utils.utils import ComicPath
|
||||
from Comics._utils.utils import imageUtils,fileUtils
|
||||
from itemloaders.processors import TakeFirst
|
||||
|
||||
# 繁体中文转为简体中文
|
||||
@ -9,9 +9,9 @@ from pathlib import Path
|
||||
from zipfile import ZipFile
|
||||
from Comics.settings import COMIC_INFO_XML_FILE,COMIC_INFO_XSD_FILE,OUTPUT_DIR,PROJECT_KEY
|
||||
import yaml
|
||||
from Comics.loader import BaseLoader
|
||||
from Comics.loader import ComicLoader
|
||||
from Comics.loader import BooksLoader
|
||||
from Comics._utils.loader import BaseLoader
|
||||
from Comics._utils.loader import ComicLoader
|
||||
from Comics._utils.loader import BooksLoader
|
||||
from tinydb import TinyDB, Query
|
||||
|
||||
# 配置类
|
||||
@ -7,12 +7,12 @@
|
||||
# useful for handling different item types with a single interface
|
||||
import os,scrapy,logging,shutil
|
||||
from Comics import settings
|
||||
from Comics.items import ComicItem
|
||||
from Comics.loader import ComicLoader
|
||||
from Comics.utils import CBZUtils,fileUtils as fu
|
||||
from Comics.utils import ComicPath
|
||||
from Comics.utils import oldUtils
|
||||
from Comics.exporters import JsonExport,ItemExporter
|
||||
from Comics._utils.items import ComicItem
|
||||
from Comics._utils.loader import ComicLoader
|
||||
from Comics._utils.utils import CBZUtils,fileUtils as fu
|
||||
from Comics._utils.utils import ComicPath
|
||||
from Comics._utils.utils import oldUtils
|
||||
from Comics._utils.exporters import JsonExport,ItemExporter
|
||||
from scrapy.pipelines.images import ImagesPipeline
|
||||
from Comics._utils.ComicInfo import ComicInfoXml
|
||||
|
||||
|
||||
@ -81,7 +81,7 @@ COOKIES_ENABLED = False
|
||||
DOWNLOADER_MIDDLEWARES = {
|
||||
# 'Comics.middlewares.ComicsDownloaderMiddleware': 543,
|
||||
# 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
|
||||
'Comics.middlewares.ProxyMiddleware': 100,
|
||||
'Comics._utils.middlewares.ProxyMiddleware': 100,
|
||||
'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 400,
|
||||
}
|
||||
|
||||
@ -121,7 +121,7 @@ HTTPCACHE_EXPIRATION_SECS = 0
|
||||
HTTPCACHE_DIR = os.path.join(BASE_OUTPUT,'httpcache')
|
||||
HTTPCACHE_ALLOW_PREFIXS = [ 'jpg', 'png', 'gif', 'JPG', "PNG", "JPEG"]
|
||||
HTTPCACHE_PROXY_DOMAINS = [ 'r5.rmcdn3.xyz' ]
|
||||
HTTPCACHE_STORAGE = 'Comics.middlewares.MyFilesystemCacheStorage'
|
||||
HTTPCACHE_STORAGE = 'Comics._utils.middlewares.MyFilesystemCacheStorage'
|
||||
|
||||
# Logging configuration
|
||||
LOG_LEVEL = "INFO" # 日志等级
|
||||
|
||||
@ -1,11 +1,11 @@
|
||||
import scrapy,logging,os,skip,re
|
||||
from Comics.items import ComicItem
|
||||
from Comics.items import BooksItem
|
||||
from Comics.loader import ComicLoader
|
||||
from Comics.loader import BooksLoader
|
||||
from Comics.utils import ComicPath
|
||||
from Comics.utils import Conf
|
||||
from Comics.utils import oldUtils
|
||||
from Comics._utils.items import ComicItem
|
||||
from Comics._utils.items import BooksItem
|
||||
from Comics._utils.loader import ComicLoader
|
||||
from Comics._utils.loader import BooksLoader
|
||||
from Comics._utils.utils import ComicPath
|
||||
from Comics._utils.utils import Conf
|
||||
from Comics._utils.utils import oldUtils
|
||||
|
||||
class RmComicSpider(scrapy.Spider):
|
||||
name = 'rm_comic'
|
||||
|
||||
Loading…
Reference in New Issue
Block a user