cwx/ComicScrapy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

clear

Browse Source
This commit is contained in:
caiwx86 2024-10-28 02:41:52 +08:00
parent ca805fe252
commit 5bd0292131
8 changed files with 20 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
Comics/exporters.py → Comics/_utils/exporters.py
View File

4
Comics/items.py → Comics/_utils/items.py
View File

@ -4,8 +4,8 @@
# https://docs.org/en/latest/topics/items.html # https://docs.org/en/latest/topics/items.html
import os,Comics.settings as settings,logging import os,Comics.settings as settings,logging
from scrapy.item import Item, Field from scrapy.item import Item, Field
from Comics.utils import ComicPath from Comics._utils.utils import ComicPath
from Comics.utils import imageUtils,fileUtils from Comics._utils.utils import imageUtils,fileUtils
from itemloaders.processors import TakeFirst from itemloaders.processors import TakeFirst
# 繁体中文转为简体中文 # 繁体中文转为简体中文

0
Comics/loader.py → Comics/_utils/loader.py
View File

0
Comics/middlewares.py → Comics/_utils/middlewares.py
View File

6
Comics/utils.py → Comics/_utils/utils.py
View File

@ -9,9 +9,9 @@ from pathlib import Path
from zipfile import ZipFile from zipfile import ZipFile
from Comics.settings import COMIC_INFO_XML_FILE,COMIC_INFO_XSD_FILE,OUTPUT_DIR,PROJECT_KEY from Comics.settings import COMIC_INFO_XML_FILE,COMIC_INFO_XSD_FILE,OUTPUT_DIR,PROJECT_KEY
import yaml import yaml
from Comics.loader import BaseLoader from Comics._utils.loader import BaseLoader
from Comics.loader import ComicLoader from Comics._utils.loader import ComicLoader
from Comics.loader import BooksLoader from Comics._utils.loader import BooksLoader
from tinydb import TinyDB, Query from tinydb import TinyDB, Query
# 配置类 # 配置类

12
Comics/pipelines.py
View File

@ -7,12 +7,12 @@
# useful for handling different item types with a single interface # useful for handling different item types with a single interface
import os,scrapy,logging,shutil import os,scrapy,logging,shutil
from Comics import settings from Comics import settings
from Comics.items import ComicItem from Comics._utils.items import ComicItem
from Comics.loader import ComicLoader from Comics._utils.loader import ComicLoader
from Comics.utils import CBZUtils,fileUtils as fu from Comics._utils.utils import CBZUtils,fileUtils as fu
from Comics.utils import ComicPath from Comics._utils.utils import ComicPath
from Comics.utils import oldUtils from Comics._utils.utils import oldUtils
from Comics.exporters import JsonExport,ItemExporter from Comics._utils.exporters import JsonExport,ItemExporter
from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.images import ImagesPipeline
from Comics._utils.ComicInfo import ComicInfoXml from Comics._utils.ComicInfo import ComicInfoXml

4
Comics/settings.py
View File

@ -81,7 +81,7 @@ COOKIES_ENABLED = False
DOWNLOADER_MIDDLEWARES = { DOWNLOADER_MIDDLEWARES = {
# 'Comics.middlewares.ComicsDownloaderMiddleware': 543, # 'Comics.middlewares.ComicsDownloaderMiddleware': 543,
# 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500, # 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
'Comics.middlewares.ProxyMiddleware': 100, 'Comics._utils.middlewares.ProxyMiddleware': 100,
'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 400, 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 400,
} }
@ -121,7 +121,7 @@ HTTPCACHE_EXPIRATION_SECS = 0
HTTPCACHE_DIR = os.path.join(BASE_OUTPUT,'httpcache') HTTPCACHE_DIR = os.path.join(BASE_OUTPUT,'httpcache')
HTTPCACHE_ALLOW_PREFIXS = [ 'jpg', 'png', 'gif', 'JPG', "PNG", "JPEG"] HTTPCACHE_ALLOW_PREFIXS = [ 'jpg', 'png', 'gif', 'JPG', "PNG", "JPEG"]
HTTPCACHE_PROXY_DOMAINS = [ 'r5.rmcdn3.xyz' ] HTTPCACHE_PROXY_DOMAINS = [ 'r5.rmcdn3.xyz' ]
HTTPCACHE_STORAGE = 'Comics.middlewares.MyFilesystemCacheStorage' HTTPCACHE_STORAGE = 'Comics._utils.middlewares.MyFilesystemCacheStorage'
# Logging configuration # Logging configuration
LOG_LEVEL = "INFO" # 日志等级 LOG_LEVEL = "INFO" # 日志等级

14
Comics/spiders/rm_comic.py
View File

@ -1,11 +1,11 @@
import scrapy,logging,os,skip,re import scrapy,logging,os,skip,re
from Comics.items import ComicItem from Comics._utils.items import ComicItem
from Comics.items import BooksItem from Comics._utils.items import BooksItem
from Comics.loader import ComicLoader from Comics._utils.loader import ComicLoader
from Comics.loader import BooksLoader from Comics._utils.loader import BooksLoader
from Comics.utils import ComicPath from Comics._utils.utils import ComicPath
from Comics.utils import Conf from Comics._utils.utils import Conf
from Comics.utils import oldUtils from Comics._utils.utils import oldUtils
class RmComicSpider(scrapy.Spider): class RmComicSpider(scrapy.Spider):
name = 'rm_comic' name = 'rm_comic'