54 lines
1.6 KiB
Python
54 lines
1.6 KiB
Python
from fake_useragent import UserAgent
|
||
import requests
|
||
from lxml import html
|
||
from utils.comic.ComicStr import comicStr
|
||
import os
|
||
|
||
class htmlUtils:
|
||
headers = {'User-Agent': UserAgent().random}
|
||
url_data = {}
|
||
|
||
@classmethod
|
||
def getHTML(cls, curl):
|
||
rstr = r"[\/\\\:\*\?\"\<\>\|\.]" # '/ \ : * ? " < > |'
|
||
#file_url = re.sub(rstr, "_", curl)
|
||
keys = cls.url_data.keys()
|
||
url_text = cls.url_data.get(curl)
|
||
#数据为空则获取数据
|
||
if url_text == None:
|
||
print("请求地址：",curl)
|
||
res = requests.get(curl, headers=cls.headers)
|
||
url_text = html.fromstring(res.text)
|
||
data = { curl : url_text}
|
||
cls.url_data.update(data)
|
||
return url_text
|
||
|
||
@classmethod
|
||
def getBytes(cls, url):
|
||
return requests.get(url, headers=cls.headers,stream=True)
|
||
|
||
@classmethod
|
||
def getJSON(cls,curl):
|
||
res = requests.get(curl, headers=cls.headers)
|
||
data_json = res.json()
|
||
return data_json
|
||
|
||
@classmethod
|
||
def xpathData(cls,c_xpath,url=None,num=None,not_eq=None):
|
||
if url == None:
|
||
url = cls.temp_url
|
||
else:
|
||
cls.temp_url = url
|
||
result = []
|
||
#获取html实体数据
|
||
et = cls.getHTML(url)
|
||
#比对数据
|
||
count = 1
|
||
xpaths = et.xpath(c_xpath)
|
||
for x in xpaths:
|
||
if not x == not_eq:
|
||
result.append(x)
|
||
count +=1
|
||
if not num == None:
|
||
result = result[num]
|
||
return result |