81 lines
2.5 KiB
Python
81 lines
2.5 KiB
Python
from fake_useragent import UserAgent
|
||
import requests
|
||
from lxml import html
|
||
import traceback
|
||
import time
|
||
from urllib3.util.retry import Retry
|
||
from requests.adapters import HTTPAdapter
|
||
|
||
class htmlUtils:
|
||
headers = {'User-Agent': UserAgent().random}
|
||
url_data = {}
|
||
|
||
@classmethod
|
||
def getHTML(cls, curl,type=None):
|
||
rstr = r"[\/\\\:\*\?\"\<\>\|\.]" # '/ \ : * ? " < > |'
|
||
#file_url = re.sub(rstr, "_", curl)
|
||
keys = cls.url_data.keys()
|
||
url_text = cls.url_data.get(curl)
|
||
retries = Retry(total=3,
|
||
backoff_factor=0.1,
|
||
status_forcelist=[ 500, 502, 503, 504 ])
|
||
s = requests.Session()
|
||
s.mount('http://', HTTPAdapter(max_retries=retries))
|
||
s.mount('https://', HTTPAdapter(max_retries=retries))
|
||
count = 1
|
||
#数据为空则获取数据
|
||
while url_text == None:
|
||
if count < 5:
|
||
try:
|
||
print("请求地址：",curl)
|
||
res = s.get(curl,stream=True, headers=cls.headers, timeout=20)
|
||
if type == "json":
|
||
url_text = res.json()
|
||
if type == "bytes":
|
||
url_text = res
|
||
else:
|
||
url_text = html.fromstring(res.text)
|
||
except:
|
||
print(f'Retry! 第{count}次')
|
||
time.sleep(3)
|
||
traceback.print_exc()
|
||
count += 1
|
||
continue
|
||
else:
|
||
print(f"fail 请求失败：{curl}")
|
||
break
|
||
if not type == "bytes":
|
||
data = { curl : url_text}
|
||
cls.url_data.update(data)
|
||
return url_text
|
||
|
||
@classmethod
|
||
def getBytes(cls, url):
|
||
return cls.getHTML(url,type="bytes")
|
||
|
||
@classmethod
|
||
def getJSON(cls,curl):
|
||
return cls.getHTML(curl,type="json")
|
||
|
||
@classmethod
|
||
def xpathData(cls,c_xpath,url=None,num=None,not_eq=None):
|
||
if url == None:
|
||
url = cls.temp_url
|
||
else:
|
||
cls.temp_url = url
|
||
result = []
|
||
#获取html实体数据
|
||
et = cls.getHTML(url)
|
||
#比对数据
|
||
count = 1
|
||
xpaths = et.xpath(c_xpath)
|
||
for x in xpaths:
|
||
if not x == not_eq:
|
||
result.append(x)
|
||
count +=1
|
||
if not num == None:
|
||
try:
|
||
result = result[num]
|
||
except:
|
||
result = None
|
||
return result |