diff --git a/utils/HtmlUtils.py b/utils/HtmlUtils.py
index cc93759..5d3b06b 100644
--- a/utils/HtmlUtils.py
+++ b/utils/HtmlUtils.py
@@ -11,7 +11,7 @@ class htmlUtils:
url_data = {}
@classmethod
- def getHTML(cls, curl):
+ def getHTML(cls, curl,type=None):
rstr = r"[\/\\\:\*\?\"\<\>\|\.]" # '/ \ : * ? " < > |'
#file_url = re.sub(rstr, "_", curl)
keys = cls.url_data.keys()
@@ -29,7 +29,10 @@ class htmlUtils:
try:
print("请求地址:",curl)
res = s.get(curl,stream=True, headers=cls.headers, timeout=20)
- url_text = html.fromstring(res.text)
+ if type == "json":
+ url_text = res.json()
+ else:
+ url_text = html.fromstring(res.text)
except:
print(f'Retry! 第{count}次')
time.sleep(3)
@@ -43,15 +46,9 @@ class htmlUtils:
cls.url_data.update(data)
return url_text
- @classmethod
- def getBytes(cls, url):
- return requests.get(url, headers=cls.headers,stream=True,timeout=30)
-
@classmethod
def getJSON(cls,curl):
- res = requests.get(curl, headers=cls.headers,timeout=30)
- data_json = res.json()
- return data_json
+ return cls.getHTML(curl,type="json")
@classmethod
def xpathData(cls,c_xpath,url=None,num=None,not_eq=None):