This commit is contained in:
cwx
2023-03-31 19:57:15 +08:00
parent 9723c6b994
commit a7cfeae7eb
2 changed files with 9 additions and 12 deletions
+8 -10
View File
@@ -25,6 +25,7 @@ class htmlUtils:
return file_path
if os.path.exists(file_path):
if type == "read":
print(f"读取cache: {file_path}")
with open(file_path,"r",encoding="utf-8") as fs:
return fs.read()
return file_path
@@ -32,12 +33,13 @@ class htmlUtils:
return None
@classmethod
def saveHtml(cls,url,data):
def saveHtml(cls,url,data,type=None):
file_path = cls.getPathSaveHtml(url,type="new")
dir_name = os.path.dirname(file_path)
if not os.path.exists(dir_name):
os.makedirs(dir_name)
with open(file_path,"w",encoding="utf-8") as fs:
if type== "json": data = json.dumps(data)
fs.write(str(data))
@classmethod
@@ -61,14 +63,10 @@ class htmlUtils:
s.mount('https://', HTTPAdapter(max_retries=retries))
count = 1
#数据为空则获取数据
try:
url_text = cls.getPathSaveHtml(curl,"read")
except:
url_text = None
if url_text != None and update == False:
return html.fromstring(url_text)
else:
url_text = None
try: url_text = cls.getPathSaveHtml(curl,"read")
except: url_text = None
if url_text != None and update == False: return html.fromstring(url_text)
else: url_text = None
while url_text == None:
if count <= 5:
try:
@@ -82,7 +80,7 @@ class htmlUtils:
if type == "json":
#url_text = res.json()
try:
cls.saveHtml(curl,res.text)
cls.saveHtml(curl,res.text,type="json")
return json.loads(res.text)
except:
return None