41 lines
1.5 KiB
Python
41 lines
1.5 KiB
Python
import os.path
|
|
from scrapy.exporters import PythonItemExporter
|
|
from scrapy.exporters import JsonItemExporter
|
|
|
|
class CommonExporter():
|
|
def getPath(self, file , sufix=None):
|
|
sufix = "."+sufix
|
|
dirname = os.path.dirname(file)
|
|
if not os.path.exists(dirname):
|
|
os.makedirs(dirname)
|
|
if sufix != None and sufix not in file:
|
|
file = file + sufix
|
|
return file
|
|
|
|
class ItemExporter(PythonItemExporter):
|
|
def convert(self, data):
|
|
if isinstance(data, bytes): return data.decode("utf-8")
|
|
if isinstance(data, dict): return dict(map(self.convert, data.items()))
|
|
if isinstance(data, tuple): return map(self.convert, data)
|
|
if isinstance(data, list): return [self.convert(i) for i in data]
|
|
return data
|
|
|
|
def export_obj(self, obj_item):
|
|
self.start_exporting()
|
|
obj_item = self.convert(self.export_item(obj_item))
|
|
self.finish_exporting()
|
|
return obj_item
|
|
|
|
class JsonExport(JsonItemExporter):
|
|
def __init__(self, file, **kwargs):
|
|
file = CommonExporter().getPath(file=file, sufix= "json")
|
|
self.file = open(file, "wb")
|
|
super(JsonExport, self).__init__(self.file, **kwargs)
|
|
|
|
def export_json(self, json_object, if_return=False):
|
|
self.start_exporting()
|
|
self.export_item(json_object)
|
|
self.finish_exporting()
|
|
self.file.close()
|
|
if if_return:
|
|
return ItemExporter().export_obj(json_object) |