Python制作数据导入导出工具

时间:2021-05-22

python 2.6编写,自己瞎写的,备用

''' Export and Import ElasticSearch Data. Simple Example At __main__ @author: wgzh159@163.com @note: uncheck consistency of data, please do it by self''' import jsonimport osimport sysimport timeimport urllib2 reload(sys)sys.setdefaultencoding('utf-8') # @UndefinedVariable class exportEsData(): size = 10000 def __init__(self, url,index,type): self.url = url+"/"+index+"/"+type+"/_search" self.index = index self.type = type def exportData(self): print("export data begin...") begin = time.time() try: os.remove(self.index+"_"+self.type+".json") except: os.mknod(self.index+"_"+self.type+".json") msg = urllib2.urlopen(self.url).read() print(msg) obj = json.loads(msg) num = obj["hits"]["total"] start = 0 end = num/self.size+1 while(start<end): msg = urllib2.urlopen(self.url+"?from="+str(start*self.size)+"&size="+str(self.size)).read() self.writeFile(msg) start=start+1 print("export data end!!!\n\t total consuming time:"+str(time.time()-begin)+"s") def writeFile(self,msg): obj = json.loads(msg) vals = obj["hits"]["hits"] try: f = open(self.index+"_"+self.type+".json","a") for val in vals: a = json.dumps(val["_source"],ensure_ascii=False) f.write(a+"\n") finally: f.flush() f.close() class importEsData(): def __init__(self,url,index,type): self.url = url+"/"+index+"/"+type self.index = index self.type = type def importData(self): print("import data begin...") begin = time.time() try: f = open(self.index+"_"+self.type+".json","r") for line in f: self.post(line) finally: f.close() print("import data end!!!\n\t total consuming time:"+str(time.time()-begin)+"s") def post(self,data): req = urllib2.Request(self.url,data,{"Content-Type":"application/json; charset=UTF-8"}) urllib2.urlopen(req) if __name__ == '__main__': ''' Export Data e.g. URL index type exportEsData("http://10.100.142.60:9200","watchdog","mexception").exportData() export file name: watchdog_mexception.json ''' #exportEsData("http://10.100.142.60:9200","watchdog","mexception").exportData() exportEsData("http://10.100.142.60:9200","watchdog","mexception").exportData() ''' Import Data *import file name:watchdog_test.json (important) "_" front part represents the elasticsearch index "_" after part represents the elasticsearch type e.g. URL index type mportEsData("http://10.100.142.60:9200","watchdog","test").importData() ''' #importEsData("http://10.100.142.60:9200","watchdog","test").importData() importEsData("http://10.100.142.60:9200","watchdog","test").importData()

以上所述就是本文的全部内容了,希望大家能够喜欢。

声明:本页内容来源网络,仅供用户参考;我单位不保证亦不表示资料全面及准确无误,也不保证亦不表示这些资料为最新信息,如因任何原因,本网内容或者用户因倚赖本网内容造成任何损失或损害,我单位将不会负任何法律责任。如涉及版权问题,请提交至online#300.cn邮箱联系删除。

相关文章