esutil.py    

'''      Export and Import ElasticSearch Data.      Simple Example At __main__      @author: wgzh159@163.com      @note:  uncheck consistency of data, please do it by self  '''    import json  import os  import sys  import time  import urllib2    reload(sys)  sys.setdefaultencoding('utf-8')  # @UndefinedVariable    class exportEsData():      size = 10000      def __init__(self, url,index,type):          self.url = url+"/"+index+"/"+type+"/_search"          self.index = index          self.type = type      def exportData(self):          print("export data begin...")          begin = time.time()          try:              os.remove(self.index+"_"+self.type+".json")          except:              os.mknod(self.index+"_"+self.type+".json")          msg = urllib2.urlopen(self.url).read()          print(msg)          obj = json.loads(msg)          num = obj["hits"]["total"]          start = 0          end =  num/self.size+1          while(start<end):              msg = urllib2.urlopen(self.url+"?from="+str(start*self.size)+"&size="+str(self.size)).read()              self.writeFile(msg)              start=start+1          print("export data end!!!\n\t total consuming time:"+str(time.time()-begin)+"s")      def writeFile(self,msg):          obj = json.loads(msg)          vals = obj["hits"]["hits"]          try:              f = open(self.index+"_"+self.type+".json","a")              for val in vals:                  a = json.dumps(val["_source"],ensure_ascii=False)                  f.write(a+"\n")          finally:              f.flush()              f.close()    class importEsData():      def __init__(self,url,index,type):          self.url = url+"/"+index+"/"+type          self.index = index          self.type = type                def importData(self):          print("import data begin...")          begin = time.time()          try:              f = open(self.index+"_"+self.type+".json","r")              for line in f:                  self.post(line)          finally:              f.close()          print("import data end!!!\n\t total consuming time:"+str(time.time()-begin)+"s")      def post(self,data):          req = urllib2.Request(self.url,data,{"Content-Type":"application/json; charset=UTF-8"})          urllib2.urlopen(req)    if __name__ == '__main__':      '''          Export Data          e.g.                              URL                    index        type          exportEsData("http://10.100.142.60:9200","watchdog","mexception").exportData()                    export file name: watchdog_mexception.json      '''      #exportEsData("http://10.100.142.60:9200","watchdog","mexception").exportData()      exportEsData("http://10.100.142.60:9200","watchdog","mexception").exportData()                  '''          Import Data                    *import file name:watchdog_test.json    (important)                      "_" front part represents the elasticsearch index                      "_" after part represents the  elasticsearch type          e.g.                              URL                    index        type          mportEsData("http://10.100.142.60:9200","watchdog","test").importData()      '''      #importEsData("http://10.100.142.60:9200","watchdog","test").importData()      importEsData("http://10.100.142.60:9200","watchdog","test").importData()