• 发布文章
  • 消息中心
点赞
收藏
评论
分享
原创

elasticsearch使用python脚本批量导入数据

2023-05-22 00:55:12
79
0
准备meta文件data-meta.json
{"index":{"_index":"test","_type":"doc"}}
准备row文件data-one-row.json
{ "k1" : 100, "k2": "2023-01-01", "k3": "33333333", "k4": "444444444444", "k5": 55.0}
python生成数据脚本:
import sys,json,time,random,os,math
class TsGenEsData(object):
    def __init__(self):
        # self.genDataNum = 1000
        self.jsonDataMeta = None
        self.jsonDataRow = None
    def loadMetaData(self,JsonFilePath):
        with open(JsonFilePath, 'r',encoding="utf-8",errors="ignore") as f:
            self.jsonDataMeta = json.load(f)
    def loadRowData(self,JsonFilePath):
        with open(JsonFilePath, 'r',encoding="utf-8",errors="ignore") as f:
            self.jsonDataRow = json.load(f)
    def genData(self,rowNum,ouputFileName):
        current_path = os.path.abspath('.')#工作路径
        metaJsonFilePath = os.path.join(current_path,'./data-meta.json')
        self.loadMetaData(metaJsonFilePath)
        rowJsonFilePath = os.path.join(current_path,'./data-one-row.json')
        self.loadRowData(rowJsonFilePath)
        writeData = ""
        for idx in range(0,rowNum):
            self.jsonDataRow["k1"] = str(int(self.jsonDataRow["k1"])+1)
            writeData += json.dumps(self.jsonDataMeta) + "\n"
            writeData += json.dumps(self.jsonDataRow) + "\n"
        fileFullPath = os.path.join(current_path,ouputFileName)
            with open(fileFullPath,'w')as file:
                file.write(writeData)   
        except:
            print("writeTargetToFile file:"+fileFullPath+",failed!!!")
if __name__ =='__main__':
    genEsData = TsGenEsData()
    genEsData.genData(100,"data100.json")
curl -H "Content-Type: application/json" -XPOST http://127.0.0.1:9234/_bulk?pretty -u elastic:xxx --data-binary "@edata100.json"