#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2020-07-02 17:38
# @Author  : zhangyunfei
# @File    : tx-ex-insert.py
# @Software: PyCharm
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2020-06-26 17:16
# @Author  : zhangyunfei
# @File    : tx_es_data.py
# @Software: PyCharm

from elasticsearch import Elasticsearch
from elasticsearch import helpers
from elasticsearch.helpers import bulk
import json

'''
    读取txt里面的全部数据，插入到新的索引中
'''

def DetData(datainfo):#根据ES5中返回的结果，组织导入ES6数据结构
    isTrue = True
    action = {}
    try:
        doc = {}
        action["_index"] = "tx_ic_bigdata_business_heming_index_01"
        # action["_type"] = "_doc"   #elasticsearch7.0+版本需要注释
        if "id" in datainfo.keys():
            id = datainfo["id"]
            doc["id"] = id
            action["_id"] = id

        if "company_name" in datainfo.keys():
            company_name = datainfo["company_name"]
            if company_name:
                doc["company_name"] = company_name
                doc["company_name_standard"] = company_name
            else:
                doc["company_name"] = None
                doc["company_name_standard"] = None
        else:
            doc["company_name"] = None

        action["_source"] = doc

    except Exception as e:
        isTrue = False
    return isTrue, action

def main():
    error = "ES_error.txt"
    f1 = open(error, "a")
    index_name = "tx_ic_bigdata_business_heming_index_01"
    es = Elasticsearch(['9.223.8.84'], http_auth=("elastic", "Brg2020!"), port=9200)
    readfile = open('2.txt','r',encoding='utf-8')
    lines = readfile.readlines()   # 获取全部
    bulkList = []
    for line in lines:
        line = line.strip()
        ll = json.loads(line)
        isTrue, action = DetData(ll)
        if isTrue:
            bulkList.append(action)
        else:
            f1.write("获取" + str(ll) + "的数据出错！\r\n")
        if len(bulkList) == 500:
            print('插入500...............')
            try:
                bulk(es, bulkList, index=index_name, raise_on_error=True)  # 批量插入
            except Exception as e:
                try:
                    for er in e.args[1]:  # 和上面的for是一样的，都能取到错误信息
                        f1.write("插入" + str(er["index"]["data"]) + "的数据出错！\r\n")
                except:
                    f1.write("插入" + str(bulkList) + "的数据出错！\r\n")
            bulkList = []
    if len(bulkList) > 0:
        try:
            bulk(es, bulkList, index=index_name, raise_on_error=True)  # 批量插入
        except Exception as e:
            try:
                for er in e.args[1]:  # 和上面的for是一样的，都能取到错误信息
                    f1.write("插入" + str(er["index"]["data"]) + "的数据出错！\r\n")
            except:
                f1.write("插入" + str(bulkList) + "的数据出错！\r\n")
        bulkList = []
    f1.flush()
    f1.close()
    print("over")

main()