目的: MongoShake 数据同步到kafka(json格式),是带有{"Name":"key","Value":"123"} 的json, 实际上这个json 为 {"key":"123"}
转换如下:
import json
''' k:节点name v:节点value cn:当前节点 ''' def setNode(k, v, cn): pass if type(v) == list: # 无论是map还是list,都是list封装 # get type if v: v_type = type(v[0]) # 子节点的type if v_type == dict: # dict c_value = {} for v1 in v: k2 = v1["Name"] v2 = v1["Value"] # v_value[v1["Name"]] = v1["Value"] setNode(k2, v2, c_value) cn[k] = c_value if v_type == list: # list c_value_dict = {} # list_dict c_value_list = [] # list list for v2 in v: # list 子节点可能为list,和 dict if type(v2) == dict: # v2 is dict k2 = v2["Name"] v2 = v2["Value"] setNode(k2, v2, c_value_dict) if type(v2) == list: # # TODO if list in dict,here just add it # c_value_list.append(v2) pass set_list_list(v2, c_value_list) if c_value_list: cn[k] = c_value_list if c_value_dict: cn[k] = c_value_dict if (not v_type == list) and (not v_type == dict): # 子节点的节点为普通类型 cn[k] = v else: # 普通节点 cn[k] = v ''' nodelist: list 节点 cnode: 当前节点_list ''' def set_list_list(nodelist, cnode): c_value_dict = {} # list_dict c_value_list = [] # list list c_value_other = [] # list other for v2 in nodelist: # list 子节点可能为list,和 dict if type(v2) == dict: # list dict # v2 is dict k2 = v2["Name"] v2 = v2["Value"] setNode(k2, v2, c_value_dict) if type(v2) == list: # list list set_list_list(v2, c_value_list) if (not type(v2) == list) and (not type(v2) == dict): c_value_other.append(v2) if c_value_list: cnode.append(c_value_list) if c_value_dict: cnode.append(c_value_dict) if (not c_value_dict) or (not c_value_list): cnode.append(c_value_other) ''' dict to dict ''' def json_json(tb_value_json_dict): etl_dict = {} for kv in tb_value_json_dict: setNode(kv["Name"], kv["Value"], etl_dict) return etl_dict """ json_str to python dict or list """ def json_to_json(tb_value_json_str): etl_dict = {} etl_list = [] tb_value_json = json.loads(tb_value_json_str) if type(tb_value_json) == list: for tb_value_dict in tb_value_json: node_dict = {} for kv in tb_value_dict: setNode(kv["Name"], kv["Value"], node_dict) if node_dict: etl_list.append(node_dict) return etl_list if type(tb_value_json) == dict: for kv in tb_value_json: setNode(kv["Name"], kv["Value"], etl_dict) return etl_dict if __name__ == '__main__': # start_kafkaListener() value_dict = {"ts": 7012463593182461954, "v": 2, "op": "u", "ns": "sea.test", "o": [[{"Name": "_id", "Value": "6150239397b76701bac32d04"}, {"Name": "sea", "Value": "lshan523@163.com"}, {"Name": "age", "Value": 22}, {"Name": "mapmap", "Value": [{"Name": "sea", "Value": [{"Name": "ss1", "Value": "ss1v"}, {"Name": "ss2", "Value": "ss2v"}]}, {"Name": "sa", "Value": "s"}]}, {"Name": "idList", "Value": ["21", "22", "23"]}, {"Name": "idListList", "Value": [["1", "2", "3", "4"], ["21", "22", "23"]]}, {"Name": "iListdict", "Value": [[{"Name": "lld", "Value": "lldv"}], [{"Name": "lld1", "Value": "lldv1"}]]} ]] } tb_values = value_dict["o"] print(tb_values) print("=" * 100) etl = json_json(tb_values[0]) print(etl) print("*" * 100) etl = json_to_json(json.dumps(tb_values)) print(etl)
result :
[[{'Name': '_id', 'Value': '6150239397b76701bac32d04'}, {'Name': 'sea', 'Value': 'test32'}, {'Name': 'age', 'Value': 22}, {'Name': 'mapmap', 'Value': [{'Name': 'sea', 'Value': [{'Name': 'ss1', 'Value': 'ss1v'}, {'Name': 'ss2', 'Value': 'ss2v'}]}, {'Name': 'sa', 'Value': 's'}]}, {'Name': 'idList', 'Value': ['21', '22', '23']}, {'Name': 'idListList', 'Value': [['1', '2', '3', '4'], ['21', '22', '23']]}, {'Name': 'iListdict', 'Value': [[{'Name': 'lld', 'Value': 'lldv'}], [{'Name': 'lld1', 'Value': 'lldv1'}]]}]]
====================================================================================================
{'_id': '6150239397b76701bac32d04', 'sea': 'test32', 'age': 22, 'mapmap': {'sea': {'ss1': 'ss1v', 'ss2': 'ss2v'}, 'sa': 's'}, 'idList': ['21', '22', '23'], 'idListList': [['1', '2', '3', '4'], ['21', '22', '23']], 'iListdict': [{'lld': 'lldv'}, ['lldv'], {'lld1': 'lldv1'}, ['lldv1']]}
****************************************************************************************************
[{'_id': '6150239397b76701bac32d04', 'sea': 'test32', 'age': 22, 'mapmap': {'sea': {'ss1': 'ss1v', 'ss2': 'ss2v'}, 'sa': 's'}, 'idList': ['21', '22', '23'], 'idListList': [['1', '2', '3', '4'], ['21', '22', '23']], 'iListdict': [{'lld': 'lldv'}, ['lldv'], {'lld1': 'lldv1'}, ['lldv1']]}]
