testmysqltoelasticsearch71.json 未测试,仅参考

{
  "job": {
    "setting": {
      "speed": {
        "channel": 2
      }
    },
    "content": [
      {
        "reader": {
          "name": "mysqlreader",
          "parameter": {
            "username": "datax",
            "password": "123456",
            "where":"updated_at>='${start_time} 00:00:00' and updated_at<='${end_time} 23:59:59'",
            "column": [
              "id",
              "app_id",        
              "collection_phone",
              "transaction_number",
              "pay_amount",             
              "if(auto_tags is null,'',replace(replace(replace(auto_tags,'[',''),']',''),'\"','')) as auto_tags",
              "if(manual_tags is null,'',replace(replace(replace(manual_tags,'[',''),']',''),'\"','')) as manual_tags",
              "if(latest_days_ordered_at is null,'',replace(replace(latest_days_ordered_at,'[',''),']','')) as latest_days_ordered_at",
              "if(latest_days_paid_at is null,'',replace(replace(latest_days_paid_at,'[',''),']','')) as latest_days_paid_at",
              "if(latest_days_visited_at is null,'',replace(replace(latest_days_visited_at,'[',''),']','')) as latest_days_visited_at",
              "latest_ordered_at",            
              "visited_products",
              "ordered_products"
            ],
            "connection": [
              {
                "jdbcUrl": ["jdbc:mysql://127.0.0.1:3306/db_user?com.mysql.jdbc.faultInjection.serverCharsetIndex=45"],
                "table": [
                  "user"
                ]
              }
            ]
          }
        },
        "writer": {
          "name": "elasticsearchwriter",
          "parameter": {
            "endpoint": "http://127.0.0.1:9200",
            "accessId": "elastic",
            "accessKey": "123456",
            "index":"user",
            "type":"traces",
            "settings": {"index" :{"number_of_shards": 5, "number_of_replicas": 1}},
            "batchSize": 5000,
            "splitter": ",",
            "column": [
              {"name":"pk","type":"id"},
              {"name":"app_id","type":"keyword"},            
              {"name":"collection_phone","type":"keyword"},
              {"name":"transaction_number","type":"integer"},
              {"name":"pay_amount","type":"integer"},
              {"name":"auto_tags","type":"keyword","array":true},
              {"name":"manual_tags","type":"keyword","array":true},
              {"name":"latest_days_ordered_at","type":"long","array":true},
              {"name":"latest_days_paid_at","type":"long","array":true},
              {"name":"latest_days_visited_at","type":"long","array":true},
              {"name":"latest_ordered_at","type":"long"},           
              {"name":"visited_products","type":"nested"},
              {"name":"ordered_products","type":"nested"}
            ]
          }
        }
      }
    ]
  }
}

 

posted @ 2021-09-08 14:47  DarJeely  阅读(21)  评论(0编辑  收藏  举报