Addax 备忘

下载编译好的二进制文件

https://github.com/wgzhao/Addax/releases/download/4.0.2/addax-4.0.2.tar.gz

tar -xzf addax-4.0.2.tar.gz
cd addax-4.0.2

 

读http写pgsql

{
    "job": {
        "setting": {
            "speed": {
                "channel": 1,
                "bytes": -1
            }
        },
        "content": [
            {
                "reader": {
                    "name": "httpreader",
                    "parameter": {
                        "connection": [
                            {
                                "url": "http://127.0.0.1/dsideal_yy/admin/new_base/depinfo_getOrgTree"
                            }
                        ],
                        "reqParams": {
                            "org_id": ${org_id}
                        },
                        "resultKey": "list",
                        "method": "GET",
                        "column": [
                            "id",
                            "name"
                        ],
                        "username": "",
                        "password": "",
                        "headers": {
                            "X-Powered-by": "Addax"
                        }
                    }
                },
                "writer": {
                    "name": "postgresqlwriter",
                    "parameter": {
                        "column": [
                            "id",
                            "name"
                        ],
                        "preSql": [
                            "truncate table @table"
                        ],
                        "connection": [
                            {
                                "jdbcUrl": "jdbc:postgresql://127.0.0.1:5432/edu_db",
                                "table": [
                                    "test"
                                ]
                            }
                        ],
                        "username": "root",
                        "password": "123456",
                        "writeMode": "insert"
                    }
                }
            }
        ]
    }
}

执行（-p加动态参数，配置文件中有${org_id}的参数，所以用"-Dorg_id=400195"方式赋值，多个就中间加空格，例："-Dorg_id=400195 -Dperson_id=369587"）

bin/addax.sh -p "-Dorg_id=400195" job/http2stream.json

 

读ES写pgsql

{
    "job": {
        "setting": {
            "speed": {
                "byte": -1,
                "channel": 1
            }
        },
        "content": [
            {
                "reader": {
                    "name": "elasticsearchreader",
                    "parameter": {
                        "endpoint": "http://127.0.0.1:9200",
                        "accessId": "",
                        "accesskey": "",
                        "index": "chat_record",
                        "type": "_doc",
                        "searchType": "dfs_query_then_fetch",
                        "headers": {},
                        "scroll": "3m",
                        "search": [
                            {
                                "size": 2,
                                "_source": [
                                    "chat_type",
                                    "message_type",
                                    "receiver_user_name"
                                ]
                            }
                        ],
                        "column": [
                            "chat_type",
                            "message_type",
                            "receiver_user_name"
                        ]
                    }
                },
                "writer": {
                    "name": "postgresqlwriter",
                    "parameter": {
                        "column": [
                            "chat_type",
                            "message_type",
                            "receiver_user_name"
                        ],
                        "preSql": [
                            "truncate table @table"
                        ],
                        "connection": [
                            {
                                "jdbcUrl": "jdbc:postgresql://127.0.0.1:5432/dsideal_db",
                                "table": [
                                    "test2"
                                ]
                            }
                        ],
                        "username": "root",
                        "password": "123456",
                        "writeMode": "insert"
                    }
                },
                "transformer": [                
                    {
                        "name": "dx_substr",　　　　//将第三个字段的内容，截取前两个字符
                        "parameter": {
                            "columnIndex": 2,
                            "paras": [
                                "0",
                                "2"
                            ]
                        }
                    }
                ]
            }
        ]
    }
}

 

读CSV写pgsql

{
    "job": {
        "setting": {
            "speed": {
                "channel": 2,
                "bytes": -1
            }
        },
        "content": [
            {
                "reader": {
                    "name": "txtfilereader",
                    "parameter": {
                        "path": [
                            "/usr/local/csv/"
                        ],
                        "encoding": "UTF-8",
                        "column": [
                            "*"
                        ],
                        "fieldDelimiter": ",",
                        "skipHeader":true　　　　#是否跳过标题行
                    }
                },
                "writer": {
                    "name": "postgresqlwriter",
                    "parameter": {
                        "column": [
                            "a",
                            "b",
                            "c",
                            "d",
                            "e"
                        ],
                        "preSql": [
                            "truncate table @table"
                        ],
                        "connection": [
                            {
                                "jdbcUrl": "jdbc:postgresql://127.0.0.1:5432/dsideal_db",
                                "table": [
                                    "test3"
                                ]
                            }
                        ],
                        "username": "root",
                        "password": "123456",
                        "writeMode": "insert"
                    }
                }
            }
        ]
    }
}