SpringBoot将数据源同步到DataHub；动态推送到DataHub动态采集

Mysql数据示例

核心思想：就是调 http://172.16.5.52:9002/api/v2/graphql 接口推送数据源摄入

请求体：

{
	"variables": {
		"input": {
			"schedule": {
				"timezone": "Asia/Shanghai",
				"interval": "0 0 * * *"
			},
			"name": "数据资产Mysql数据源",
			"type": "mysql",
			"config": {
				"extraArgs": [],
				"executorId": "default",
				"recipe": "{\"source\":{\"type\":\"mysql\",\"config\":{\"include_tables\":true,\"database\":\"datahub\",\"password\":\"datahub\",\"profiling\":{\"enabled\":true,\"profile_table_level_only\":true},\"host_port\":\"172.16.5.52:3306\",\"include_views\":true,\"stateful_ingestion\":{\"enabled\":true},\"username\":\"datahub\"}}}",
				"debugMode": false
			}
		}
	},
	"query": "mutation createIngestionSource($input: UpdateIngestionSourceInput!) { createIngestionSource(input: $input)}",
	"operationName": "createIngestionSource"
}

注意 source 是字符串提交


{
	"source": {
		"type": "mysql",
		"config": {
			"include_tables": true,
			"database": "datahub",
			"password": "datahub",
			"profiling": {
				"enabled": true,
				"profile_table_level_only": true
			},
			"host_port": "172.16.5.52:3306",
			"include_views": true,
			"stateful_ingestion": {
				"enabled": true
			},
			"username": "datahub "
		}
	}
}

第一步 Http登录DataHub获取Cookie标识

 public static List<String> executeDataHubLogin(String loginUrl, String username, String password) throws IOException {
        List<String> SetCookies = new ArrayList<>();
        JSONObject jsonBody = new JSONObject();
        jsonBody.put("username", username);
        jsonBody.put("password", password);
        //  构建Request对象
        RequestBody requestBody = RequestBody.create(MediaType.parse(httpHeaderMediaType), jsonBody.toJSONString());
        Request request = new Request.Builder()
                .addHeader(httpHeaderContentType, httpHeaderMediaType)
                .addHeader("Accept", httpHeaderMediaType)
                .post(requestBody)
                .url(loginUrl)
                .build();
        Response signRsp = null;
        try {
            signRsp = client.newCall(request).execute();
            if (signRsp.isSuccessful()) {
                Headers headers = signRsp.headers();
                List<String> values = headers.values("Set-Cookie");
                for (String key : values) {
                    SetCookies.add(key);
                }
            }
        } catch (IOException e) {
            logger.error("Http连接OpenapiAPi地址：openapiLoginUrl  失败", e);
        }
        return SetCookies;
    }

第二步 拼装好请求参数，携带Cookie标识进行数据源推送


  /**
     * 执行保存数据库
     *
     * @param saveUrl
     */
    public static void executeDataHubSaveDataSource(String saveUrl, JSONObject mapParamas, List<String> cookies) throws IOException {
        String cookieHttpPost = "";
        for (String cookie : cookies) {
            cookieHttpPost += cookie + ";";
        }
        //  构建Request对象
        RequestBody requestBody = RequestBody.create(MediaType.parse(httpHeaderMediaType), mapParamas.toJSONString());
        Request request = new Request.Builder()
                .addHeader(httpHeaderContentType, httpHeaderMediaType)
                .addHeader("Accept", httpHeaderMediaType)
                .addHeader("Cookie", cookieHttpPost)
                .post(requestBody)
                .url(saveUrl)
                .build();
        Response signRsp = null;
        try {
            signRsp = client.newCall(request).execute();
            if (signRsp.isSuccessful()) {
                logger.info("执行成功");
            }
        } catch (IOException e) {
            logger.error("Http连接OpenapiAPi地址：openapiLoginUrl  失败", e);
        }
        logger.error("+========================================");
        logger.info(signRsp.message());
    }

全部代码

@Entity
@JsonIdentityInfo(generator = ObjectIdGenerators.IntSequenceGenerator.class, property = "@id")
@Table(name = "data_source_service")
public class ThirdServiceManager implements Serializable {

    private static final long serialVersionUID = -429065239198706646L;


    @Id
    @GeneratedValue(strategy = GenerationType.IDENTITY)
    private Long id;
    @Column(name = "jdbc_url")
    private String jdbcUrl;

    @Column(name = "cookie")
    private String cookie;

    public String getCookie() {
        return cookie;
    }

    public void setCookie(String cookie) {
        this.cookie = cookie;
    }

    @Column(name = "data_source_id")
    private Long dataSourceId;
    @Column(name = "data_source_type")
    private String dataSourceType;

    @Column(name = "username")
    private String username;

    @Column(name = "data_source_name")
    private String dataSourceName;

    @Column(name = "password")
    private String password;

    @Column(name = "service_name")
    private String serviceName;

    @Column(name = "service_address")
    private String serviceAddress;

    @Column(name = "status")
    private String status;

    public String getStatus() {
        return status;
    }

    public void setStatus(String status) {
        this.status = status;
    }

    @Override
    public String toString() {
        return "ThirdServiceManager{" +
                "id=" + id +
                ", jdbcUrl='" + jdbcUrl + '\'' +
                ", dataSourceId=" + dataSourceId +
                ", dataSourceType='" + dataSourceType + '\'' +
                ", username='" + username + '\'' +
                ", dataSourceName='" + dataSourceName + '\'' +
                ", password='" + password + '\'' +
                ", serviceName='" + serviceName + '\'' +
                ", serviceAddress='" + serviceAddress + '\'' +
                '}';
    }

    public ThirdServiceManager() {
    }

    public ThirdServiceManager(Long id, String jdbcUrl, Long dataSourceId, String dataSourceType, String username, String dataSourceName, String password, String serviceName, String serviceAddress) {
        this.id = id;
        this.jdbcUrl = jdbcUrl;
        this.dataSourceId = dataSourceId;
        this.dataSourceType = dataSourceType;
        this.username = username;
        this.dataSourceName = dataSourceName;
        this.password = password;
        this.serviceName = serviceName;
        this.serviceAddress = serviceAddress;
    }

    public Long getId() {
        return id;
    }

    public void setId(Long id) {
        this.id = id;
    }

    public String getJdbcUrl() {
        return jdbcUrl;
    }

    public void setJdbcUrl(String jdbcUrl) {
        this.jdbcUrl = jdbcUrl;
    }

    public Long getDataSourceId() {
        return dataSourceId;
    }

    public void setDataSourceId(Long dataSourceId) {
        this.dataSourceId = dataSourceId;
    }

    public String getDataSourceType() {
        return dataSourceType;
    }

    public void setDataSourceType(String dataSourceType) {
        this.dataSourceType = dataSourceType;
    }

    public String getUsername() {
        return username;
    }

    public void setUsername(String username) {
        this.username = username;
    }

    public String getDataSourceName() {
        return dataSourceName;
    }

    public void setDataSourceName(String dataSourceName) {
        this.dataSourceName = dataSourceName;
    }

    public String getPassword() {
        return password;
    }

    public void setPassword(String password) {
        this.password = password;
    }

    public String getServiceName() {
        return serviceName;
    }

    public void setServiceName(String serviceName) {
        this.serviceName = serviceName;
    }

    public String getServiceAddress() {
        return serviceAddress;
    }

    public void setServiceAddress(String serviceAddress) {
        this.serviceAddress = serviceAddress;
    }
}

接口

 /**
     * 发布 数据源
     *
     * @param request
     * @param thirdServiceManager
     * @return
     */
    @RequestMapping(value = {"/dataSourceCall"}, method = {RequestMethod.GET, RequestMethod.POST})
    @ResponseBody
    @Transactional
    public Resp<DataSourcePoJo> dataSourceCall(HttpServletRequest request, @RequestBody ThirdServiceManager thirdServiceManager) throws IOException {
        if (thirdServiceManager.getServiceName().contains("血缘分析")) {
            // 先进入登录
            String serviceAddress = thirdServiceManager.getServiceAddress();
            String thatLoginUrl = serviceAddress.replace("/api/v2/graphql", "/logIn");
            String cookie = thirdServiceManager.getCookie();
            JSONObject requestBody = JSONObject.parseObject(cookie, JSONObject.class);
            List<String> listCookies = HttpClientExecute.executeDataHubLogin(thatLoginUrl, requestBody.getString("username"), requestBody.getString("password"));

            // 获取数据源
            String url = thirdServiceManager.getServiceAddress();
            JSONObject jsonObject = new JSONObject();
            jsonObject.put("operationName", "createIngestionSource");
            JSONObject schedule = new JSONObject();
            schedule.put("interval", "0 0 * * *");
            schedule.put("timezone", "Asia/Shanghai");
            JSONObject input = new JSONObject();
            input.put("name", thirdServiceManager.getDataSourceName());
            input.put("type", "mysql");
            input.put("schedule", schedule);
            JSONObject config = new JSONObject();
            String recipe = "{\"source\":{\"type\":\"mysql\",\"config\":{\"host_port\":\"172.16.5.20:3306\",\"database\":null,\"username\":\"root\",\"include_tables\":true,\"include_views\":true,\"profiling\":{\"enabled\":true,\"profile_table_level_only\":true},\"stateful_ingestion\":{\"enabled\":true},\"password\":\"123456\"}}}";
            String jdbcUrl = thirdServiceManager.getJdbcUrl().substring("jdbc:mysql://".length());
            String hostName = jdbcUrl.split(":3306")[0];
            // 使用字符串的lastIndexOf和substring方法来提取数据库名
            int startIndex = jdbcUrl.lastIndexOf('/') + 1;
            int endIndex = jdbcUrl.indexOf('?');
            if (endIndex == -1) {
                // 如果没有查询参数，则直接提取剩余的字符串
                endIndex = jdbcUrl.length();
            }
            String databaseName = jdbcUrl.substring(startIndex, endIndex);
            JSONObject jsonObjectVar = JSONObject.parseObject(recipe);
            JSONObject configVar = jsonObjectVar.getJSONObject("source").getJSONObject("config");
            configVar.put("host_port", hostName + ":3306");
            configVar.put("database", databaseName);
            configVar.put("username", thirdServiceManager.getUsername());
            configVar.put("password", thirdServiceManager.getPassword());
            // 数据源过滤
            JSONObject database_pattern = new JSONObject();
            JSONArray allowS = new JSONArray();
            allowS.add(databaseName);
            database_pattern.put("allow", allowS);
            configVar.put("database_pattern",database_pattern );
            // 这里是recipe 的JSONString,不是JSON
            config.put("recipe", jsonObjectVar.toJSONString());
            config.put("executorId", "default");
            config.put("debugMode", false);
            config.put("extraArgs", new JSONArray());
            input.put("config", config);
            JSONObject variables = new JSONObject();
            variables.put("input", input);
            jsonObject.put("variables", variables);
            jsonObject.put("query", "mutation createIngestionSource($input: UpdateIngestionSourceInput!) { createIngestionSource(input: $input)}");
            logger.error("+========================================");
            logger.info(jsonObject.toJSONString());
//            ResponseEntity<JSONObject> entity = restTemplate.postForEntity(url, jsonObject, JSONObject.class);
            HttpClientExecute.executeDataHubSaveDataSource(url,jsonObject,listCookies);
            Long id = thirdServiceManager.getId();
            // 获取当前的状态
            ThirdServiceManager one = thirdServiceManagerRepository.getOne(id);
            one.setStatus("已发布");
            thirdServiceManagerRepository.saveAndFlush(one);
            return Resp.Ok();
        }
        return Resp.Ok();
    }

推送到DataHub

package com.webank.wedatasphere.qualitis.handler;

import com.alibaba.fastjson.JSONObject;
import okhttp3.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;

public class HttpClientExecute {
    private static Logger logger = LoggerFactory.getLogger(HttpClientExecute.class);
    private final static String httpHeaderMediaType = "application/json;charset=utf-8";

    private final static String httpHeaderContentType = "Content-Type";

    /**
     * 设置默认30秒Http调用超时
     */
    private final static OkHttpClient client = new OkHttpClient.Builder()
            .connectTimeout(30, TimeUnit.SECONDS)
            .readTimeout(30, TimeUnit.SECONDS)
            .callTimeout(30, TimeUnit.SECONDS)
            .build();

    /**
     * 执行登录
     *
     * @param loginUrl
     */
    public static List<String> executeDataHubLogin(String loginUrl, String username, String password) throws IOException {
        List<String> SetCookies = new ArrayList<>();
        JSONObject jsonBody = new JSONObject();
        jsonBody.put("username", username);
        jsonBody.put("password", password);
        //  构建Request对象
        RequestBody requestBody = RequestBody.create(MediaType.parse(httpHeaderMediaType), jsonBody.toJSONString());
        Request request = new Request.Builder()
                .addHeader(httpHeaderContentType, httpHeaderMediaType)
                .addHeader("Accept", httpHeaderMediaType)
                .post(requestBody)
                .url(loginUrl)
                .build();
        Response signRsp = null;
        try {
            signRsp = client.newCall(request).execute();
            if (signRsp.isSuccessful()) {
                Headers headers = signRsp.headers();
                List<String> values = headers.values("Set-Cookie");
                for (String key : values) {
                    SetCookies.add(key);
                }
            }
        } catch (IOException e) {
            logger.error("Http连接OpenapiAPi地址：openapiLoginUrl  失败", e);
        }
        return SetCookies;
    }

    /**
     * 执行保存数据库
     *
     * @param saveUrl
     */
    public static void executeDataHubSaveDataSource(String saveUrl, JSONObject mapParamas, List<String> cookies) throws IOException {
        String cookieHttpPost = "";
        for (String cookie : cookies) {
            cookieHttpPost += cookie + ";";
        }
        //  构建Request对象
        RequestBody requestBody = RequestBody.create(MediaType.parse(httpHeaderMediaType), mapParamas.toJSONString());
        Request request = new Request.Builder()
                .addHeader(httpHeaderContentType, httpHeaderMediaType)
                .addHeader("Accept", httpHeaderMediaType)
                .addHeader("Cookie", cookieHttpPost)
                .post(requestBody)
                .url(saveUrl)
                .build();
        Response signRsp = null;
        try {
            signRsp = client.newCall(request).execute();
            if (signRsp.isSuccessful()) {
                logger.info("执行成功");
            }
        } catch (IOException e) {
            logger.error("Http连接OpenapiAPi地址：openapiLoginUrl  失败", e);
        }
        logger.error("+========================================");
        logger.info(signRsp.message());
    }
}

示例

2024-05-28 16:22:51,432 INFO [qtp1654355983-611] impl.DssMsgBuilderOperationImpl getBuiltMsg: Set cookies from dssMsg: {}
2024-05-28 16:22:51,614 ERROR [qtp1654355983-611] thymeleaf.ThirdServiceDataSourceManager dataSourceCall: +========================================
2024-05-28 16:22:51,615 INFO [qtp1654355983-611] thymeleaf.ThirdServiceDataSourceManager dataSourceCall: {"variables":{"input":{"schedule":{"timezone":"Asia/Shanghai","interval":"0 0 * * *"},"name":"数据资产Mysql数据源","type":"mysql","config":{"extraArgs":[],"executorId":"default","recipe":"{"source":{"type":"mysql","config":{"include_tables":true,"database":"datahub","password":"datahub","profiling":{"enabled":true,"profile_table_level_only":true},"host_port":"172.16.5.52:3306","include_views":true,"stateful_ingestion":{"enabled":true},"username":"datahub"}}}","debugMode":false}}},"query":"mutation createIngestionSource($input: UpdateIngestionSourceInput!) { createIngestionSource(input: $input)}","operationName":"createIngestionSource"}
2024-05-28 16:22:51,616 WARN [OkHttp ConnectionPool] okhttp3.OkHttpClient log: A connection to http://172.16.5.52:9002/ was leaked. Did you forget to close a response body? To see where this was allocated, set the OkHttpClient logger level to FINE: Logger.getLogger(OkHttpClient.class.getName()).setLevel(Level.FINE);
2024-05-28 16:22:51,679 INFO [qtp1654355983-611] handler.HttpClientExecute executeDataHubSaveDataSource: 执行成功
2024-05-28 16:22:51,679 ERROR [qtp1654355983-611] handler.HttpClientExecute executeDataHubSaveDataSource: +========================================
2024-05-28 16:22:51,680 INFO [qtp1654355983-611] handler.HttpClientExecute executeDataHubSaveDataSource: OK
Disconnected from the target VM, address: '127.0.0.1:58195', transport: 'socket'
2024-05-28 16:33:17,314 INFO [Thread-33] concurrent.ThreadPoolTaskExecutor shutdown: Shutting down ExecutorService 'asyncServiceExecutor'
2024-05-28 16:33:17,325 INFO [Thread-33] jpa.LocalContainerEntityManagerFactoryBean destroy: Closing JPA EntityManagerFactory for persistence unit 'default'
2024-05-28 16:33:17,326 INFO [Thread-33] hikari.HikariDataSource close: HikariPool-1 - Shutdown initiated...
2024-05-28 16:33:17,327 INFO [Thread-33] hikari.HikariDataSource close: HikariPool-1 - Shutdown completed.
2024-05-28 16:33:17,332 INFO [Thread-33] server.AbstractConnector doStop: Stopped ServerConnector@1228a579{HTTP/1.1,[http/1.1]}{0.0.0.0:8090}
2024-05-28 16:33:17,332 INFO [Thread-33] server.session stopScavenging: node0 Stopped scavenging
2024-05-28 16:33:17,333 INFO [Thread-33] ContextHandler.application log: Destroying Spring FrameworkServlet 'dispatcherServlet'
2024-05-28 16:33:17,339 INFO [Thread-33] handler.ContextHandler doStop: Stopped o.s.b.w.e.j.JettyEmbeddedWebAppContext@32d3b965

Process finished with exit code -1

image

image

posted on   白嫖老郭  阅读(62)  评论(0)  编辑  收藏  举报

编辑推荐：
· SQL Server 2025 AI相关能力初探
· Linux系列：如何用 C#调用 C方法造成内存泄露
· AI与.NET技术实操系列（二）：开始使用ML.NET
· 记一次.NET内存居高不下排查解决与启示
· 探究高空视频全景AR技术的实现原理
阅读排行：
· 阿里最新开源QwQ-32B，效果媲美deepseek-r1满血版，部署成本又又又降低了！
· SQL Server 2025 AI相关能力初探
· AI编程工具终极对决：字节Trae VS Cursor，谁才是开发者新宠？
· 开源Multi-agent AI智能体框架aevatar.ai，欢迎大家贡献代码
· Manus重磅发布：全球首款通用AI代理技术深度解析与实战指南

导航

< 2025年3月 >
日 一 二 三 四 五 六
23 24 25 26 27 28 1
2 3 4 5 6 7 8
9 10 11 12 13 14 15
16 17 18 19 20 21 22
23 24 25 26 27 28 29
30 31 1 2 3 4 5
点击右上角即可分享
微信分享提示
AI IDE Trae