openmetadata es 存储安全认证添加
openmetadata 默认是没有安全控制的(比较危险),所以简单添加了下对于用户密码的支持(已经pr官方了,应该0.6版本会发布)
需要修改的部分
python 的elasticsearch sink 部分以及rest server 的es 部分
代码修改
- rest api 部分
主要是配置以及search
配置
ElasticSearchConfiguration.java
package org.openmetadata.catalog;
import javax.validation.constraints.NotEmpty;
public class ElasticSearchConfiguration {
@NotEmpty
private String host;
@NotEmpty
private Integer port;
private String username;
private String password;
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public Integer getPort() {
return port;
}
public void setPort(Integer port) {
this.port = port;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
@Override
public String toString() {
return "ElasticSearchConfiguration{" +
"host='" + host + '\'' +
", port=" + port +
", username='" + username + '\'' +
", password='" + password + '\'' +
'}';
}
}
- search api 部分
SearchResource.java
public class SearchResource {
private final RestHighLevelClient client;
private static final Logger LOG = LoggerFactory.getLogger(SearchResource.class);
public SearchResource(ElasticSearchConfiguration esConfig) {
RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(esConfig.getHost(), esConfig.getPort(), "http"));
if(StringUtils.isNotEmpty(esConfig.getUsername())){
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(esConfig.getUsername(), esConfig.getPassword()));
restClientBuilder.setHttpClientConfigCallback(httpAsyncClientBuilder -> {
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
});
}
this.client = new RestHighLevelClient(restClientBuilder);
}
- python sink 部分
ingestion/src/metadata/ingestion/sink/elasticsearch.py
class ElasticSearchConfig(ConfigModel):
es_host: str
es_port: int = 9200
es_username: str
es_password: str
index_tables: Optional[bool] = True
index_topics: Optional[bool] = True
index_dashboards: Optional[bool] = True
index_pipelines: Optional[bool] = True
table_index_name: str = "table_search_index"
topic_index_name: str = "topic_search_index"
dashboard_index_name: str = "dashboard_search_index"
pipeline_index_name: str = "pipeline_search_index"
class ElasticsearchSink(Sink):
""" """
DEFAULT_ELASTICSEARCH_INDEX_MAPPING = TABLE_ELASTICSEARCH_INDEX_MAPPING
@classmethod
def create(
cls, config_dict: dict, metadata_config_dict: dict, ctx: WorkflowContext
):
config = ElasticSearchConfig.parse_obj(config_dict)
metadata_config = MetadataServerConfig.parse_obj(metadata_config_dict)
return cls(ctx, config, metadata_config)
def __init__(
self,
ctx: WorkflowContext,
config: ElasticSearchConfig,
metadata_config: MetadataServerConfig,
) -> None:
self.config = config
self.metadata_config = metadata_config
self.ctx = ctx
self.status = SinkStatus()
self.rest = OpenMetadataAPIClient(self.metadata_config)
self.elasticsearch_doc_type = "_doc"
http_auth = None
if self.config.es_username:
http_auth = (self.config.es_username, self.config.es_password)
self.elasticsearch_client = Elasticsearch(
[
{"host": self.config.es_host, "port": self.config.es_port},
],
http_auth=http_auth,
)
使用
使用主要是对于es 开启basic auth 然后api 以及sink 部分配置就可以了
rest api server
openmetadata.yaml (使用了默认密码,可以自己调整)
elasticsearch:
host: localhost
port: 9200
username: admin
password: admin
sink
pipeline 中的metadata_to_es.json 文件
{
"source": {
"type": "metadata",
"config": {
"include_tables": "true",
"include_topics": "true",
"include_dashboards": "true",
"limit_records": 10
}
},
"sink": {
"type": "elasticsearch",
"config": {
"index_tables": "true",
"index_topics": "true",
"index_dashboards": "true",
"es_host": "localhost",
"es_username":"admin",
"es_password":"admin",
"es_port": 9200
}
},
"metadata_server": {
"type": "metadata-server",
"config": {
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
}
}
说明
因为python 模块还没有发布,所以不知直接使用,但是可以先安装,然后通过修改源码的模式打补丁,这样就可以使用了
参考效果
参考资料
https://open-metadata.org/
https://docs.open-metadata.org/install/metadata-ingestion/ingest-sample-data
https://github.com/open-metadata/OpenMetadata/pull/894