[ERROR: metadata_store_server_main.cc:550] Connection Aborted with error: ABORTED: There are a subset of tables in MLMD instance. This may be due to concurrent connection to the empty database. Please retry the connection. checks: 15 errors: 14, present tables: type_table, missing tables: parent_type_table, type_property_table, artifact_table, artifact_property_table, execution_table, execution_property_table, event_table, event_path_table, mlmd_env_table, context_table, parent_context_table, context_property_table, association_table, attribution_table ]
After deploying standalone kubeflow pipeline on kubernetes cluster,
kubectl logs metadata-grpc-deployment-pod-name -n kubeflow
-->
I0113 09:00:13.597116 1 metadata_store_server_main.cc:551] Retry attempt 3
W0113 09:00:15.678181 1 metadata_store_server_main.cc:550] Connection Aborted with error: ABORTED: There are a subset of tables in MLMD instance. This may be due to concurrent connection to the empty database. Please retry the connection. checks: 15 errors: 14, present tables: type_table, missing tables: parent_type_table, type_property_table, artifact_table, artifact_property_table, execution_table, execution_property_table, event_table, event_path_table, mlmd_env_table, context_table, parent_context_table, context_property_table, association_table, attribution_table Errors: INTERNAL: mysql_query failed: errno: Table 'metadb.ParentType' doesn't exist, error: Table 'metadb.ParentType' doesn't exist [mysql-error-info='\x08\xfa\x08']
INTERNAL: mysql_query failed: errno: Table 'metadb.TypeProperty' doesn't exist, error: Table 'metadb.TypeProperty' doesn't exist [mysql-error-info='\x08\xfa\x08']
[ANALYSIS]
1. see the python source code of the function who raised the error in the log, to see the concrete cause of the error.
"""
github-repository ml_metadata/ml_metadata/metadata_store/metadata_store_server_main.cc
"""
int main(int argc, char** argv) {
gflags::ParseCommandLineFlags(&argc, &argv, true);
if ((FLAGS_grpc_port) <= 0) {
LOG(ERROR) << "grpc_port is invalid: " << (FLAGS_grpc_port);
return -1;
}
...
// Creates a metadata_store in the main thread and init schema if necessary.
std::unique_ptr<ml_metadata::MetadataStore> metadata_store;
absl::Status status = ml_metadata::CreateMetadataStore(
connection_config, server_config.migration_options(), &metadata_store);
for (int i = 0; i < (FLAGS_metadata_store_connection_retries);
i++) {
if (status.ok() || !absl::IsAborted(status)) {
break;
}
LOG(WARNING) << "Connection Aborted with error: " << status;
LOG(INFO) << "Retry attempt " << i;
status = ml_metadata::CreateMetadataStore( ## (1)
connection_config, server_config.migration_options(), &metadata_store);
}
CHECK_EQ(absl::OkStatus(), status)
<< "MetadataStore cannot be created with the given connection config.";
// At this point, schema initialization and migration are done.
metadata_store.reset();
# github-repository ml-metadata/ml_metadata/metadata_store/metadata_access_object.h
absl::Status CreateMetadataStore(const ConnectionConfig& config,
const MigrationOptions& options,
std::unique_ptr<MetadataStore>* result) {
switch (config.config_case()) {
case ConnectionConfig::CONFIG_NOT_SET:
// TODO(b/123345695): make this longer when that bug is resolved.
// Must specify a metadata store type.
return absl::InvalidArgumentError("Unset");
case ConnectionConfig::kFakeDatabase:
// Creates an in-memory SQLite database for testing.
return CreateSqliteMetadataStore(SqliteMetadataSourceConfig(), options,
result);
case ConnectionConfig::kMysql:
return CreateMySQLMetadataStore(config.mysql(), options, result); ## (2)
case ConnectionConfig::kSqlite:
return CreateSqliteMetadataStore(config.sqlite(), options, result);
case ConnectionConfig::kPostgresql:
return CreatePostgreSQLMetadataStore(config.postgresql(), options,
result);
default:
return absl::UnimplementedError("Unknown database type.");
}
}
// namespace ml_metadata
# github-repository ml-metadata/ml_metadata/metadata_store/metadata_store_factory.cc
namespace ml_metadata {
namespace {
absl::Status CreateMySQLMetadataStore(const MySQLDatabaseConfig& config,
const MigrationOptions& migration_options,
std::unique_ptr<MetadataStore>* result) {
auto metadata_source = std::make_unique<MySqlMetadataSource>(config);
auto transaction_executor =
std::make_unique<RdbmsTransactionExecutor>(metadata_source.get());
MLMD_RETURN_IF_ERROR(MetadataStore::Create(
util::GetMySqlMetadataSourceQueryConfig(), migration_options,
std::move(metadata_source), std::move(transaction_executor), result));
return (*result)->InitMetadataStoreIfNotExists( ## (3)
migration_options.enable_upgrade_migration());
}
# github-repository ml-metadata/ml_metadata/metadata_store/query_config_executor.cc
absl::Status QueryConfigExecutor::InitMetadataSourceIfNotExists( ## (3)
const bool enable_upgrade_migration) {
// If |query_schema_version_| is given, then the query executor is expected to
// work with an existing db with an earlier schema version equals to that.
if (query_schema_version()) {
return CheckSchemaVersionAlignsWithQueryVersion();
}
// When working at head, we reuse existing db or create a new db.
// check db version, and make it to align with the lib version.
MLMD_RETURN_IF_ERROR(
UpgradeMetadataSourceIfOutOfDate(enable_upgrade_migration));
// if lib and db versions align, we check the required tables for the lib.
std::vector<std::pair<absl::Status, std::string>> checks;
checks.push_back({CheckTypeTable(), "type_table"});
checks.push_back({CheckParentTypeTable(), "parent_type_table"});
checks.push_back({CheckTypePropertyTable(), "type_property_table"});
checks.push_back({CheckArtifactTable(), "artifact_table"});
checks.push_back({CheckArtifactPropertyTable(), "artifact_property_table"});
checks.push_back({CheckExecutionTable(), "execution_table"});
checks.push_back({CheckExecutionPropertyTable(), "execution_property_table"});
checks.push_back({CheckEventTable(), "event_table"});
checks.push_back({CheckEventPathTable(), "event_path_table"});
checks.push_back({CheckMLMDEnvTable(), "mlmd_env_table"});
checks.push_back({CheckContextTable(), "context_table"});
checks.push_back({CheckParentContextTable(), "parent_context_table"});
checks.push_back({CheckContextPropertyTable(), "context_property_table"});
checks.push_back({CheckAssociationTable(), "association_table"});
checks.push_back({CheckAttributionTable(), "attribution_table"});
std::vector<std::string> missing_schema_error_messages;
std::vector<std::string> successful_checks;
std::vector<std::string> failing_checks;
for (const auto& check_pair : checks) {
const absl::Status& check = check_pair.first;
const std::string& name = check_pair.second;
if (!check.ok()) {
missing_schema_error_messages.push_back(check.ToString());
failing_checks.push_back(name);
} else {
successful_checks.push_back(name);
}
}
// all table required by the current lib version exists
if (missing_schema_error_messages.empty()) return absl::OkStatus();
// some table exists, but not all.
if (checks.size() != missing_schema_error_messages.size()) {
return absl::AbortedError(absl::StrCat(
"There are a subset of tables in MLMD instance. This may be due to "
"concurrent connection to the empty database. " ## [Concrete error
"Please retry the connection. checks: ", ## message]
checks.size(), " errors: ", missing_schema_error_messages.size(),
", present tables: ", absl::StrJoin(successful_checks, ", "),
", missing tables: ", absl::StrJoin(failing_checks, ", "),
" Errors: ", absl::StrJoin(missing_schema_error_messages, "\n")));
}
// no table exists, then init the MetadataSource
return InitMetadataSource(); ## (4)
}
absl::Status QueryConfigExecutor::InitMetadataSource() {
MLMD_RETURN_IF_ERROR(ExecuteQuery(query_config_.create_type_table()));
MLMD_RETURN_IF_ERROR(
ExecuteQuery(query_config_.create_type_property_table())); ## (5)
MLMD_RETURN_IF_ERROR(ExecuteQuery(query_config_.create_parent_type_table()));
MLMD_RETURN_IF_ERROR(ExecuteQuery(query_config_.create_artifact_table())); ## (5)
MLMD_RETURN_IF_ERROR(
ExecuteQuery(query_config_.create_artifact_property_table()));
MLMD_RETURN_IF_ERROR(ExecuteQuery(query_config_.create_execution_table()));
MLMD_RETURN_IF_ERROR(
ExecuteQuery(query_config_.create_execution_property_table()));
MLMD_RETURN_IF_ERROR(ExecuteQuery(query_config_.create_event_table()));
MLMD_RETURN_IF_ERROR(ExecuteQuery(query_config_.create_event_path_table()));
MLMD_RETURN_IF_ERROR(ExecuteQuery(query_config_.create_mlmd_env_table()));
MLMD_RETURN_IF_ERROR(ExecuteQuery(query_config_.create_context_table()));
MLMD_RETURN_IF_ERROR(
ExecuteQuery(query_config_.create_context_property_table()));
MLMD_RETURN_IF_ERROR(
ExecuteQuery(query_config_.create_parent_context_table()));
MLMD_RETURN_IF_ERROR(ExecuteQuery(query_config_.create_association_table()));
MLMD_RETURN_IF_ERROR(ExecuteQuery(query_config_.create_attribution_table()));
for (const MetadataSourceQueryConfig::TemplateQuery& index_query :
query_config_.secondary_indices()) {
const absl::Status status = ExecuteQuery(index_query);
// For databases (e.g., MySQL), idempotency of indexing creation is not
// supported well. We handle it here and covered by the `InitForReset` test.
if (!status.ok() && absl::StrContains(std::string(status.message()),
"Duplicate key name")) {
continue;
}
MLMD_RETURN_IF_ERROR(status);
}
int64_t library_version = GetLibraryVersion();
absl::Status insert_schema_version_status =
InsertSchemaVersion(library_version);
if (!insert_schema_version_status.ok()) {
int64_t db_version = -1;
MLMD_RETURN_IF_ERROR(GetSchemaVersion(&db_version));
if (db_version != library_version) {
return absl::DataLossError(absl::StrCat(
"The database cannot be initialized with the schema_version in the "
"current library. Current library version: ",
library_version, ", the db version on record is: ", db_version,
". It may result from a data race condition caused by other "
"concurrent MLMD's migration procedures."));
}
}
return absl::OkStatus();
}
ml-metadata/ml_metadata_util/metadata_source_query_config.cc
namespace ml_metadata {
namespace util {
namespace {
...
// Template queries for MySQLMetadataSources.
// no-lint to support vc (C2026) 16380 max length for char[].
const std::string kMySQLMetadataSourceQueryConfig = absl::StrCat( // NOLINT
R"pb(
metadata_source_type: MYSQL_METADATA_SOURCE
select_last_insert_id { query: " SELECT last_insert_id(); " }
select_type_by_name {
query: " SELECT `id`, `name`, `version`, `description`, "
" `input_type`, `output_type` FROM `Type` "
" WHERE name = $0 AND version IS NULL AND type_kind = $1 "
" LOCK IN SHARE MODE; "
parameter_num: 2
}
select_type_by_name_and_version {
query: " SELECT `id`, `name`, `version`, `description`, "
" `input_type`, `output_type` FROM `Type` "
" WHERE name = $0 AND version = $1 AND type_kind = $2 "
" LOCK IN SHARE MODE; "
parameter_num: 3
}
select_types_by_names {
query: " SELECT `id`, `name`, `version`, `description`, "
" `input_type`, `output_type` FROM `Type` "
" WHERE name IN ($0) AND version IS NULL AND type_kind = $1 "
" LOCK IN SHARE MODE; "
parameter_num: 2
}
select_types_by_names_and_versions {
query: " SELECT `id`, `name`, `version`, `description`, "
" `input_type`, `output_type` FROM `Type` "
" WHERE (name, version) IN ($0) AND type_kind = $1 "
" LOCK IN SHARE MODE; "
parameter_num: 2
}
select_context_by_id {
query: " SELECT C.id, C.type_id, C.name, C.external_id, "
" C.create_time_since_epoch, C.last_update_time_since_epoch, "
" T.name AS `type`, T.version AS type_version, "
" T.description AS type_description, "
" T.external_id AS type_external_id "
" FROM `Context` AS C "
" LEFT JOIN `Type` AS T ON (T.id = C.type_id) "
" WHERE C.id IN ($0) LOCK IN SHARE MODE; "
parameter_num: 1
}
select_execution_by_id {
query: " SELECT E.id, E.type_id, E.last_known_state, E.name, "
" E.external_id, E.create_time_since_epoch, "
" E.last_update_time_since_epoch, "
" T.name AS `type`, T.version AS type_version, "
" T.description AS type_description, "
" T.external_id AS type_external_id "
" FROM `Execution` AS E "
" LEFT JOIN `Type` AS T "
" ON (T.id = E.type_id) "
" WHERE E.id IN ($0) LOCK IN SHARE MODE; "
parameter_num: 1
}
select_artifact_by_id {
query: " SELECT A.id, A.type_id, A.uri, A.state, A.name, "
" A.external_id, A.create_time_since_epoch, "
" A.last_update_time_since_epoch, "
" T.name AS `type`, T.version AS type_version, "
" T.description AS type_description, "
" T.external_id AS type_external_id "
" FROM `Artifact` AS A "
" LEFT JOIN `Type` AS T "
" ON (T.id = A.type_id) "
" WHERE A.id IN ($0) LOCK IN SHARE MODE; "
parameter_num: 1
}
select_parent_type_by_type_id {
query: " SELECT `type_id`, `parent_type_id` "
" FROM `ParentType` WHERE type_id IN ($0) "
" LOCK IN SHARE MODE; "
parameter_num: 1
}
create_type_table {
query: " CREATE TABLE IF NOT EXISTS `Type` ( "
" `id` INT PRIMARY KEY AUTO_INCREMENT, "
" `name` VARCHAR(255) NOT NULL, "
" `version` VARCHAR(255), "
" `type_kind` TINYINT(1) NOT NULL, "
" `description` TEXT, "
" `input_type` TEXT, "
" `output_type` TEXT, "
" `external_id` VARCHAR(255) UNIQUE "
" ); "
}
create_artifact_table { ## (6)
query: " CREATE TABLE IF NOT EXISTS `Artifact` ( "
" `id` INTEGER PRIMARY KEY AUTO_INCREMENT, "
" `type_id` INT NOT NULL, "
" `uri` TEXT, "
" `state` INT, "
" `name` VARCHAR(255), "
" `external_id` VARCHAR(255) UNIQUE, "
" `create_time_since_epoch` BIGINT NOT NULL DEFAULT 0, "
" `last_update_time_since_epoch` BIGINT NOT NULL DEFAULT 0, "
" CONSTRAINT UniqueArtifactTypeName UNIQUE(`type_id`, `name`) "
" ); "
}
create_artifact_property_table {
query: " CREATE TABLE IF NOT EXISTS `ArtifactProperty` ( "
" `artifact_id` INT NOT NULL, "
" `name` VARCHAR(255) NOT NULL, "
" `is_custom_property` TINYINT(1) NOT NULL, "
" `int_value` INT, "
" `double_value` DOUBLE, "
" `string_value` MEDIUMTEXT, "
" `byte_value` MEDIUMBLOB, "
" `proto_value` MEDIUMBLOB, "
" `bool_value` BOOLEAN, "
" PRIMARY KEY (`artifact_id`, `name`, `is_custom_property`)); "
}
create_execution_table {
query: " CREATE TABLE IF NOT EXISTS `Execution` ( "
" `id` INTEGER PRIMARY KEY AUTO_INCREMENT, "
" `type_id` INT NOT NULL, "
" `last_known_state` INT, "
" `name` VARCHAR(255), "
" `external_id` VARCHAR(255) UNIQUE, "
" `create_time_since_epoch` BIGINT NOT NULL DEFAULT 0, "
" `last_update_time_since_epoch` BIGINT NOT NULL DEFAULT 0, "
" CONSTRAINT UniqueExecutionTypeName UNIQUE(`type_id`, `name`) "
" ); "
}
create_execution_property_table {
query: " CREATE TABLE IF NOT EXISTS `ExecutionProperty` ( "
" `execution_id` INT NOT NULL, "
" `name` VARCHAR(255) NOT NULL, "
" `is_custom_property` TINYINT(1) NOT NULL, "
" `int_value` INT, "
" `double_value` DOUBLE, "
" `string_value` MEDIUMTEXT, "
" `byte_value` MEDIUMBLOB, "
" `proto_value` MEDIUMBLOB, "
" `bool_value` BOOLEAN, "
" PRIMARY KEY (`execution_id`, `name`, `is_custom_property`)); "
}
create_context_table {
query: " CREATE TABLE IF NOT EXISTS `Context` ( "
" `id` INTEGER PRIMARY KEY AUTO_INCREMENT, "
" `type_id` INT NOT NULL, "
" `name` VARCHAR(255) NOT NULL, "
" `external_id` VARCHAR(255) UNIQUE, "
" `create_time_since_epoch` BIGINT NOT NULL DEFAULT 0, "
" `last_update_time_since_epoch` BIGINT NOT NULL DEFAULT 0, "
" UNIQUE(`type_id`, `name`) "
" ); "
}
create_context_property_table {
query: " CREATE TABLE IF NOT EXISTS `ContextProperty` ( "
" `context_id` INT NOT NULL, "
" `name` VARCHAR(255) NOT NULL, "
" `is_custom_property` TINYINT(1) NOT NULL, "
" `int_value` INT, "
" `double_value` DOUBLE, "
" `string_value` MEDIUMTEXT, "
" `byte_value` MEDIUMBLOB, "
" `proto_value` MEDIUMBLOB, "
" `bool_value` BOOLEAN, "
" PRIMARY KEY (`context_id`, `name`, `is_custom_property`)); "
}
create_event_table {
query: " CREATE TABLE IF NOT EXISTS `Event` ( "
" `id` INTEGER PRIMARY KEY AUTO_INCREMENT, "
" `artifact_id` INT NOT NULL, "
" `execution_id` INT NOT NULL, "
" `type` INT NOT NULL, "
" `milliseconds_since_epoch` BIGINT, "
" CONSTRAINT UniqueEvent UNIQUE( "
" `artifact_id`, `execution_id`, `type`) "
" ); "
}
create_association_table {
query: " CREATE TABLE IF NOT EXISTS `Association` ( "
" `id` INTEGER PRIMARY KEY AUTO_INCREMENT, "
" `context_id` INT NOT NULL, "
" `execution_id` INT NOT NULL, "
" UNIQUE(`context_id`, `execution_id`) "
" ); "
}
create_attribution_table {
query: " CREATE TABLE IF NOT EXISTS `Attribution` ( "
" `id` INTEGER PRIMARY KEY AUTO_INCREMENT, "
" `context_id` INT NOT NULL, "
" `artifact_id` INT NOT NULL, "
" UNIQUE(`context_id`, `artifact_id`) "
" ); "
}
In summary, The ERROR is raised due to "// some table exists, but not all.", namely some table exists, but not all, in database metadb. And, when this error raised, retry,
but when retry, still meet the same condition: "// some table exists, but not all.", since not try to create missing tables before retry, so here, retry is meaningless.
# github-repository ml-metadata/ml_metadata/metadata_store/query_config_executor.cc
absl::Status QueryConfigExecutor::InitMetadataSourceIfNotExists( ## (3)
// some table exists, but not all.
if (checks.size() != missing_schema_error_messages.size()) {
return absl::AbortedError(absl::StrCat(
"There are a subset of tables in MLMD instance. This may be due to "
"concurrent connection to the empty database. " ## [Concrete error
"Please retry the connection. checks: ", ## message]
checks.size(), " errors: ", missing_schema_error_messages.size(),
", present tables: ", absl::StrJoin(successful_checks, ", "),
", missing tables: ", absl::StrJoin(failing_checks, ", "),
" Errors: ", absl::StrJoin(missing_schema_error_messages, "\n")));
}
"""
github-repository ml_metadata/ml_metadata/metadata_store/metadata_store_server_main.cc
"""
int main(int argc, char** argv) {
gflags::ParseCommandLineFlags(&argc, &argv, true);
for (int i = 0; i < (FLAGS_metadata_store_connection_retries);
i++) {
if (status.ok() || !absl::IsAborted(status)) {
break;
}
LOG(WARNING) << "Connection Aborted with error: " << status;
LOG(INFO) << "Retry attempt " << i;
status = ml_metadata::CreateMetadataStore( ## (1)
connection_config, server_config.migration_options(), &metadata_store);
}
CHECK_EQ(absl::OkStatus(), status)
<< "MetadataStore cannot be created with the given connection config.";
2. So, why "// some table exists, but not all." occurs?
From the function call chain following InitMetadataSource(), It can be seen at last, table is created by sql sentence. Only table Type has been created, all other tables have not, but all tables are created by the same sql sentence --
"CREATE TABLE table_name IF NOT EXISTS ...",
and the error message says "There are a subset of tables in MLMD instance., This may be due to concurrent connection to the empty database.", it can be inferred that maybe there are two threads who access the empty database metadb concurrently, and both of them InitMetadataSourceIfNotExists(), and after one thread finishes creating table Type, and is creating the second table, the another thread meet the condition: "// some table exists, but not all.", so the another thread will abort, then the process of the tow threads will exit with error since one of its thread has raised error, then the thread who is creating the second table also has to exit since its process has exited.
[SOLUTION]
Do things done by QueryConfigExecutor::InitMetadataSource() (in file github-repository:ml-metadata/ml_metadata/metadata_store/query_config_executor.cc) manually, using sql sentences.when this done, then when pod metadata-grpc-deployment 'crash back off' (namely container exit with error,recreate container) it will be ok:
- Create the missing tables using sql sentence manually.
kubectl exec --it mysql-pod-name -n kubeflow -- bash
--> entered pod mysql:
mysql> use metadb;
mysql> show tables;
"""
copy from ml-#metadata/ml_metadata_util/metadata_source_query_config.cc
"""
mysql> CREATE TABLE IF NOT EXISTS `Artifact` (
`id` INTEGER PRIMARY KEY AUTO_INCREMENT,
`type_id` INT NOT NULL,
`uri` TEXT,
`state` INT,
`name` VARCHAR(255),
`external_id` VARCHAR(255) UNIQUE,
`create_time_since_epoch` BIGINT NOT NULL DEFAULT 0,
`last_update_time_since_epoch` BIGINT NOT NULL DEFAULT 0,
CONSTRAINT UniqueArtifactTypeName UNIQUE(`type_id`, `name`)
);
"""
in file ml-metadata/ml_metadata_util/metadata_source_query_config.cc
"""
// Template queries for MySQLMetadataSources.
// no-lint to support vc (C2026) 16380 max length for char[].
const std::string kMySQLMetadataSourceQueryConfig = absl::StrCat( // NOLINT
R"pb(
...
create_type_table {
query: " CREATE TABLE IF NOT EXISTS `Type` ( "
" `id` INT PRIMARY KEY AUTO_INCREMENT, "
" `name` VARCHAR(255) NOT NULL, "
" `version` VARCHAR(255), "
" `type_kind` TINYINT(1) NOT NULL, "
" `description` TEXT, "
" `input_type` TEXT, "
" `output_type` TEXT, "
" `external_id` VARCHAR(255) UNIQUE "
" ); "
}
create_artifact_table {
query: " CREATE TABLE IF NOT EXISTS `Artifact` ( "
" `id` INTEGER PRIMARY KEY AUTO_INCREMENT, "
" `type_id` INT NOT NULL, "
" `uri` TEXT, "
" `state` INT, "
" `name` VARCHAR(255), "
" `external_id` VARCHAR(255) UNIQUE, "
" `create_time_since_epoch` BIGINT NOT NULL DEFAULT 0, "
" `last_update_time_since_epoch` BIGINT NOT NULL DEFAULT 0, "
" CONSTRAINT UniqueArtifactTypeName UNIQUE(`type_id`, `name`) "
" ); "
}
create_artifact_property_table {
query: " CREATE TABLE IF NOT EXISTS `ArtifactProperty` ( "
" `artifact_id` INT NOT NULL, "
" `name` VARCHAR(255) NOT NULL, "
" `is_custom_property` TINYINT(1) NOT NULL, "
" `int_value` INT, "
" `double_value` DOUBLE, "
" `string_value` MEDIUMTEXT, "
" `byte_value` MEDIUMBLOB, "
" `proto_value` MEDIUMBLOB, "
" `bool_value` BOOLEAN, "
" PRIMARY KEY (`artifact_id`, `name`, `is_custom_property`)); "
}
create_execution_table {
query: " CREATE TABLE IF NOT EXISTS `Execution` ( "
" `id` INTEGER PRIMARY KEY AUTO_INCREMENT, "
" `type_id` INT NOT NULL, "
" `last_known_state` INT, "
" `name` VARCHAR(255), "
" `external_id` VARCHAR(255) UNIQUE, "
" `create_time_since_epoch` BIGINT NOT NULL DEFAULT 0, "
" `last_update_time_since_epoch` BIGINT NOT NULL DEFAULT 0, "
" CONSTRAINT UniqueExecutionTypeName UNIQUE(`type_id`, `name`) "
" ); "
}
create_execution_property_table {
query: " CREATE TABLE IF NOT EXISTS `ExecutionProperty` ( "
" `execution_id` INT NOT NULL, "
" `name` VARCHAR(255) NOT NULL, "
" `is_custom_property` TINYINT(1) NOT NULL, "
" `int_value` INT, "
" `double_value` DOUBLE, "
" `string_value` MEDIUMTEXT, "
" `byte_value` MEDIUMBLOB, "
" `proto_value` MEDIUMBLOB, "
" `bool_value` BOOLEAN, "
" PRIMARY KEY (`execution_id`, `name`, `is_custom_property`)); "
}
create_context_table {
query: " CREATE TABLE IF NOT EXISTS `Context` ( "
" `id` INTEGER PRIMARY KEY AUTO_INCREMENT, "
" `type_id` INT NOT NULL, "
" `name` VARCHAR(255) NOT NULL, "
" `external_id` VARCHAR(255) UNIQUE, "
" `create_time_since_epoch` BIGINT NOT NULL DEFAULT 0, "
" `last_update_time_since_epoch` BIGINT NOT NULL DEFAULT 0, "
" UNIQUE(`type_id`, `name`) "
" ); "
}
create_context_property_table {
query: " CREATE TABLE IF NOT EXISTS `ContextProperty` ( "
" `context_id` INT NOT NULL, "
" `name` VARCHAR(255) NOT NULL, "
" `is_custom_property` TINYINT(1) NOT NULL, "
" `int_value` INT, "
" `double_value` DOUBLE, "
" `string_value` MEDIUMTEXT, "
" `byte_value` MEDIUMBLOB, "
" `proto_value` MEDIUMBLOB, "
" `bool_value` BOOLEAN, "
" PRIMARY KEY (`context_id`, `name`, `is_custom_property`)); "
}
create_event_table {
query: " CREATE TABLE IF NOT EXISTS `Event` ( "
" `id` INTEGER PRIMARY KEY AUTO_INCREMENT, "
" `artifact_id` INT NOT NULL, "
" `execution_id` INT NOT NULL, "
" `type` INT NOT NULL, "
" `milliseconds_since_epoch` BIGINT, "
" CONSTRAINT UniqueEvent UNIQUE( "
" `artifact_id`, `execution_id`, `type`) "
" ); "
}
create_association_table {
query: " CREATE TABLE IF NOT EXISTS `Association` ( "
" `id` INTEGER PRIMARY KEY AUTO_INCREMENT, "
" `context_id` INT NOT NULL, "
" `execution_id` INT NOT NULL, "
" UNIQUE(`context_id`, `execution_id`) "
" ); "
}
create_attribution_table {
query: " CREATE TABLE IF NOT EXISTS `Attribution` ( "
" `id` INTEGER PRIMARY KEY AUTO_INCREMENT, "
" `context_id` INT NOT NULL, "
" `artifact_id` INT NOT NULL, "
" UNIQUE(`context_id`, `artifact_id`) "
" ); "
}
create_event_path_table {
query: " CREATE TABLE IF NOT EXISTS `EventPath` ( "
" `event_id` INT NOT NULL, "
" `is_index_step` TINYINT(1) NOT NULL, "
" `step_index` INT, "
" `step_key` TEXT "
" ); "
}
create_mlmd_env_table {
query: " CREATE TABLE IF NOT EXISTS `MLMDEnv` ( "
" `schema_version` INTEGER PRIMARY KEY "
" ); "
}
create_parent_context_table {
query: " CREATE TABLE IF NOT EXISTS `ParentContext` ( "
" `context_id` INT NOT NULL, "
" `parent_context_id` INT NOT NULL, "
" PRIMARY KEY (`context_id`, `parent_context_id`)); "
}
create_parent_type_table {
query: " CREATE TABLE IF NOT EXISTS `ParentType` ( "
" `type_id` INT NOT NULL, "
" `parent_type_id` INT NOT NULL, "
" PRIMARY KEY (`type_id`, `parent_type_id`)); "
}
After creating missing tables:
mysal> use database;
mysql> show tables;
+-------------------+
| Tables_in_metadb |
+-------------------+
| Artifact |
| ArtifactProperty |
| Association |
| Attribution |
| Context |
| ContextProperty |
| Event |
| EventPath |
| Execution |
| ExecutionProperty |
| MLMDEnv |
| ParentContext |
| ParentType |
| Type |
| TypeProperty |
+-------------------+
15 rows in set (0.01 sec)
- execute
query_config_.secondary_indices()using sql sentences:
"""
in file ml-metadata/ml_metadata_util/metadata_source_query_config.cc
"""
# secondary indices in the current schema.
secondary_indices {
# MySQL does not support arbitrary length string index. Only prefix
# index
# is supported. Max size for 5.6/5.7 is 255 char for utf8 charset.
query: " ALTER TABLE `Artifact` "
" ADD INDEX `idx_artifact_uri`(`uri`(255)), "
" ADD INDEX `idx_artifact_create_time_since_epoch` "
" (`create_time_since_epoch`), "
" ADD INDEX `idx_artifact_last_update_time_since_epoch` "
" (`last_update_time_since_epoch`); "
}
secondary_indices {
query: " ALTER TABLE `Event` "
" ADD INDEX `idx_event_execution_id` (`execution_id`); "
}
secondary_indices {
query: " ALTER TABLE `ParentContext` "
" ADD INDEX "
" `idx_parentcontext_parent_context_id` (`parent_context_id`); "
}
secondary_indices {
query: " ALTER TABLE `Type` "
" ADD INDEX `idx_type_name` (`name`); "
}
secondary_indices {
query: " ALTER TABLE `Execution` "
" ADD INDEX `idx_execution_create_time_since_epoch` "
" (`create_time_since_epoch`), "
" ADD INDEX `idx_execution_last_update_time_since_epoch` "
" (`last_update_time_since_epoch`); "
}
secondary_indices {
query: " ALTER TABLE `Context` "
" ADD INDEX `idx_context_create_time_since_epoch` "
" (`create_time_since_epoch`), "
" ADD INDEX `idx_context_last_update_time_since_epoch` "
" (`last_update_time_since_epoch`); "
}
secondary_indices {
query: " ALTER TABLE `EventPath` "
" ADD INDEX `idx_eventpath_event_id`(`event_id`); "
}
secondary_indices {
query: " ALTER TABLE `ArtifactProperty` "
" ADD INDEX `idx_artifact_property_int`( "
" `name`, `is_custom_property`, `int_value`); "
}
secondary_indices {
query: " ALTER TABLE `ArtifactProperty` "
" ADD INDEX `idx_artifact_property_double`( "
" `name`, `is_custom_property`, `double_value`); "
}
secondary_indices {
query: " ALTER TABLE `ArtifactProperty` "
" ADD INDEX `idx_artifact_property_string`( "
" `name`, `is_custom_property`, `string_value`(255)); "
}
secondary_indices {
query: " ALTER TABLE `ExecutionProperty` "
" ADD INDEX `idx_execution_property_int`( "
" `name`, `is_custom_property`, `int_value`); "
}
secondary_indices {
query: " ALTER TABLE `ExecutionProperty` "
" ADD INDEX `idx_execution_property_double`( "
" `name`, `is_custom_property`, `double_value`); "
}
secondary_indices {
query: " ALTER TABLE `ExecutionProperty` "
" ADD INDEX `idx_execution_property_string`( "
" `name`, `is_custom_property`, `string_value`(255)); "
}
secondary_indices {
query: " ALTER TABLE `ContextProperty` "
" ADD INDEX `idx_context_property_int`( "
" `name`, `is_custom_property`, `int_value`); "
}
secondary_indices {
query: " ALTER TABLE `ContextProperty` "
" ADD INDEX `idx_context_property_double`( "
" `name`, `is_custom_property`, `double_value`); "
}
secondary_indices {
query: " ALTER TABLE `ContextProperty` "
" ADD INDEX `idx_context_property_string`( "
" `name`, `is_custom_property`, `string_value`(255)); "
}
secondary_indices {
query: " ALTER TABLE `Type` "
" ADD INDEX `idx_type_external_id` (`external_id`);"
}
secondary_indices {
query: " ALTER TABLE `Artifact` "
" ADD INDEX `idx_artifact_external_id` (`external_id`);"
}
secondary_indices {
query: " ALTER TABLE `Execution` "
" ADD INDEX `idx_execution_external_id` (`external_id`);"
}
secondary_indices {
query: " ALTER TABLE `Context` "
" ADD INDEX `idx_context_external_id` (`external_id`);"
}
- execute
insert_schema_version_status = InsertSchemaVersion(library_version);using sql sentence:
"""
github repository:ml-metadata/ml_metadata/metadata_store/query_config_executor.h
"""
// Insert the schema version.
absl::Status InsertSchemaVersion(int64_t schema_version) final {
return ExecuteQuery(query_config_.insert_schema_version(),
{Bind(schema_version)});
}
int64_t GetLibraryVersion() final {
CHECK_GT(query_config_.schema_version(), 0);
return query_config_.schema_version();
}
"""
github-repository:ml-metadata/ml_metadata/util/metadata_source_query_config.cc
"""
schema_version: 10
insert_schema_version {
query: " INSERT INTO `MLMDEnv`(`schema_version`) VALUES($0); "
parameter_num: 1
}
