dremio 系统内部插件
dremio 不少处理都是基于插件开发的，以前我也简单说明过，以下是结合启动日志列一个相对完整的内部存储插件名称
目前系统内部插件
目前包含了如下几个，可以通过启动日志看出来
- 系统存储插件
INFORMATION_SCHEMA
__jobResultsStore
__logs
__support
__system_iceberg_tables_storage
__datasetDownload
sys
$scratch
__home
__metadata
__accelerator
创建
有一些插件我以前是简单介绍过，具体是SystemStoragePluginInitializer 类处理的，比较特殊的是INFORMATION_SCHEMA 以及sys这几个存储插件
这个是由CatalogServiceImpl服务创建的
- 参考处理
public void start() throws Exception {
SabotContext context = this.sabotContext.get();
this.allocator = bufferAllocator.get().newChildAllocator("catalog-protocol", 0, Long.MAX_VALUE);
this.systemNamespace = context.getNamespaceService(SystemUser.SYSTEM_USERNAME);
this.sourceDataStore = kvStoreProvider.get().getStore(CatalogSourceDataCreator.class);
this.modifiableSchedulerService = modifiableSchedulerServiceProvider.get();
this.modifiableSchedulerService.start();
this.plugins = newPluginsManager();
plugins.start();
this.protocol =
new CatalogProtocol(allocator, new CatalogChangeListener(), config.getSabotConfig());
tunnelFactory = fabric.get().registerProtocol(protocol);
boolean isDistributedCoordinator =
config.isMasterlessEnabled() && roles.contains(Role.COORDINATOR);
if (roles.contains(Role.MASTER) || isDistributedCoordinator) {
final CountDownLatch wasRun = new CountDownLatch(1);
final String taskName =
optionManager.get().getOption(ExecConstants.CATALOG_SERVICE_LOCAL_TASK_LEADER_NAME);
final Cancellable task =
scheduler
.get()
.schedule(
Schedule.SingleShotBuilder.now()
.asClusteredSingleton(taskName)
.inLockStep()
.build(),
() -> {
try {
// 创建INFORMATION_SCHEMA 存储插件
if (createSourceIfMissing(
new SourceConfig()
.setConfig(new InfoSchemaConf().toBytesString())
.setName("INFORMATION_SCHEMA")
.setType("INFORMATION_SCHEMA")
.setMetadataPolicy(CatalogService.NEVER_REFRESH_POLICY))) {
logger.debug("Refreshing 'INFORMATION_SCHEMA' source");
try {
refreshSource(
new NamespaceKey("INFORMATION_SCHEMA"),
CatalogService.NEVER_REFRESH_POLICY,
UpdateType.FULL);
} catch (NamespaceException e) {
throw new RuntimeException(e);
}
}
if (optionManager.get().getOption(ExecConstants.ENABLE_SYSFLIGHT_SOURCE)) {
if (sysFlightTableConfProvider != null) {
logger.info("Creating SysFlight source plugin.");
// check if the sys plugin config type is matching as expected with the
// flight config type {SYSFLIGHT},
// if not, delete and recreate with flight config.
if (getPlugins().get(SYSTEM_TABLE_SOURCE_NAME) != null
&& !getPlugins()
.get(SYSTEM_TABLE_SOURCE_NAME)
.getConnectionConf()
.getType()
.equals(sysFlightTableConfProvider.get().get().getType())) {
deleteSource(SYSTEM_TABLE_SOURCE_NAME);
}
// 创建sys 系统存储插件
createSourceIfMissing(
new SourceConfig()
.setConnectionConf(sysFlightTableConfProvider.get().get())
.setName(SYSTEM_TABLE_SOURCE_NAME)
.setMetadataPolicy(CatalogService.DEFAULT_METADATA_POLICY));
}
} else {
// check if the sys plugin config type is matching as expected with the old
// system table config type {SYS},
// if not, delete and recreate with old system table config type.
if (getPlugins().get(SYSTEM_TABLE_SOURCE_NAME) != null
&& !getPlugins()
.get(SYSTEM_TABLE_SOURCE_NAME)
.getConnectionConf()
.getType()
.equals(sysTableConfProvider.get().get().getType())) {
deleteSource(SYSTEM_TABLE_SOURCE_NAME);
}
createSourceIfMissing(
new SourceConfig()
.setConnectionConf(sysTableConfProvider.get().get())
.setName(SYSTEM_TABLE_SOURCE_NAME)
.setMetadataPolicy(CatalogService.NEVER_REFRESH_POLICY));
}
logger.debug("Refreshing {} source", SYSTEM_TABLE_SOURCE_NAME);
try {
refreshSource(
new NamespaceKey(SYSTEM_TABLE_SOURCE_NAME),
CatalogService.NEVER_REFRESH_POLICY,
UpdateType.FULL);
} catch (NamespaceException e) {
throw new RuntimeException(e);
}
} finally {
wasRun.countDown();
}
});
if (task.isScheduled()) {
// wait till task is done only if the task is scheduled locally
logger.debug("Awaiting local lock step schedule completion for task {}", taskName);
wasRun.await();
logger.debug("Lock step schedule completed for task {}", taskName);
} else {
logger.debug("Lock step schedule for task {} completed remotely", taskName);
}
}
}
说明
以上实际是对于以前几个系统存储插件的一个扩展说明，sys 以及INFORMATION_SCHEMA 属于一个比较重要的东西，应该好好看看
参考资料
dac/backend/src/main/java/com/dremio/dac/daemon/SystemStoragePluginInitializer.java
dac/backend/src/main/java/com/dremio/dac/support/SupportService.java
sabot/kernel/src/main/java/com/dremio/exec/catalog/CatalogServiceImpl.java
sabot/kernel/src/main/java/com/dremio/exec/store/sys/SystemStoragePlugin.java
sabot/kernel/src/main/java/com/dremio/exec/store/ischema/InfoSchemaStoragePlugin.java
sabot/kernel/src/main/java/com/dremio/exec/store/metadatarefresh/MetadataRefreshExecConstants.java
sabot/kernel/src/main/java/com/dremio/exec/store/dfs/system/SystemIcebergTablesStoragePluginConfig.java
