dremio 联邦数据源arp 扩展简单说明

dremio 联邦查询dremio 从代码上没太多高深的东西，实际上也是一个arp 扩展 ，同时官方做了不少优化

核心参考

主要是关于arp 扩展的

• conf 类

@SourceType(

   value = "DREMIOTODREMIO",

   label = "Dremio-to-Dremio (preview)",

   uiConfig = "dremio-to-dremio-layout.json",

   externalQuerySupported = true

)

public class DremioToDremioConf extends AbstractArpConf<DremioToDremioConf> {

    // dremio-to-dremio-arp.yaml 类型处理的，很不错值得看看

   private static final String ARP_FILENAME = "arp/implementation/dremio-to-dremio-arp.yaml";

   private static final DremioToDremioDialect DREMIO_TO_DREMIO_ARP_DIALECT = (DremioToDremioDialect)AbstractArpConf.loadArpFile("arp/implementation/dremio-to-dremio-arp.yaml", DremioToDremioDialect::new);

   @Tag(1)

   @DisplayMetadata(

      label = "Endpoint Type"

   )

   public DremioToDremioConf.HostType hostType;

   @NotBlank

   @Tag(2)

   @DisplayMetadata(

      label = "Host"

   )

   public String hostname;

   @NotBlank

   @Tag(3)

   @Min(1L)

   @Max(65535L)

   @DisplayMetadata(

      label = "Port"

   )

   public String port;

   @Tag(4)

   @DisplayMetadata(

      label = "Username"

   )

   public String username;

   @Tag(5)

   @DisplayMetadata(

      label = "Password"

   )

   @Secret

   public String password;

   @Tag(6)

   @DisplayMetadata(

      label = "Use SSL"

   )

   public boolean useSsl;

   @Tag(7)

   @DisplayMetadata(

      label = "User Impersonation"

   )

   public boolean userImpersonation;

   @Tag(8)

   @DisplayMetadata(

      label = "Maximum idle connections"

   )

   @NotMetadataImpacting

   public int maxIdleConns;

   @Tag(9)

   @DisplayMetadata(

      label = "Connection idle time (s)"

   )

   @NotMetadataImpacting

   public int idleTimeSec;

   @Tag(10)

   @DisplayMetadata(

      label = "Query timeout (s)"

   )

   @NotMetadataImpacting

   public int queryTimeoutSec;

 

   public DremioToDremioConf() {

      this.hostType = DremioToDremioConf.HostType.DIRECT;

      this.port = "31010";

      this.maxIdleConns = 8;

      this.idleTimeSec = 60;

      this.queryTimeoutSec = 0;

   }

// 官方基于commons-dbcp2 开发的数据源，具体参考源码

   private CloseableDataSource newDataSource() {

      DremioToDremioPoolDataSource dataSource = new DremioToDremioPoolDataSource(this);

      return CloseableDataSource.wrap(dataSource);

   }

   // jdbc 配置，包含了方言以及数据源创建的

   public JdbcPluginConfig buildPluginConfig(Builder configBuilder, CredentialsService credentialsService, OptionManager optionManager) {

      return configBuilder.withDialect(this.getDialect()).withDatasourceFactory(this::newDataSource).withShowOnlyConnDatabase(true).withQueryTimeout(this.queryTimeoutSec).build();

   }

 

   public DremioToDremioDialect getDialect() {

      return DREMIO_TO_DREMIO_ARP_DIALECT;

   }

 

   @VisibleForTesting

// 官方标准方言处理

   public static DremioToDremioDialect getDialectSingleton() {

      return DREMIO_TO_DREMIO_ARP_DIALECT;

   }

 

   public static enum HostType {

      @Tag(0)

      DIRECT,

      @Tag(1)

      ZOOKEEPER;

   }

}

• DremioToDremioDialect 方言

public class DremioToDremioDialect extends ArpDialect {

   public DremioToDremioDialect(ArpYaml yaml) {

      super(yaml);

   }

 

   public DremioToDremioRelToSqlConverter getConverter() {

     // 关系算子转sql 的

      return new DremioToDremioRelToSqlConverter(this);

   }

}

• DremioToDremioRelToSqlConverter 扩展

public class DremioToDremioRelToSqlConverter extends JdbcDremioRelToSqlConverter {

   public DremioToDremioRelToSqlConverter(JdbcDremioSqlDialect dremioDialect) {

      super(dremioDialect);

   }

 

   protected JdbcDremioRelToSqlConverter getJdbcDremioRelToSqlConverter() {

      return this;

   }

   // 目前看主要是对于sql 做了一些处理

   public void addSelect(List<SqlNode> selectList, SqlNode node, RelDataType rowType) {

      String name = (String)rowType.getFieldNames().get(selectList.size());

      String alias = SqlValidatorUtil.getAlias((SqlNode)node, -1);

      if (alias == null || !alias.equals(name)) {

         name = AliasSanitizer.sanitizeAlias(name);

         node = SqlStdOperatorTable.AS.createCall(POS, new SqlNode[]{(SqlNode)node, new SqlIdentifier(name, POS)});

      }

 

      selectList.add(node);

   }

}

• arp 类型映射
  

  　参考图，具体查看源码，因为不支持catalog 所以配置了catalog 为false
  

   

   

参考资料

https://docs.dremio.com/software/data-sources/dremio/