dremio mysql arp 扩展

2021年11月23日 阅读数:2
这篇文章主要向大家介绍dremio mysql arp 扩展,主要内容包括基础应用、实用技巧、原理机制等方面,希望对大家有所帮助。

开发此arp 扩展的目的是加强dremio 对于类mysql 数据库的兼容(好比oceanbase。。。doris 暂时还没测试)
由于默认官方的是基于maridb的(对于类mysql db 兼容有点问题)java

参考代码

核心是arp conf 调整了支持使用mysql jdbc,默认使用了maridb 的datasource 存在一些兼容问题mysql

 
package com.dremio.exec.store.jdbc.conf;
 
import com.dremio.exec.catalog.conf.*;
import com.dremio.exec.store.jdbc.CloseableDataSource;
import com.dremio.exec.store.jdbc.DataSources;
import com.dremio.exec.store.jdbc.JdbcPluginConfig;
import com.dremio.exec.store.jdbc.JdbcPluginConfig.Builder;
import com.dremio.exec.store.jdbc.dialect.JdbcDremioSqlDialect;
import com.dremio.exec.store.jdbc.dialect.MyMySQLDialect;
import com.dremio.options.OptionManager;
import com.dremio.security.CredentialsService;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import io.protostuff.Tag;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
 
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotBlank;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;
 
@SourceType(
        value = "MYMYSQL",
        label = "MYMYSQL",
        uiConfig = "my-mysql-layout.json",
        externalQuerySupported = true
)
public class MyMySQLConf extends AbstractArpConf<MyMySQLConf> {
    private static final Logger logger = LoggerFactory.getLogger(MyMySQLConf.class);
    private static final String ARP_FILENAME = "arp/implementation/my-mysql-arp.yaml";
    private static final MyMySQLDialect MYSQL_ARP_DIALECT = AbstractArpConf.loadArpFile(ARP_FILENAME, MyMySQLDialect::new);
    private static final String MYSQLDRIVER = "com.mysql.jdbc.Driver";
    @NotBlank
    @Tag(1)
    @DisplayMetadata(
            label = "Host"
    )
    public String hostname;
    @NotBlank
    @Tag(2)
    @Min(1L)
    @Max(65535L)
    @DisplayMetadata(
            label = "Port"
    )
    public String port = "3306";
    @Tag(4)
    public String username;
    @Tag(5)
    @Secret
    public String password;
    @Tag(6)
    public AuthenticationType authenticationType;
    @Tag(7)
    @DisplayMetadata(
            label = "Record fetch size"
    )
    @NotMetadataImpacting
    public int fetchSize = 200;
    @Tag(8)
    @DisplayMetadata(
            label = "Net write timeout (in seconds)"
    )
    @NotMetadataImpacting
    public int netWriteTimeout = 60;
    @Tag(9)
    @DisplayMetadata(
            label = "Enable legacy dialect"
    )
    @JsonIgnore
    public boolean useLegacyDialect = false;
    @Tag(10)
    @NotMetadataImpacting
    @JsonIgnore
    public boolean enableExternalQuery = false;
    @Tag(11)
    public List<Property> propertyList;
    @Tag(12)
    @DisplayMetadata(
            label = "Maximum idle connections"
    )
    @NotMetadataImpacting
    public int maxIdleConns = 8;
    @Tag(13)
    @DisplayMetadata(
            label = "Connection idle time (s)"
    )
    @NotMetadataImpacting
    public int idleTimeSec = 60;
    @Tag(14)
    @DisplayMetadata(
            label = "Query timeout (s)"
    )
    @NotMetadataImpacting
    public int queryTimeoutSec = 0;
 
    @VisibleForTesting
    CloseableDataSource newDataSource()  {
        Properties properties = new Properties();
        properties.put("useJDBCCompliantTimezoneShift", "true");
        properties.put("sessionVariables", String.format("net_write_timeout=%d", this.netWriteTimeout));
        // 此处使用了newGenericConnectionPoolDataSource没有使用sharedatasource
        return DataSources.newGenericConnectionPoolDataSource(MYSQLDRIVER, this.toJdbcConnectionString(), this.username, this.password, properties, DataSources.CommitMode.FORCE_MANUAL_COMMIT_MODE, this.maxIdleConns, (long)this.idleTimeSec);
    }
 
    @VisibleForTesting
    String toJdbcConnectionString() {
        String hostname = (String)Preconditions.checkNotNull(this.hostname, "missing hostname");
        String portAsString = (String)Preconditions.checkNotNull(this.port, "missing port");
        int port = Integer.parseInt(portAsString);
        String url = String.format("jdbc:mysql://%s:%d", hostname, port);
        logger.info("url:{}",url);
        System.out.println("url"+url);
        String connectUrl =  null != this.propertyList && !this.propertyList.isEmpty() ? url + (String)this.propertyList.stream().map((p) -> {
            return p.name + "=" + p.value;
        }).collect(Collectors.joining("&", "?", "")) : url;
        System.out.println("connectUrl"+url);
        return  connectUrl;
 
    }
 
    @Override
    public JdbcDremioSqlDialect getDialect() {
        return this.MYSQL_ARP_DIALECT;
    }
 
    @Override
    public JdbcPluginConfig buildPluginConfig(Builder builder, CredentialsService credentialsService, OptionManager optionManager) {
      return builder.withDialect(this.getDialect()).withDatasourceFactory(this::newDataSource).withShowOnlyConnDatabase(false).withFetchSize(this.fetchSize).withQueryTimeout(this.queryTimeoutSec).build();
    }
}
 

说明,具体代码能够参考githubgit

使用

构建,放jar 文件到dremio 的jar 目录,同时选择合适的mysql 驱动放到jars/3rdparty 推荐使用 mysql-connector-java-5.1.47 以及以上的
好处是能够支持oceanbase 。。。。
github

 

 


 

 

 

 

说明

相关构建好的jar 我也已经push github 了,能够直接使用sql

参考资料

https://github.com/rongfengliang/dremio-mysql-arp
https://open.oceanbase.com/docs/community/oceanbase-database/V3.1.0/java-database-connection-driver-jdbc-2
https://github.com/rongfengliang/dremio-oceanbase-learning数据库