cube.js s3 FileRepository 新功能

为了更好的集成cube.js 多租户的能力，进行了一些调整，支持基于配置参数的s3 schema 访问处理
同时加强了数据桶过滤的支持

核心修改的代码

主要是关于构造函数以及minio链接部分

 
constructor(config) {
        const { ...restConfig } = config || {};
        this.config = {
            endPoint: process.env.CUBEJS_S3_ENDPOINT,
            port: parseInt(process.env.CUBEJS_S3_PORT),
            useSSL: process.env.CUBEJS_S3_SSL == "true" ? true : false,
            accessKey: process.env.CUBEJS_S3_ACCESSKEY,
            secretKey: process.env.CUBEJS_s3_SECRETKEY,
            bucket:process.env.CUBEJS_S3_BUCKET,
            objectPrefix:"", // s3 object search  prefix default set ""
            ...restConfig,
        };
        this.minioClient = new Minio.Client({
            endPoint: this.config.endPoint,
            port: this.config.port,
            useSSL: this.config.useSSL,
            accessKey: this.config.accessKey,
            secretKey: this.config.secretKey
        });
    }
    async dataSchemaFiles(includeDependencies = true) {
        var self = this
        var bucket = self.config.bucket
        var objectPrefix = self.config.objectPrefix
        var Files = await streamToPromise(self.minioClient.listObjectsV2(bucket, objectPrefix, true))
        var fileContents = []
        for (const file of Files) {
            try {
                const fileBuffer = await streamToPromise(await self.minioClient.getObject(bucket, file.name))
                let fileItemContent = fileBuffer.toString('utf-8');
                fileContents.push({ fileName: file.name, content: fileItemContent })
            }
            catch (e) {
                console.log(e)
            }
        }
        if (includeDependencies) {
            fileContents = fileContents.concat(await this.readModules());
        }
        return fileContents;
    }

参考使用

新的调整支持基于环境变量以及配置参数的模式

  • env 模式
    .env 文件
CUBEJS_S3_ENDPOINT=localhost
CUBEJS_S3_PORT=9000
CUBEJS_S3_SSL=false
CUBEJS_S3_ACCESSKEY=minio
CUBEJS_s3_SECRETKEY=minio123
CUBEJS_S3_BUCKET=demo

cube.js 配置
使用了一个自己开发的dremio driver

 
// Cube.js configuration options: https://cube.dev/docs/config
const {DremioDriver,DremioQuery} = require("@dalongrong/mydremio-driver")
const MyS3FileRepository = require("@dalongrong/cube-s3repository")
 
module.exports = {
    dialectFactory: (dataSource) => {
        // need config  datasource  for multitenant env
        return DremioQuery
    },
    queryTransformer: (query, { securityContext }) => {
        console.log(securityContext)
        return query;
      },
    dbType: ({ dataSource } = {}) => {
        return "mydremio"
    },
    driverFactory: ({ dataSource } = {}) => {
        return new DremioDriver({})
    },
    repositoryFactory: ({securityContext})=>{
        return new MyS3FileRepository.S3FileRepository({
            objectPrefix:"appdemo/"
        })
    }
};

参考资料

https://github.com/rongfengliang/cube.js-s3-filerepository
https://www.npmjs.com/package/@dalongrong/cube-s3repository

posted on 2021-04-08 20:23  荣锋亮  阅读(204)  评论(0)  编辑  收藏  举报

导航