apache pinot 学习

apache pinot 是一个很不错的olap 服务，功能强大，低延迟，包含了实时以及离线处理

支持的特性

非常快（支持离线以及实时处理）
水平扩展（基于了apache helix 框架）
插件化索引，包含sorted index，bitmap index,invertes index, startree index , bloom filter,range index, 文本检索，json index，geo index
近实时的数据进入（kafka，pulsar）
类sql 查询
基于prestodb，trino，进行多表查询（主要是当前不支持，只能基于此处理，当然dremio也是一种选择，但是肯定没有官方推荐的好）
异常分析，只能报警，主要基于了thirdeye

参考组件

组件结构

查询处理流程

docker-compose 运行

组件分离部署

version: '3.7'

services:

  zookeeper:

    image: zookeeper:3.5.6

    hostname: zookeeper

    container_name: manual-zookeeper

    ports:

      - "2181:2181"

    environment:

      ZOOKEEPER_CLIENT_PORT: 2181

      ZOOKEEPER_TICK_TIME: 2000

  pinot-controller:

    image: apachepinot/pinot:0.9.3

    command: "StartController -zkAddress manual-zookeeper:2181"

    container_name: "manual-pinot-controller"

    restart: unless-stopped

    ports:

      - "9000:9000"

    environment:

      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms1G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-controller.log"

    depends_on:

      - zookeeper

  pinot-broker:

    image: apachepinot/pinot:0.9.3

    command: "StartBroker -zkAddress manual-zookeeper:2181"

    restart: unless-stopped

    container_name: "manual-pinot-broker"

    ports:

      - "8099:8099"

    environment:

      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-broker.log"

    depends_on:

      - pinot-controller

  pinot-server:

    image: apachepinot/pinot:0.9.3

    command: "StartServer -zkAddress manual-zookeeper:2181"

    restart: unless-stopped

    container_name: "manual-pinot-server" 

    environment:

      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx16G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-server.log"

    depends_on:

      - pinot-broker