airbyte 整体架构
内容来自官方文档，基于此图我们可以了解airbyte 的整体运行&&方便问题解决&&环境部署
参考图
简单说明
- ui 进行配置，以及通过界面查看同步&&日志
- config store 关于认证&&同步次信息存储
- scheduler 存储关于调度执行情况
- config api 方便ui 进行连接配置
- scheduler api ui 进行调度job 配置
- scheduler 进行数据任务的调度编排以及状态追踪
- worker 具体数据从source 到sink 的操作
- tep store 临时存储（需要数据写入磁盘的场景）
官方参考部署
注意目前关于ha以及集群的部署官方缺少说明，后边可以通过源码分析下
- docker-compose
version: "3.7"
services:
# hook in case we need to add init behavior
# every root service (no depends_on) should depend on init
init:
image: airbyte/init:${VERSION}
container_name: init
command: /bin/sh -c "./scripts/create_mount_directories.sh /local_parent ${HACK_LOCAL_ROOT_PARENT} ${LOCAL_ROOT}"
environment:
- LOCAL_ROOT=${LOCAL_ROOT}
- HACK_LOCAL_ROOT_PARENT=${HACK_LOCAL_ROOT_PARENT}
volumes:
- ${HACK_LOCAL_ROOT_PARENT}:/local_parent
db:
image: airbyte/db:${VERSION}
container_name: airbyte-db
environment:
- POSTGRES_USER=${DATABASE_USER}
- POSTGRES_PASSWORD=${DATABASE_PASSWORD}
- POSTGRES_DB=db-${DATABASE_DB}
volumes:
- db:/var/lib/postgresql/data
seed:
image: airbyte/seed:${VERSION}
container_name: airbyte-data-seed
# Pre-populate the volume if it is empty.
# See: https://docs.docker.com/storage/volumes/#populate-a-volume-using-a-container
volumes:
- data:/app/seed
scheduler:
image: airbyte/scheduler:${VERSION}
container_name: airbyte-scheduler
environment:
- DATABASE_USER=${DATABASE_USER}
- DATABASE_PASSWORD=${DATABASE_PASSWORD}
- DATABASE_URL=jdbc:postgresql://db:5432/${DATABASE_DB}
- WAIT_BEFORE_HOSTS=5
- WAIT_HOSTS_TIMEOUT=45
- WAIT_HOSTS=db:5432
- WORKSPACE_ROOT=${WORKSPACE_ROOT}
- WORKSPACE_DOCKER_MOUNT=${WORKSPACE_DOCKER_MOUNT}
- LOCAL_DOCKER_MOUNT=${LOCAL_DOCKER_MOUNT}
- CONFIG_ROOT=${CONFIG_ROOT}
- TRACKING_STRATEGY=${TRACKING_STRATEGY}
- AIRBYTE_VERSION=${VERSION}
- AIRBYTE_ROLE=${AIRBYTE_ROLE:-}
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- workspace:${WORKSPACE_ROOT}
- ${LOCAL_ROOT}:${LOCAL_ROOT}
- data:${CONFIG_ROOT}
server:
image: airbyte/server:${VERSION}
container_name: airbyte-server
environment:
- DATABASE_USER=${DATABASE_USER}
- DATABASE_PASSWORD=${DATABASE_PASSWORD}
- DATABASE_URL=jdbc:postgresql://db:5432/${DATABASE_DB}
- WAIT_BEFORE_HOSTS=5
- WAIT_HOSTS_TIMEOUT=45
- WAIT_HOSTS=db:5432
- CONFIG_ROOT=${CONFIG_ROOT}
- TRACKING_STRATEGY=${TRACKING_STRATEGY}
- AIRBYTE_VERSION=${VERSION}
- AIRBYTE_ROLE=${AIRBYTE_ROLE:-}
- WORKSPACE_ROOT=${WORKSPACE_ROOT}
ports:
- 8001:8001
volumes:
- workspace:${WORKSPACE_ROOT}
- data:${CONFIG_ROOT}
webapp:
image: airbyte/webapp:${VERSION}
container_name: airbyte-webapp
ports:
- 8000:80
environment:
- AIRBYTE_ROLE=${AIRBYTE_ROLE:-}
- AIRBYTE_VERSION=${VERSION}
- API_URL=${API_URL}
- IS_DEMO=${IS_DEMO:-}
- PAPERCUPS_STORYTIME=${PAPERCUPS_STORYTIME:-}
- TRACKING_STRATEGY=${TRACKING_STRATEGY}
volumes:
workspace:
name: ${WORKSPACE_DOCKER_MOUNT}
data:
name: ${DATA_DOCKER_MOUNT}
db:
name: ${DB_DOCKER_MOUNT}
