airbyte 整體架構

內容來自官方文檔，基於此圖我們可以了解airbyte 的整體運行&&方便問題解決&&環境部署

參考圖

 

 

 

簡單說明

• ui 進行配置，以及通過界面查看同步&&日志
• config store 關於認證&&同步次信息存儲
• scheduler 存儲關於調度執行情況
• config api 方便ui 進行連接配置
• scheduler api ui 進行調度job 配置
• scheduler 進行數據任務的調度編排以及狀態追蹤
• worker 具體數據從source 到sink 的操作
• tep store 臨時存儲（需要數據寫入磁盤的場景）

官方參考部署

注意目前關於ha以及集群的部署官方缺少說明，后邊可以通過源碼分析下

• docker-compose

version: "3.7"

services:

  # hook in case we need to add init behavior

  # every root service (no depends_on) should depend on init

  init:

    image: airbyte/init:${VERSION}

    container_name: init

    command: /bin/sh -c "./scripts/create_mount_directories.sh /local_parent ${HACK_LOCAL_ROOT_PARENT} ${LOCAL_ROOT}"

    environment:

      - LOCAL_ROOT=${LOCAL_ROOT}

      - HACK_LOCAL_ROOT_PARENT=${HACK_LOCAL_ROOT_PARENT}

    volumes:

      - ${HACK_LOCAL_ROOT_PARENT}:/local_parent

  db:

    image: airbyte/db:${VERSION}

    container_name: airbyte-db

    environment:

      - POSTGRES_USER=${DATABASE_USER}

      - POSTGRES_PASSWORD=${DATABASE_PASSWORD}

      - POSTGRES_DB=db-${DATABASE_DB}

    volumes:

      - db:/var/lib/postgresql/data

  seed:

    image: airbyte/seed:${VERSION}

    container_name: airbyte-data-seed

    # Pre-populate the volume if it is empty.

    # See: https://docs.docker.com/storage/volumes/#populate-a-volume-using-a-container

    volumes:

      - data:/app/seed

  scheduler:

    image: airbyte/scheduler:${VERSION}

    container_name: airbyte-scheduler

    environment:

      - DATABASE_USER=${DATABASE_USER}

      - DATABASE_PASSWORD=${DATABASE_PASSWORD}

      - DATABASE_URL=jdbc:postgresql://db:5432/${DATABASE_DB}

      - WAIT_BEFORE_HOSTS=5

      - WAIT_HOSTS_TIMEOUT=45

      - WAIT_HOSTS=db:5432

      - WORKSPACE_ROOT=${WORKSPACE_ROOT}

      - WORKSPACE_DOCKER_MOUNT=${WORKSPACE_DOCKER_MOUNT}

      - LOCAL_DOCKER_MOUNT=${LOCAL_DOCKER_MOUNT}

      - CONFIG_ROOT=${CONFIG_ROOT}

      - TRACKING_STRATEGY=${TRACKING_STRATEGY}

      - AIRBYTE_VERSION=${VERSION}

      - AIRBYTE_ROLE=${AIRBYTE_ROLE:-}

    volumes:

      - /var/run/docker.sock:/var/run/docker.sock

      - workspace:${WORKSPACE_ROOT}

      - ${LOCAL_ROOT}:${LOCAL_ROOT}

      - data:${CONFIG_ROOT}

  server:

    image: airbyte/server:${VERSION}

    container_name: airbyte-server

    environment:

      - DATABASE_USER=${DATABASE_USER}

      - DATABASE_PASSWORD=${DATABASE_PASSWORD}

      - DATABASE_URL=jdbc:postgresql://db:5432/${DATABASE_DB}

      - WAIT_BEFORE_HOSTS=5

      - WAIT_HOSTS_TIMEOUT=45

      - WAIT_HOSTS=db:5432

      - CONFIG_ROOT=${CONFIG_ROOT}

      - TRACKING_STRATEGY=${TRACKING_STRATEGY}

      - AIRBYTE_VERSION=${VERSION}

      - AIRBYTE_ROLE=${AIRBYTE_ROLE:-}

      - WORKSPACE_ROOT=${WORKSPACE_ROOT}

    ports:

      - 8001:8001

    volumes:

      - workspace:${WORKSPACE_ROOT}

      - data:${CONFIG_ROOT}

  webapp:

    image: airbyte/webapp:${VERSION}

    container_name: airbyte-webapp

    ports:

      - 8000:80

    environment:

      - AIRBYTE_ROLE=${AIRBYTE_ROLE:-}

      - AIRBYTE_VERSION=${VERSION}

      - API_URL=${API_URL}

      - IS_DEMO=${IS_DEMO:-}

      - PAPERCUPS_STORYTIME=${PAPERCUPS_STORYTIME:-}

      - TRACKING_STRATEGY=${TRACKING_STRATEGY}

volumes:

  workspace:

    name: ${WORKSPACE_DOCKER_MOUNT}

  data:

    name: ${DATA_DOCKER_MOUNT}

  db:

    name: ${DB_DOCKER_MOUNT}

參考資料

https://docs.airbyte.io/architecture/high-level-view