大數據_調度平台_docker-compose部署DolphinScheduler

docker-compose 使用 dolphinscheduler

###修改配置文件：docker-compose.yml 
#~/apache-dolphinscheduler-2.0.2-src/docker/docker-swarm
## check
## config.env.sh
## docker-compose.back
## docker-compose.yml
## docker-stack.yml

修改配置文件：docker-compose.yml 解讀

version: "3.1"

services:
  
   # PostgreSQL
  dolphinscheduler-postgresql:
    image: bitnami/postgresql:11.11.0
    environment:
	# 設置時區
      TZ: Asia/Shanghai
	   # PostgreSQL 相關的配置  用戶名、密碼 名為dolphinscheduler 的數據庫
      POSTGRESQL_USERNAME: root
      POSTGRESQL_PASSWORD: root
      POSTGRESQL_DATABASE: dolphinscheduler
	 # 數據卷
    volumes:
    - dolphinscheduler-postgresql:/bitnami/postgresql
	# 重啟策：在容器退出時總是重啟容器
    restart: unless-stopped
	# 配置網絡
    networks:
    - dolphinscheduler

  dolphinscheduler-zookeeper:
    image: bitnami/zookeeper:3.6.2
    environment:
      TZ: Asia/Shanghai
	  # Zookeeper 相關配置
      ALLOW_ANONYMOUS_LOGIN: "yes"
      ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
    volumes:
    - dolphinscheduler-zookeeper:/bitnami/zookeeper
    restart: unless-stopped
    networks:
    - dolphinscheduler

  # 自身的服務模塊包括：api, alert, master, worker (有一個 logger 服務，運行在 worker 中)
  dolphinscheduler-api:
    image: apache/dolphinscheduler:2.0.2
    command: api-server
    ports:
    - 12345:12345
    environment:
      TZ: Asia/Shanghai
	 # 引入外部環境變量 config.env.sh 定義了用到的配置，通過 env_file 的方式傳入容器，它會覆蓋容器內的默認配置
    env_file: config.env.sh
     # 調用容器內的 checkpoint.sh，並傳入服務名稱，檢查該 Java 進程是否存在。
     # 兩次健康檢查的間隔 30s，超時時間為 5s，如果超過這個時間，本次健康檢查就被視為失敗，
     # retries 重試次數設置為 3，當連續失敗指定次數后，則將容器狀態視為 unhealthy
    healthcheck:
      test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
      interval: 30s
      timeout: 5s
      retries: 3
	# 依賴 PostgreSQL 和 Zookeeper
    depends_on:
    - dolphinscheduler-postgresql
    - dolphinscheduler-zookeeper
    volumes:
    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    - dolphinscheduler-shared-local:/opt/soft
    - dolphinscheduler-resource-local:/dolphinscheduler
    restart: unless-stopped
    networks:
    - dolphinscheduler

  dolphinscheduler-alert:
    image: apache/dolphinscheduler:2.0.2
    command: alert-server
    environment:
      TZ: Asia/Shanghai
    env_file: config.env.sh
    healthcheck:
      test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
      interval: 30s
      timeout: 5s
      retries: 3
    depends_on:
    - dolphinscheduler-postgresql
    volumes:
    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    restart: unless-stopped
    networks:
    - dolphinscheduler

  dolphinscheduler-master:
    image: apache/dolphinscheduler:2.0.2
    command: master-server
    environment:
      TZ: Asia/Shanghai
    env_file: config.env.sh
    healthcheck:
      test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
      interval: 30s
      timeout: 5s
      retries: 3
    depends_on:
    - dolphinscheduler-postgresql
    - dolphinscheduler-zookeeper
    volumes:
    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    - dolphinscheduler-shared-local:/opt/soft
    restart: unless-stopped
    networks:
    - dolphinscheduler

  dolphinscheduler-worker:
    image: apache/dolphinscheduler:2.0.2
    command: worker-server
    environment:
      TZ: Asia/Shanghai
    env_file: config.env.sh
    healthcheck:
      test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
      interval: 30s
      timeout: 5s
      retries: 3
    depends_on:
    - dolphinscheduler-postgresql
    - dolphinscheduler-zookeeper
    volumes:
    - dolphinscheduler-worker-data:/tmp/dolphinscheduler
    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    - dolphinscheduler-shared-local:/opt/soft
    - dolphinscheduler-resource-local:/dolphinscheduler
    restart: unless-stopped
    networks:
    - dolphinscheduler
		
  dolphinscheduler-worker2:
    image: apache/dolphinscheduler:2.0.2
    command: worker-server
    environment:
      TZ: Asia/Shanghai
    env_file: config.env.sh
    healthcheck:
      test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
      interval: 30s
      timeout: 5s
      retries: 3
    depends_on:
    - dolphinscheduler-postgresql
    - dolphinscheduler-zookeeper
    volumes:
    - dolphinscheduler-worker-data:/tmp/dolphinscheduler
    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    - dolphinscheduler-shared-local:/opt/soft
    - dolphinscheduler-resource-local:/dolphinscheduler
    restart: unless-stopped
    networks:
    - dolphinscheduler	
	
# 聲明使用到的網絡
networks:
  dolphinscheduler:
    driver: bridge
	
# 聲明使用到的數據卷
volumes:
  dolphinscheduler-postgresql:
  dolphinscheduler-zookeeper:
  dolphinscheduler-worker-data:
  dolphinscheduler-logs:
  dolphinscheduler-shared-local:
  dolphinscheduler-resource-local:

構建生產環境中使用的

# 創建一個新的 Dockerfile，用於添加 mysql 驅動包:
#其中 mysql-connector-java-8.0.17.jar 是在soft 的目錄下
cd ~/soft
vim Dockerfile

FROM apache/dolphinscheduler:2.0.3
COPY mysql-connector-java-8.0.17.jar /opt/dolphinscheduler/lib
COPY source.list /tmp/source.list
CMD  cat /tmp/source.list > /etc/apt/sources.list  
RUN apt-get update && \
    apt-get install -y --no-install-recommends ssh && \
    apt-get install -y --no-install-recommends python3 && \
    rm -rf /var/lib/apt/lists/*



# 構建一個包含 Oracle 驅動包的新鏡像:
docker build -t apache/dolphinscheduler:sql-driver .


將 docker-compose.yml 文件中的所有 image 字段修改為 apache/dolphinscheduler:mysql-driver

暫時的方法

FROM apache/dolphinscheduler:2.0.3
COPY mysql-connector-java-8.0.17.jar /opt/dolphinscheduler/lib
COPY source.list /tmp/source.list
COPY hosts /tmp/hosts
COPY resolv.conf /tmp/resolv.conf
CMD cat /tmp/hosts >> /etc/hosts
RUN cat /tmp/source.list >/etc/apt/sources.list && \
    cat /tmp/resolv.conf >/etc/resolv.conf && \
    cat /tmp/hosts >> /etc/hosts && \
    apt-get update  --allow-insecure-repositories && \
    apt-get install -y --no-install-recommends python  --allow-unauthenticated && \
    apt-get install -y --no-install-recommends python-pip --allow-unauthenticated && \
    rm -rf /var/lib/apt/lists/*

Dockerfile

 /etc/hosts
 /etc/apt/sources.list
 /etc/resolv.conf
 
 /etc/default/docker
 /etc/docker/daemon.json

 Reading package lists... Done
 E: Could not get lock /var/lib/apt/lists/lock - open (11: Resource temporarily unavailable)
 E: Unable to lock directory /var/lib/apt/lists/
   rm -rf /var/lib/apt/lists/*
   
 “Repository does not have a release file”是因為添加的儲存庫不適用於當前Ubuntu版本
   

1.Docker容器修改  hosts 
     /etc/hosts, /etc/resolv.conf和/etc/hostname 容器中的這三個文件不存在於鏡像,
           而是存在於於/var/lib/docker/containers/，在啟動容器的時候，通過mount的形式將這些文件掛載到容器內部。
     因此，如果在容器中修改這些文件的話，修改部分不會存在於容器的top layer，而是直接寫入這三個物理文件中。
2.為什么重啟后修改內容不存在了？原因是：每次Docker在啟動容器的時候，通過重新構建新的/etc/hosts文件，這又是為什么呢？
     原因是：容器重啟，IP地址為改變，hosts文件中原來的IP地址無效，因此理應修改hosts文件，否則會產生臟數據。
3.有沒有什么好的解決方法? 
 可以通過docker run命令的–add-host參數來為容器添加host與ip的映射關系。
 

Dockerfile文件
   COPY ADD
      Dockerfile中的COPY指令和ADD指令都可以將主機上的資源復制或加入到容器鏡像中，都是在構建鏡像的過程中完成的。
      COPY指令和ADD指令的唯一區別在於是否支持從遠程URL獲取資源。
      COPY指令只能從執行docker build所在的主機上讀取資源並復制到鏡像中。
      而ADD指令還支持通過URL從遠程服務器讀取資源並復制到鏡像中
   RUN CMD  ENTRYPOINT
      CMD 在docker run 時運行
      CMD 指令指定的程序可被 docker run 命令行參數中指定要運行的程序所覆蓋。
      如果 Dockerfile 中如果存在多個 CMD 指令，僅最后一個生效
   ENV  ARG
    FROM 
    VOLUME  EXPOSE  WORKDIR USER  LABEL

參考

 Apache Dolphin Scheduler - Docker Compose 詳解  https://www.cnblogs.com/aaronlinv/p/15309275.html 
  Modify hosts file in dockerfile  https://serverfault.com/questions/777714/modify-hosts-file-in-dockerfile