一個簡單的測試,不算是ha (ha 后邊會介紹使用)
環境准備
- docker-compose 文件
version: "3"
services:
zookeeper:
image: zookeeper
ports:
- "2181:2181"
- "8080:8080"
dremio1:
image: dremio/dremio-oss:13.0
environment:
- DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dsaffron.default.charset=UTF-16LE -Dsaffron.default.nationalcharset=UTF-16LE -Dsaffron.default.collation.name=UTF-16LE\$en_US
volumes:
- "./dremio1.conf:/opt/dremio/conf/dremio.conf"
- "./datas/data:/opt/dremio/data"
ports:
- "9047:9047"
- "31010:31010"
dremio2:
image: dremio/dremio-oss:13.0
environment:
- DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dsaffron.default.charset=UTF-16LE -Dsaffron.default.nationalcharset=UTF-16LE -Dsaffron.default.collation.name=UTF-16LE\$en_US
volumes:
- "./dremio3.conf:/opt/dremio/conf/dremio.conf"
ports:
- "9048:9047"
- "31011:31010"
dremio3:
image: dremio/dremio-oss:13.0
environment:
- DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dsaffron.default.charset=UTF-16LE -Dsaffron.default.nationalcharset=UTF-16LE -Dsaffron.default.collation.name=UTF-16LE\$en_US
volumes:
- "./dremio3.conf:/opt/dremio/conf/dremio.conf"
ports:
- "9049:9047"
- "31012:31010"
pg:
image: postgres:12
environment:
- "POSTGRES_PASSWORD=dalong"
ports:
- "5432:5432"
mongo:
image: mongo
ports:
- "27017:27017"
- 簡單說明
一個參考架構(使用了一個master,使用了一個獨立的外部zk,使用了兩個executor)
核心配置說明
主要是services 片段的配置
dremio 對於不同類型還是比較明確的,我們基於配置就可以了
參考配置
master dremio1.conf
paths: {
# the local path for dremio to store data.
local: ${DREMIO_HOME}"/data"
# the distributed path Dremio data including job results, downloads, uploads, etc
#dist: "pdfs://"${paths.local}"/pdfs"
}
services: {
coordinator.enabled: true,
coordinator.master.enabled: true,
coordinator.master.embedded-zookeeper.enabled: false,
executor.enabled: false
}
zookeeper: "zookeeper:2181"
executor 1 dremio2.conf
paths: {
# the local path for dremio to store data.
local: ${DREMIO_HOME}"/data"
# the distributed path Dremio data including job results, downloads, uploads, etc
#dist: "pdfs://"${paths.local}"/pdfs"
}
services: {
coordinator.enabled: false,
coordinator.master.enabled: false,
coordinator.master.embedded-zookeeper.enabled: false,
executor.enabled: true
}
zookeeper: "zookeeper:2181"
executor 2 dremio3.conf
paths: {
# the local path for dremio to store data.
local: ${DREMIO_HOME}"/data"
# the distributed path Dremio data including job results, downloads, uploads, etc
#dist: "pdfs://"${paths.local}"/pdfs"
}
services: {
coordinator.enabled: false,
coordinator.master.enabled: false,
coordinator.master.embedded-zookeeper.enabled: false,
executor.enabled: true
}
zookeeper: "zookeeper:2181"
運行效果
- 啟動
docker-compose up -d
- 效果
- 一個參考執行效果
查看執行job
參考資料
https://docs.dremio.com/deployment/nodes-config.html
https://docs.dremio.com/deployment/high-availability-config.html