superset 0.36 搭建開發編譯環境

superset

准備環境

1. python3.7.6 環境准備
2. superset安裝包准備

系統環境：centos 7.3

#安裝編譯環境依賴
yum install gcc gcc-c++  -y
yum install binutils -y
yum install postgresql-devel*
wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient18.3-basic-18.3.0.0.0-3.x86_64.rpm
yum install libaio
rpm -ivh oracle-instantclient18.3-basic-18.3.0.0.0-3.x86_64.rpm

yum install postgresql-devel*
yum install unixODBC-devel

#當前准備編譯路徑
export SUPERSET_HOME_BUILD=/opt

python3.7.6環境准備

使用conda創建python環境

1.安裝anaconda

cd $SUPERSET_HOME_BUILD
wget https://repo.anaconda.com/miniconda/Miniconda2-latest-Linux-x86_64.sh
chmod +x Miniconda2-latest-Linux-x86_64.sh
sh Miniconda2-latest-Linux-x86_64.sh

#設置不自動加載base環境
conda config --set auto_activate_base false

2.創建python環境

#創建python存放路徑
python_home=${SUPERSET_HOME_BUILD}/py3.7
mkdir ${python_home}
#創建python3.7環境
conda create  -p ${python_home} python=3.7
#查看並試運行
cd ${python_home} && cd bin 
./python --version
#顯示 Python 3.7.6 即表示成功

superset源碼下載編譯安裝

下載superset源碼

mkdir -p ${SUPERSET_HOME_BUILD}
cd ${SUPERSET_HOME_BUILD}
git clone https://github.com/apache/incubator-superset
export SUPERSET_SOURCE_PATH=${SUPERSET_HOME_BUILD}/incubator-superset-master

創建對應的superset依賴

export PATH=${SUPERSET_HOME_BUILD}/py3.7/bin:$PATH
#安裝依賴
python -m pip install -e ${SUPERSET_SOURCE_PATH}/requirements.txt
#安裝mysql 連接依賴
yum install mariadb* -y
python -m pip install -i https://pypi.tuna.tsinghua.edu.cn/simple mysqlclient

下載nodejs

mkdir ${SUPERSET_HOME_BUILD}
export NODE_PATH=${SUPERSET_HOME_BUILD}/node-v12.18.2-linux-x64
https://nodejs.org/dist/v12.18.2/node-v12.18.2-linux-x64.tar.xz
tar -xvf node-v12.18.2-linux-x64.tar.xz

編譯前端

export SUPERSET_PATH=${SUPERSET_HOME_BUILD}/superset
export PATH=$PATH:$NODE_PATH/bin
mkdir -p ${SUPERSET_PATH}/app/superset-frontend
mkdir -p ${SUPERSET_PATH}/app/superset/assets
export SUPERSET_SOURCE_PATH=${SUPERSET_HOME_BUILD}/incubator-superset-master

#復制前端源碼
cp -r ${SUPERSET_SOURCE_PATH}/superset-frontend/* ${SUPERSET_PATH}/app/superset-frontend
#編譯安裝 // docker容器內編譯需要調整 docker daemon ：   mem 4G swap 2G
cd ${SUPERSET_PATH}/app/superset-frontend && npm ci && npm run build && rm -rf node_modules

編譯后端

#設置環境變量
export SUPERSET_PATH=${SUPERSET_HOME_BUILD}/superset
export PYTHON_HOME=${SUPERSET_HOME_BUILD}/py3.7

export LANG=C.UTF-8
export LC_ALL=C.UTF-8
export FLASK_ENV=production
export FLASK_APP="superset.app:create_app()"
export PYTHONPATH="${PYTHON_HOME}"
export SUPERSET_HOME="$SUPERSET_PATH/app/superset"
export SUPERSET_PORT=8080
export PATH=$PATH:$PYTHON_HOME/bin
export PATH=$PYTHON_HOME/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${SUPERSET_HOME_BUILD}/mysql

cp -r ${SUPERSET_SOURCE_PATH}/superset ${SUPERSET_PATH}/app/
cp ${SUPERSET_SOURCE_PATH}/setup.py ${SUPERSET_PATH}/app/
cp ${SUPERSET_SOURCE_PATH}/MANIFEST.in ${SUPERSET_PATH}/app/
cp ${SUPERSET_SOURCE_PATH}/README.md ${SUPERSET_PATH}/app/

#替換python腳本內的解釋執行器
grep /bin/python ${PYTHON_HOME}/bin/*  -l | xargs sed -i '1s/^.*$/#!\/usr\/bin\/env python/'     

#安裝superset
cd ${SUPERSET_PATH}/app/
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -e .
cd -

superset初始化

#設置初始化admin角色的用戶名密碼
export ADMIN_USERNAME=admin
export ADMIN_FIRSTNAME=Superset
export ADMIN_LASTNAME=Admin
export ADMIN_PASSWORD=admin
export ADMIN_EMAIL=[e-mail]

STEP_CNT=4

echo_step() {
cat <<EOF
######################################################################
Init Step ${1}/${STEP_CNT} [${2}] -- ${3}
######################################################################
EOF
}

# Create an admin user
  echo_step "1" "Starting" "Setting up admin user ( admin / admin )"
  superset fab create-admin \
                --username ${ADMIN_USERNAME} \
                --firstname ${ADMIN_FIRSTNAME} \
                --lastname ${ADMIN_LASTNAME} \
                --email ${ADMIN_EMAIL} \
                --password ${ADMIN_PASSWORD}
  echo_step "1" "Complete" "Setting up admin user"

  # Initialize the database
  echo_step "2" "Starting" "Applying DB migrations"
  superset db upgrade
  echo_step "2" "Complete" "Applying DB migrations"

  if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
      # Load some data to play with
      echo_step "3" "Starting" "Loading examples"
      superset load_examples
      echo_step "3" "Complete" "Loading examples"
  fi

  # Create default roles and permissions
  echo_step "4" "Starting" "Setting up roles and perms"
  superset init
  echo_step "4" "Complete" "Setting up roles and perms"

啟動

#設置環境變量
#當前編譯根路徑，當前選擇的是/opt
export SUPERSET_HOME_BUILD=`pwd`
export SUPERSET_PATH=${SUPERSET_HOME_BUILD}/superset
export PYTHON_HOME=${SUPERSET_HOME_BUILD}/py3.7
#export LANG=C.UTF-8
#export LC_ALL=C.UTF-8
export LC_ALL=en_US.UTF-8
export LANG=en_US.UTF-8
export FLASK_ENV=production
export FLASK_APP="superset.app:create_app()"
export PYTHONPATH="${PYTHON_HOME}"
export SUPERSET_HOME="$SUPERSET_PATH/app/superset"
export SUPERSET_PORT=8080
export PATH=$PATH:$PYTHON_HOME/bin
export PATH=$PYTHON_HOME/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${SUPERSET_HOME_BUILD}/mysql

gunicorn \
        --bind  "0.0.0.0:${SUPERSET_PORT}" \
        --access-logfile '-' \
        --error-logfile '-' \
        --workers 1 \
        --worker-class gthread \
        --threads 20 \
        --timeout 60 \
        --limit-request-line 0 \
        --limit-request-field_size 0 \
        "${FLASK_APP}"