superset 0.36 搭建开发编译环境

superset

准备环境

1. python3.7.6 环境准备
2. superset安装包准备

系统环境：centos 7.3

#安装编译环境依赖
yum install gcc gcc-c++  -y
yum install binutils -y
yum install postgresql-devel*
wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient18.3-basic-18.3.0.0.0-3.x86_64.rpm
yum install libaio
rpm -ivh oracle-instantclient18.3-basic-18.3.0.0.0-3.x86_64.rpm

yum install postgresql-devel*
yum install unixODBC-devel

#当前准备编译路径
export SUPERSET_HOME_BUILD=/opt

python3.7.6环境准备

使用conda创建python环境

1.安装anaconda

cd $SUPERSET_HOME_BUILD
wget https://repo.anaconda.com/miniconda/Miniconda2-latest-Linux-x86_64.sh
chmod +x Miniconda2-latest-Linux-x86_64.sh
sh Miniconda2-latest-Linux-x86_64.sh

#设置不自动加载base环境
conda config --set auto_activate_base false

2.创建python环境

#创建python存放路径
python_home=${SUPERSET_HOME_BUILD}/py3.7
mkdir ${python_home}
#创建python3.7环境
conda create  -p ${python_home} python=3.7
#查看并试运行
cd ${python_home} && cd bin 
./python --version
#显示 Python 3.7.6 即表示成功

superset源码下载编译安装

下载superset源码

mkdir -p ${SUPERSET_HOME_BUILD}
cd ${SUPERSET_HOME_BUILD}
git clone https://github.com/apache/incubator-superset
export SUPERSET_SOURCE_PATH=${SUPERSET_HOME_BUILD}/incubator-superset-master

创建对应的superset依赖

export PATH=${SUPERSET_HOME_BUILD}/py3.7/bin:$PATH
#安装依赖
python -m pip install -e ${SUPERSET_SOURCE_PATH}/requirements.txt
#安装mysql 连接依赖
yum install mariadb* -y
python -m pip install -i https://pypi.tuna.tsinghua.edu.cn/simple mysqlclient

下载nodejs

mkdir ${SUPERSET_HOME_BUILD}
export NODE_PATH=${SUPERSET_HOME_BUILD}/node-v12.18.2-linux-x64
https://nodejs.org/dist/v12.18.2/node-v12.18.2-linux-x64.tar.xz
tar -xvf node-v12.18.2-linux-x64.tar.xz

编译前端

export SUPERSET_PATH=${SUPERSET_HOME_BUILD}/superset
export PATH=$PATH:$NODE_PATH/bin
mkdir -p ${SUPERSET_PATH}/app/superset-frontend
mkdir -p ${SUPERSET_PATH}/app/superset/assets
export SUPERSET_SOURCE_PATH=${SUPERSET_HOME_BUILD}/incubator-superset-master

#复制前端源码
cp -r ${SUPERSET_SOURCE_PATH}/superset-frontend/* ${SUPERSET_PATH}/app/superset-frontend
#编译安装 // docker容器内编译需要调整 docker daemon ：   mem 4G swap 2G
cd ${SUPERSET_PATH}/app/superset-frontend && npm ci && npm run build && rm -rf node_modules

编译后端

#设置环境变量
export SUPERSET_PATH=${SUPERSET_HOME_BUILD}/superset
export PYTHON_HOME=${SUPERSET_HOME_BUILD}/py3.7

export LANG=C.UTF-8
export LC_ALL=C.UTF-8
export FLASK_ENV=production
export FLASK_APP="superset.app:create_app()"
export PYTHONPATH="${PYTHON_HOME}"
export SUPERSET_HOME="$SUPERSET_PATH/app/superset"
export SUPERSET_PORT=8080
export PATH=$PATH:$PYTHON_HOME/bin
export PATH=$PYTHON_HOME/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${SUPERSET_HOME_BUILD}/mysql

cp -r ${SUPERSET_SOURCE_PATH}/superset ${SUPERSET_PATH}/app/
cp ${SUPERSET_SOURCE_PATH}/setup.py ${SUPERSET_PATH}/app/
cp ${SUPERSET_SOURCE_PATH}/MANIFEST.in ${SUPERSET_PATH}/app/
cp ${SUPERSET_SOURCE_PATH}/README.md ${SUPERSET_PATH}/app/

#替换python脚本内的解释执行器
grep /bin/python ${PYTHON_HOME}/bin/*  -l | xargs sed -i '1s/^.*$/#!\/usr\/bin\/env python/'     

#安装superset
cd ${SUPERSET_PATH}/app/
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -e .
cd -

superset初始化

#设置初始化admin角色的用户名密码
export ADMIN_USERNAME=admin
export ADMIN_FIRSTNAME=Superset
export ADMIN_LASTNAME=Admin
export ADMIN_PASSWORD=admin
export ADMIN_EMAIL=admin@superset.com

STEP_CNT=4

echo_step() {
cat <<EOF
######################################################################
Init Step ${1}/${STEP_CNT} [${2}] -- ${3}
######################################################################
EOF
}

# Create an admin user
  echo_step "1" "Starting" "Setting up admin user ( admin / admin )"
  superset fab create-admin \
                --username ${ADMIN_USERNAME} \
                --firstname ${ADMIN_FIRSTNAME} \
                --lastname ${ADMIN_LASTNAME} \
                --email ${ADMIN_EMAIL} \
                --password ${ADMIN_PASSWORD}
  echo_step "1" "Complete" "Setting up admin user"

  # Initialize the database
  echo_step "2" "Starting" "Applying DB migrations"
  superset db upgrade
  echo_step "2" "Complete" "Applying DB migrations"

  if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
      # Load some data to play with
      echo_step "3" "Starting" "Loading examples"
      superset load_examples
      echo_step "3" "Complete" "Loading examples"
  fi

  # Create default roles and permissions
  echo_step "4" "Starting" "Setting up roles and perms"
  superset init
  echo_step "4" "Complete" "Setting up roles and perms"

启动

#设置环境变量
#当前编译根路径，当前选择的是/opt
export SUPERSET_HOME_BUILD=`pwd`
export SUPERSET_PATH=${SUPERSET_HOME_BUILD}/superset
export PYTHON_HOME=${SUPERSET_HOME_BUILD}/py3.7
#export LANG=C.UTF-8
#export LC_ALL=C.UTF-8
export LC_ALL=en_US.UTF-8
export LANG=en_US.UTF-8
export FLASK_ENV=production
export FLASK_APP="superset.app:create_app()"
export PYTHONPATH="${PYTHON_HOME}"
export SUPERSET_HOME="$SUPERSET_PATH/app/superset"
export SUPERSET_PORT=8080
export PATH=$PATH:$PYTHON_HOME/bin
export PATH=$PYTHON_HOME/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${SUPERSET_HOME_BUILD}/mysql

gunicorn \
        --bind  "0.0.0.0:${SUPERSET_PORT}" \
        --access-logfile '-' \
        --error-logfile '-' \
        --workers 1 \
        --worker-class gthread \
        --threads 20 \
        --timeout 60 \
        --limit-request-line 0 \
        --limit-request-field_size 0 \
        "${FLASK_APP}"