Airflow安装

一、依赖

yum install -y freetds-bin krb5-user ldap-utils libffi6 libsasl2-2 libsasl2-modules libssl1.1 locales lsb-release sasl2-bin sqlite3 unixodbc
yum install MySQL-python
pip install PyMySQL==0.7.1

二、PyPI安装

1、说明

Python版本：3.6.5
Airflow版本：2.2.5
建议运行 Airflow 所需的最低内存为 4GB

2、安装

#设置安装目录
export AIRFLOW_HOME=/opt/module/airflow

#忽略依赖
pip install --ignore-installed wrapt typing swagger-ui-bundle sqlalchemy-utils pytzdata python-slugify python-dateutil pygments prison marshmallow-sqlalchemy marshmallow-enum Mako lockfile kombu importlib-resources httpcore flask-wtf Flask-SQLAlchemy Flask-OpenID flask-login Flask-JWT-Extended Flask-Babel email-validator docutils connexion commonmark colorama click-repl click-plugins click-didyoumean cffi cachelib billiard async-generator tornado termcolor tenacity tabulate sqlalchemy-jsonfield setproctitle rich python-nvd3 python-daemon psutil prometheus-client pep562 pendulum marshmallow-oneofschema markdown lazy-object-proxy iso8601 humanize httpx gunicorn graphviz flask-session flask-caching flask-appbuilder dill deprecated cryptography croniter colorlog celery cattrs blinker argcomplete apache-airflow-providers-sqlite apache-airflow-providers-imap apache-airflow-providers-http apache-airflow-providers-ftp alembic flower apache-airflow apache-airflow-providers-celery
pip install mysqlclient

#执行安装命令
pip install "apache-airflow[celery]==2.2.5" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.2.5/constraints-3.6.txt"

3、设置MySQL数据库并启动服务

#跟踪已应用的迁移
airflow db upgrade

#修改配置文件
cd /opt/module/airflow
vim airflow.cfg

#修改如下内容
executor = LocalExecutor
sql_alchemy_conn = mysql://root:TSf96KHSW4Ie511YsjQ1@192.168.6.30:3306/airflow_db?charset=utf8

#修改mysqldb.py，解决(MySQLdb 对 python3.*支持)
vim /opt/anaconda3/lib/python3.6/site-packages/sqlalchemy/dialects/mysql/mysqldb.py

#在代码开头添加
import pymysql
pymysql.install_as_MySQLdb()

#检查数据库URL
airflow config get-value core sql_alchemy_conn

#初始化数据库
airflow db init

#设置用户名密码邮箱
airflow users create \
--username admin \
--firstname aa \
--lastname bb \
--role Admin \
--email xxx@superhero.org

# 守护进程运行webserver, 默认端口为8080，也可以通过`-p`来指定端口
airflow webserver -D --port 8080
airflow webserver --port 8080

# 守护进程运行调度器，守护进程即表示后台启动
airflow scheduler -D

#启动调度
airflow scheduler

4、访问8080端口

http://127.0.0.1:8080