Centos7部署AirFlow

Centos7部署AirFlow

官方文檔

1.準備環境

1.1.安裝Python3

yum install epel-release -y
yum install https://centos7.iuscommunity.org/ius-release.rpm -y
yum install python36u -y
ln -s /bin/python3.6 /bin/python3
yum install python36u-pip -y
ln -s /bin/pip3.6 /bin/pip3

1.2.換pip國內源

mkdir ~/.pip
vim ~/.pip/pip.conf

[global]
index-url = https://pypi.tuna.tsinghua.edu.cn/simple

1.3.配置虛擬環境

mkdir -p /usr/local/thirdparty/
cd /usr/local/thirdparty/
python3 -m venv airflow_env
source /usr/local/thirdparty/airflow_env/bin/activate

1.4.安裝操作系統依賴

yum install -y gcc gcc-c++ python-devel python36u-devel

2.安裝AirFlow

• 安裝

export SLUGIFY_USES_TEXT_UNIDECODE=yes
pip install apache-airflow[all]

• 啓動測試

mkdir /usr/local/airflow
cd /usr/local/airflow
export AIRFLOW_HOME=`pwd`
airflow initdb
# 開啓兩個終端，第一個執行
airflow webserver -p 8080
# 第二個執行
airflow scheduler

瀏覽器訪問http://ip:8080查看服務是否啓動成功

3.Systemd部署

3.1.創建MySQL庫和用戶

mysql> CREATE DATABASE airflow; 
mysql> GRANT all privileges on airflow.* TO 'airflow'@'localhost'  IDENTIFIED BY 'airflow'; 
mysql> FLUSH PRIVILEGES; 

3.2.創建airflow系統用戶

useradd airflow

3.3.修改配置

vim /usr/local/airflow/airflow.cfg

# [core]部分修改
# 替換成mysql配置
sql_alchemy_conn =  mysql://airflow:[email protected]:3306/airflow
# 移除示例Dags
load_examples = False

# [webserver] 部分修改
# 開啓認證
authenticate = True
# 添加認證後端(該項配置需要新增)
auth_backend = airflow.contrib.auth.backends.password_auth
# 按用戶過濾
filter_by_owner = True

3.4.修改/usr/local/airflow屬主

chown -R airflow:airflow /usr/local/airflow

3.5.配置systemd相關服務文件

• 獲取文件

GitHub獲取airflow源碼,啓動腳本在scripts/systemd目錄下

獲取systemd目錄到/root下

cd /root/systemd

• 配置airflow文件

vim airflow

替換配置

AIRFLOW_CONFIG=/usr/local/airflow/airflow.cfg
AIRFLOW_HOME=/usr/local/airflow

• 配置airflow-webserver.service文件

 vim airflow-webserver.service
 

替換啓動腳本成如下

ExecStart=/bin/sh -c 'source /usr/local/thirdparty/airflow_env/bin/activate && airflow webserver --pid /run/airflow/webserver.pid'

• 配置airflow-scheduler.service文件

vim airflow-scheduler.service

替換啓動腳本成如下

ExecStart=/bin/sh -c 'source /usr/local/thirdparty/airflow_env/bin/activate && airflow scheduler'

• 配置airflow-flower.service文件

 vim airflow-flower.service

替換啓動腳本成如下

ExecStart=/bin/sh -c 'source /usr/local/thirdparty/airflow_env/bin/activate && airflow flower'

• 配置airflow-worker.service文件

vim airflow-worker.service

替換啓動腳本成如下

ExecStart=/bin/sh -c 'source /usr/local/thirdparty/airflow_env/bin/activate && airflow worker'

• 移動相關文件到對應目錄

mkdir /run/airflow
chown -R airflow:airflow /run/airflow

cp airflow /etc/sysconfig/

cp airflow.conf /etc/tmpfiles.d/

cp airflow-*.service /usr/lib/systemd/system/

3.6.初始化db

source /usr/local/thirdparty/airflow_env/bin/activate
cd /usr/local/airflow
export AIRFLOW_HOME=`pwd`
airflow initdb

3.7.創建用戶

source /usr/local/thirdparty/airflow_env/bin/activate
cd /usr/local/airflow
export AIRFLOW_HOME=`pwd`
vim user_manger.py

複製如下內容

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
from airflow import models, settings
from airflow.contrib.auth.backends.password_auth import PasswordUser


class UserManager(object):

    def __init__(self):
        self.user = models.User()
        self.passwd_user = PasswordUser(self.user)
        self.session = settings.Session()

    def add_user(self, username, passwd, email=None, is_super=False):
        self.passwd_user.username = username
        self.passwd_user.password = passwd
        if email:
            self.passwd_user.email = email
        if is_super is True:
            self.passwd_user.superuser = True
        self.session.add(self.passwd_user)
        self.session.commit()

    def change_passwd(self, username, new_passwd=None, email=None, superuser=None):
        if not any([new_passwd, email, superuser]):
            return
        info = {}
        if new_passwd:
            self.passwd_user.password = new_passwd
            info['password'] = self.passwd_user.password
        if email:
            info['email'] = email
        if superuser is True:
            info['superuser'] = True
        if superuser is False:
            info['superuser'] = False
        try:
            self.session.query(PasswordUser).filter(PasswordUser.username == username).update(info)
        except Exception as e:
            pass
        else:
            self.session.commit()

    def __del__(self):
        self.session.close()


if __name__ == '__main__':

    manager = UserManager()
    manager.add_user('admin', 'admin', 'admin@localhost', True)
    # manager.change_passwd('admin', 'admin', 'admin@localhost', True)

python user_manger.py

3.8.啓動服務

systemctl start airflow-webserver.service
systemctl start airflow-scheduler.service

3.9.設置開機啓動

systemctl enable airflow-webserver.service
systemctl enable airflow-scheduler.service

4.訪問web

瀏覽器訪問http://ip:8080

admin/admin