游走的鱼

导航

airflow dags 任务 python脚本

from datetime import datetime, timedelta
from textwrap import dedent
from airflow import DAG
from airflow.operators.bash import BashOperator
from airflow.contrib.operators.ssh_operator import SSHOperator
from airflow.sensors.external_task import ExternalTaskMarker, ExternalTaskSensor
from airflow.operators.dummy import DummyOperator

default_args = {
    'owner': 'LinuxOperation',
    'email': ['airflow@maetimes.com'],
    'email_on_failure': True,
    'email_on_retry': True,
    'depends_on_past': False,
    'retries': 1,
    'retry_delay': timedelta(minutes=5),
    'concurrency': 1,
    'execution_timeout': timedelta(minutes=30),
}

with DAG(
        dag_id='mongo_export',
        description='pokekara_chat_messages doki_chat_messages export json file',
        default_args=default_args,
        schedule_interval=("10 3 * * *"),
        start_date=datetime(2021, 12, 3,),
        catchup=False,
        tags=['mongo_export_task'],
) as dag:

    poke_chat_exec = dedent(
    """
                bash {{ params.dir }}/{{ params.exec_file }}
    """
        )
    doki_chat_exec = dedent(
    """
                bash {{ params.dir }}/{{ params.exec_file }}
    """
        )

    mongo_export = SSHOperator(
        ssh_conn_id='ssh-datanode04',
        task_id='poke_chat_messages_export',
        depends_on_past=False,
        command=poke_chat_exec,
        params={
            'exec_file': 'mongo_export.sh',
            'dir': '/usr/local/src/scripts'
        },
    )
    mongo_export = SSHOperator(
        ssh_conn_id='ssh-datanode04',
        task_id='doki_chat_messages_export',
        depends_on_past=False,
        command=doki_chat_exec,
        params={
            'exec_file': 'doki_chat_messages.sh',
            'dir': '/usr/local/src/scripts'
        },
    )

  airflow会自动识别该脚本任务并执行。

posted on 2022-03-16 18:50  游走的鱼  阅读(230)  评论(0编辑  收藏  举报