mirror of
https://github.com/datahub-project/datahub.git
synced 2025-07-28 20:09:59 +00:00
47 lines
1.3 KiB
Python
47 lines
1.3 KiB
Python
|
"""Lineage Backend
|
||
|
|
|
||
|
|
An example DAG demonstrating the usage of DataHub's Airflow lineage backend.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from datetime import timedelta
|
||
|
|
|
||
|
|
from airflow import DAG
|
||
|
|
from airflow.utils.dates import days_ago
|
||
|
|
|
||
|
|
try:
|
||
|
|
from airflow.operators.bash import BashOperator
|
||
|
|
except ImportError:
|
||
|
|
from airflow.operators.bash_operator import BashOperator
|
||
|
|
|
||
|
|
from datahub.integrations.airflow.entities import Dataset
|
||
|
|
|
||
|
|
default_args = {
|
||
|
|
"owner": "airflow",
|
||
|
|
"depends_on_past": False,
|
||
|
|
"email": ["jdoe@example.com"],
|
||
|
|
"email_on_failure": False,
|
||
|
|
"execution_timeout": timedelta(minutes=5),
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
with DAG(
|
||
|
|
"datahub_lineage_backend_demo",
|
||
|
|
default_args=default_args,
|
||
|
|
description="An example DAG demonstrating the usage of DataHub's Airflow lineage backend.",
|
||
|
|
schedule_interval=timedelta(days=1),
|
||
|
|
start_date=days_ago(2),
|
||
|
|
catchup=False,
|
||
|
|
) as dag:
|
||
|
|
task1 = BashOperator(
|
||
|
|
task_id="run_data_task",
|
||
|
|
dag=dag,
|
||
|
|
bash_command="echo 'This is where you might run your data tooling.'",
|
||
|
|
inlets={
|
||
|
|
"datasets": [
|
||
|
|
Dataset("snowflake", "mydb.schema.tableA"),
|
||
|
|
Dataset("snowflake", "mydb.schema.tableB"),
|
||
|
|
],
|
||
|
|
},
|
||
|
|
outlets={"datasets": [Dataset("snowflake", "mydb.schema.tableC")]},
|
||
|
|
)
|