diff --git a/ingestion/examples/sample_data/pipelines/pipelines.json b/ingestion/examples/sample_data/pipelines/pipelines.json index 2b3e2ed5a5f..b2d7280cd74 100644 --- a/ingestion/examples/sample_data/pipelines/pipelines.json +++ b/ingestion/examples/sample_data/pipelines/pipelines.json @@ -4,56 +4,184 @@ "displayName": "Presto ETL", "description": "Presto ETL pipeline", "pipelineUrl": "http://localhost:8080/tree?dag_id=presto_etl", - "tasks": ["presto_task", "assert_table_exists"] + "tasks": [ + { + "name": "presto_task", + "displayName": "Presto Task", + "description": "Airflow operator to perform ETL on presto tables", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists", + "downstreamTasks": ["assert_table_exists"], + "taskType": "PrestoOperator" + }, + { + "name": "assert_table_exists", + "displayName": "Assert Table Exists", + "description": "Assert if a table exists", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists", + "downstreamTasks": [], + "taskType": "HiveOperator" + } + ] }, { "name": "dim_address_etl", "displayName": "dim_address etl", "description": "dim_address ETL pipeline", "pipelineUrl": "http://localhost:8080/tree?dag_id=dim_address_etl", - "tasks": ["dim_address_task", "assert_table_exists"] + "tasks": [{ + "name": "dim_address_task", + "displayName": "dim_address Task", + "description": "Airflow operator to perform ETL and generate dim_address table", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=dim_address_task", + "downstreamTasks": ["assert_table_exists"], + "taskType": "PrestoOperator" + }, + { + "name": "assert_table_exists", + "displayName": "Assert Table Exists", + "description": "Assert if a table exists", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists", + "downstreamTasks": [], + "taskType": "HiveOperator" + } + ] }, { "name": "dim_user_etl", "displayName": "dim_user etl", "description": "dim_user ETL pipeline", "pipelineUrl": "http://localhost:8080/tree?dag_id=dim_user_etl", - "tasks": ["dim_user_task", "assert_table_exists"] + "tasks": [{ + "name": "dim_user_task", + "displayName": "dim_user Task", + "description": "Airflow operator to perform ETL and generate dim_user table", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=dim_user_task", + "downstreamTasks": ["assert_table_exists"], + "taskType": "PrestoOperator" + }, + { + "name": "assert_table_exists", + "displayName": "Assert Table Exists", + "description": "Assert if a table exists", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists", + "downstreamTasks": [], + "taskType": "HiveOperator" + } + ] }, { "name": "dim_location_etl", "displayName": "dim_location etl", "description": "diim_location ETL pipeline", "pipelineUrl": "http://localhost:8080/tree?dag_id=dim_address_etl", - "tasks": ["dim_location_task", "assert_table_exists"] + "tasks": [{ + "name": "dim_location_task", + "displayName": "dim_location Task", + "description": "Airflow operator to perform ETL and generate dim_location table", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=dim_location_task", + "downstreamTasks": ["assert_table_exists"], + "taskType": "PrestoOperator" + }, + { + "name": "assert_table_exists", + "displayName": "Assert Table Exists", + "description": "Assert if a table exists", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists", + "downstreamTasks": [], + "taskType": "HiveOperator" + } + ] }, { "name": "dim_product_etl", "displayName": "dim_product etl", "description": "diim_product ETL pipeline", "pipelineUrl": "http://localhost:8080/tree?dag_id=dim_address_etl", - "tasks": ["dim_product_task", "assert_table_exists"] + "tasks": [{ + "name": "dim_product_task", + "displayName": "dim_product Task", + "description": "Airflow operator to perform ETL and generate dim_product table", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=dim_product_task", + "downstreamTasks": ["assert_table_exists"], + "taskType": "PrestoOperator" + }, + { + "name": "assert_table_exists", + "displayName": "Assert Table Exists", + "description": "Assert if a table exists", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists", + "downstreamTasks": [], + "taskType": "HiveOperator" + } + ] }, { "name": "trino_etl", "displayName": "Trino ETL", "description": "Trino ETL pipeline", "pipelineUrl": "http://localhost:8080/tree?dag_id=trino_etl", - "tasks": ["trino_task", "assert_table_exists"] + "tasks": [{ + "name": "trino_task", + "displayName": "Trino Task", + "description": "Airflow operator to perform ETL on trino tables", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists", + "downstreamTasks": ["assert_table_exists"], + "taskType": "TrinoOperator" + }, + { + "name": "assert_table_exists", + "displayName": "Assert Table Exists", + "description": "Assert if a table exists", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists", + "downstreamTasks": [], + "taskType": "HiveOperator" + } + ] }, { "name": "hive_etl", "displayName": "Hive ETL", "description": "Hive ETL pipeline", "pipelineUrl": "http://localhost:8080/tree?dag_id=hive_etl", - "tasks": ["hive_create_table", "assert_table_exists"] + "tasks": [{ + "name": "hive_create_table", + "displayName": "Hive Create Table", + "description": "Hive Create Table Task", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=hive_create_table", + "downstreamTasks": ["assert_table_exits"], + "taskType": "HiveOperator" + }, + { + "name": "assert_table_exists", + "displayName": "Assert Table Exists", + "description": "Assert if a table exists", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists", + "downstreamTasks": [], + "taskType": "HiveOperator" + } + ] }, { "name": "snowflake_etl", "displayName": "Snowflake ETL", "description": "Snowflake ETL pipeline", "pipelineUrl": "http://localhost:8080/tree?dag_id=snowflake_etl", - "tasks": ["snowflake_etl", "assert_table_exists"] + "tasks": [{ + "name": "snowflake_task", + "displayName": "Snowflake Task", + "description": "Airflow operator to perform ETL on snowflake tables", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists", + "downstreamTasks": ["assert_table_exists"], + "taskType": "SnowflakeOperator" + }, + { + "name": "assert_table_exists", + "displayName": "Assert Table Exists", + "description": "Assert if a table exists", + "taskUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists", + "downstreamTasks": [], + "taskType": "HiveOperator" + }] } ] } \ No newline at end of file diff --git a/ingestion/src/metadata/generated/data/tags/__init__.py b/ingestion/src/metadata/generated/data/tags/__init__.py index a103b390350..aa11f3c6b42 100644 --- a/ingestion/src/metadata/generated/data/tags/__init__.py +++ b/ingestion/src/metadata/generated/data/tags/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 diff --git a/ingestion/src/metadata/generated/data/tags/personalDataTags.py b/ingestion/src/metadata/generated/data/tags/personalDataTags.py index 0e07a8fa37c..2829ce7cdcd 100644 --- a/ingestion/src/metadata/generated/data/tags/personalDataTags.py +++ b/ingestion/src/metadata/generated/data/tags/personalDataTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/personalDataTags.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/data/tags/piiTags.py b/ingestion/src/metadata/generated/data/tags/piiTags.py index 4dd45049526..3ba06c2f3da 100644 --- a/ingestion/src/metadata/generated/data/tags/piiTags.py +++ b/ingestion/src/metadata/generated/data/tags/piiTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/piiTags.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/data/tags/tierTags.py b/ingestion/src/metadata/generated/data/tags/tierTags.py index ba0762a1dfa..4d7a218136b 100644 --- a/ingestion/src/metadata/generated/data/tags/tierTags.py +++ b/ingestion/src/metadata/generated/data/tags/tierTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/tierTags.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/data/tags/userTags.py b/ingestion/src/metadata/generated/data/tags/userTags.py index dcfaa1bcc66..d889f0dd3d9 100644 --- a/ingestion/src/metadata/generated/data/tags/userTags.py +++ b/ingestion/src/metadata/generated/data/tags/userTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/userTags.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/__init__.py b/ingestion/src/metadata/generated/schema/api/__init__.py index a103b390350..aa11f3c6b42 100644 --- a/ingestion/src/metadata/generated/schema/api/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/catalogVersion.py b/ingestion/src/metadata/generated/schema/api/catalogVersion.py index deae2218161..1d5058514b7 100644 --- a/ingestion/src/metadata/generated/schema/api/catalogVersion.py +++ b/ingestion/src/metadata/generated/schema/api/catalogVersion.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/catalogVersion.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/__init__.py b/ingestion/src/metadata/generated/schema/api/data/__init__.py index a103b390350..aa11f3c6b42 100644 --- a/ingestion/src/metadata/generated/schema/api/data/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/data/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/data/createChart.py b/ingestion/src/metadata/generated/schema/api/data/createChart.py index 849b9161c17..ad67160fa7c 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createChart.py +++ b/ingestion/src/metadata/generated/schema/api/data/createChart.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createChart.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createDashboard.py b/ingestion/src/metadata/generated/schema/api/data/createDashboard.py index 9048fc9aa7b..e1f98a0f280 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createDashboard.py +++ b/ingestion/src/metadata/generated/schema/api/data/createDashboard.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createDashboard.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createDatabase.py b/ingestion/src/metadata/generated/schema/api/data/createDatabase.py index c7ae0cf5dca..23b1cab6503 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createDatabase.py +++ b/ingestion/src/metadata/generated/schema/api/data/createDatabase.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createDatabase.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createModel.py b/ingestion/src/metadata/generated/schema/api/data/createModel.py index 2f2af309933..45bc58bc27e 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createModel.py +++ b/ingestion/src/metadata/generated/schema/api/data/createModel.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createModel.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createPipeline.py b/ingestion/src/metadata/generated/schema/api/data/createPipeline.py index c95802a0446..0ebc7220c09 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createPipeline.py +++ b/ingestion/src/metadata/generated/schema/api/data/createPipeline.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createPipeline.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations @@ -8,6 +8,7 @@ from typing import List, Optional from pydantic import AnyUrl, BaseModel, Field, constr +from ...entity.data import pipeline from ...type import basic, entityReference, tagLabel @@ -32,7 +33,7 @@ class CreatePipelineEntityRequest(BaseModel): startDate: Optional[basic.DateTime] = Field( None, description='Start date of the workflow' ) - tasks: Optional[List[entityReference.EntityReference]] = Field( + tasks: Optional[List[pipeline.Task]] = Field( None, description='All the tasks that are part of pipeline.' ) tags: Optional[List[tagLabel.TagLabel]] = Field( diff --git a/ingestion/src/metadata/generated/schema/api/data/createTable.py b/ingestion/src/metadata/generated/schema/api/data/createTable.py index a71c39b9dbe..b2468e24ef3 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createTable.py +++ b/ingestion/src/metadata/generated/schema/api/data/createTable.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createTable.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createTopic.py b/ingestion/src/metadata/generated/schema/api/data/createTopic.py index 1ee9cca1381..4f149ebccdb 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createTopic.py +++ b/ingestion/src/metadata/generated/schema/api/data/createTopic.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createTopic.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/feed/__init__.py b/ingestion/src/metadata/generated/schema/api/feed/__init__.py index a103b390350..aa11f3c6b42 100644 --- a/ingestion/src/metadata/generated/schema/api/feed/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/feed/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/feed/createThread.py b/ingestion/src/metadata/generated/schema/api/feed/createThread.py index 11a58a84fbc..de57a0a1823 100644 --- a/ingestion/src/metadata/generated/schema/api/feed/createThread.py +++ b/ingestion/src/metadata/generated/schema/api/feed/createThread.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/feed/createThread.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/lineage/__init__.py b/ingestion/src/metadata/generated/schema/api/lineage/__init__.py index a103b390350..aa11f3c6b42 100644 --- a/ingestion/src/metadata/generated/schema/api/lineage/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/lineage/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/lineage/addLineage.py b/ingestion/src/metadata/generated/schema/api/lineage/addLineage.py index 59288cc6a76..adf75a038c1 100644 --- a/ingestion/src/metadata/generated/schema/api/lineage/addLineage.py +++ b/ingestion/src/metadata/generated/schema/api/lineage/addLineage.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/lineage/addLineage.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/__init__.py b/ingestion/src/metadata/generated/schema/api/services/__init__.py index a103b390350..aa11f3c6b42 100644 --- a/ingestion/src/metadata/generated/schema/api/services/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/services/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/services/createDashboardService.py b/ingestion/src/metadata/generated/schema/api/services/createDashboardService.py index 53e648bcf8e..652cf00a4e5 100644 --- a/ingestion/src/metadata/generated/schema/api/services/createDashboardService.py +++ b/ingestion/src/metadata/generated/schema/api/services/createDashboardService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/createDashboardService.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py b/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py index d201ee766cf..ca85d330ae3 100644 --- a/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py +++ b/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/createDatabaseService.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/createMessagingService.py b/ingestion/src/metadata/generated/schema/api/services/createMessagingService.py index 206798cd2fd..c41c5222951 100644 --- a/ingestion/src/metadata/generated/schema/api/services/createMessagingService.py +++ b/ingestion/src/metadata/generated/schema/api/services/createMessagingService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/createMessagingService.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/createPipelineService.py b/ingestion/src/metadata/generated/schema/api/services/createPipelineService.py index 19c687ba485..03c4a046660 100644 --- a/ingestion/src/metadata/generated/schema/api/services/createPipelineService.py +++ b/ingestion/src/metadata/generated/schema/api/services/createPipelineService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/createPipelineService.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/updateDashboardService.py b/ingestion/src/metadata/generated/schema/api/services/updateDashboardService.py index 9cb6b1b9776..3759b79b1d7 100644 --- a/ingestion/src/metadata/generated/schema/api/services/updateDashboardService.py +++ b/ingestion/src/metadata/generated/schema/api/services/updateDashboardService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/updateDashboardService.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py b/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py index 7ba5a3bc86b..c519a9b0e1d 100644 --- a/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py +++ b/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/updateDatabaseService.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/updateMessagingService.py b/ingestion/src/metadata/generated/schema/api/services/updateMessagingService.py index 804c8ae764f..d0ec7aeecfd 100644 --- a/ingestion/src/metadata/generated/schema/api/services/updateMessagingService.py +++ b/ingestion/src/metadata/generated/schema/api/services/updateMessagingService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/updateMessagingService.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/updatePipelineService.py b/ingestion/src/metadata/generated/schema/api/services/updatePipelineService.py index cd4db3cbb15..4198654bde4 100644 --- a/ingestion/src/metadata/generated/schema/api/services/updatePipelineService.py +++ b/ingestion/src/metadata/generated/schema/api/services/updatePipelineService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/updatePipelineService.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/setOwner.py b/ingestion/src/metadata/generated/schema/api/setOwner.py index 8547d417da8..1c6107935bd 100644 --- a/ingestion/src/metadata/generated/schema/api/setOwner.py +++ b/ingestion/src/metadata/generated/schema/api/setOwner.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/setOwner.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/tags/__init__.py b/ingestion/src/metadata/generated/schema/api/tags/__init__.py index a103b390350..aa11f3c6b42 100644 --- a/ingestion/src/metadata/generated/schema/api/tags/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/tags/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/tags/createTag.py b/ingestion/src/metadata/generated/schema/api/tags/createTag.py index 29b4338c7a0..15b95a112db 100644 --- a/ingestion/src/metadata/generated/schema/api/tags/createTag.py +++ b/ingestion/src/metadata/generated/schema/api/tags/createTag.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/tags/createTag.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py b/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py index 7961f85d34a..a591c11692f 100644 --- a/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py +++ b/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/tags/createTagCategory.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/teams/__init__.py b/ingestion/src/metadata/generated/schema/api/teams/__init__.py index a103b390350..aa11f3c6b42 100644 --- a/ingestion/src/metadata/generated/schema/api/teams/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/teams/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/teams/createTeam.py b/ingestion/src/metadata/generated/schema/api/teams/createTeam.py index 1a818dd8f47..729153d3578 100644 --- a/ingestion/src/metadata/generated/schema/api/teams/createTeam.py +++ b/ingestion/src/metadata/generated/schema/api/teams/createTeam.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/teams/createTeam.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/teams/createUser.py b/ingestion/src/metadata/generated/schema/api/teams/createUser.py index c21733107a0..359fdfd7bdc 100644 --- a/ingestion/src/metadata/generated/schema/api/teams/createUser.py +++ b/ingestion/src/metadata/generated/schema/api/teams/createUser.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/teams/createUser.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/__init__.py b/ingestion/src/metadata/generated/schema/entity/__init__.py index a103b390350..aa11f3c6b42 100644 --- a/ingestion/src/metadata/generated/schema/entity/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/bots.py b/ingestion/src/metadata/generated/schema/entity/bots.py index 4807cf70d09..44a041c07e2 100644 --- a/ingestion/src/metadata/generated/schema/entity/bots.py +++ b/ingestion/src/metadata/generated/schema/entity/bots.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/bots.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/__init__.py b/ingestion/src/metadata/generated/schema/entity/data/__init__.py index a103b390350..aa11f3c6b42 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/data/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/data/chart.py b/ingestion/src/metadata/generated/schema/entity/data/chart.py index 9125ae7ea57..cab6d8afbf9 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/chart.py +++ b/ingestion/src/metadata/generated/schema/entity/data/chart.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/chart.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/dashboard.py b/ingestion/src/metadata/generated/schema/entity/data/dashboard.py index 93eb81cb6dd..5d84984b338 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/dashboard.py +++ b/ingestion/src/metadata/generated/schema/entity/data/dashboard.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/dashboard.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/database.py b/ingestion/src/metadata/generated/schema/entity/data/database.py index dd367022586..e367b56a472 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/database.py +++ b/ingestion/src/metadata/generated/schema/entity/data/database.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/database.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/metrics.py b/ingestion/src/metadata/generated/schema/entity/data/metrics.py index 84b06eb28a5..a7f42f4e215 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/metrics.py +++ b/ingestion/src/metadata/generated/schema/entity/data/metrics.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/metrics.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/model.py b/ingestion/src/metadata/generated/schema/entity/data/model.py index d53b90285ab..185817568af 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/model.py +++ b/ingestion/src/metadata/generated/schema/entity/data/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/model.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/pipeline.py b/ingestion/src/metadata/generated/schema/entity/data/pipeline.py index b680c2e74bd..96bd6b1439c 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/pipeline.py +++ b/ingestion/src/metadata/generated/schema/entity/data/pipeline.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/pipeline.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations @@ -11,6 +11,37 @@ from pydantic import AnyUrl, BaseModel, Field, constr from ...type import basic, entityHistory, entityReference, tagLabel +class Task(BaseModel): + name: constr(min_length=1, max_length=64) = Field( + ..., description='Name that identifies this task instance uniquely.' + ) + displayName: Optional[str] = Field( + None, + description='Display Name that identifies this Task. It could be title or label from the pipeline services.', + ) + fullyQualifiedName: Optional[constr(min_length=1, max_length=64)] = Field( + None, + description="A unique name that identifies a pipeline in the format 'ServiceName.PipelineName.TaskName'.", + ) + description: Optional[str] = Field(None, description='Description of this Task.') + taskUrl: Optional[AnyUrl] = Field( + None, + description='Task URL to visit/manage. This URL points to respective pipeline service UI.', + ) + downstreamTasks: Optional[List[constr(min_length=1, max_length=64)]] = Field( + None, description='All the tasks that are downstream of this task.' + ) + taskType: Optional[str] = Field( + None, description='Type of the Task. Usually refers to the class it implements.' + ) + taskSQL: Optional[basic.SqlQuery] = Field( + None, description='SQL used in the task. Can be used to determine the lineage.' + ) + tags: Optional[List[tagLabel.TagLabel]] = Field( + None, description='Tags for this task.' + ) + + class Pipeline(BaseModel): id: basic.Uuid = Field( ..., description='Unique identifier that identifies a pipeline instance.' @@ -46,7 +77,7 @@ class Pipeline(BaseModel): startDate: Optional[basic.DateTime] = Field( None, description='Start date of the workflow.' ) - tasks: Optional[List[entityReference.EntityReference]] = Field( + tasks: Optional[List[Task]] = Field( None, description='All the tasks that are part of pipeline.' ) followers: Optional[entityReference.EntityReferenceList] = Field( diff --git a/ingestion/src/metadata/generated/schema/entity/data/report.py b/ingestion/src/metadata/generated/schema/entity/data/report.py index f1ffbcec5d2..b0f7a96c305 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/report.py +++ b/ingestion/src/metadata/generated/schema/entity/data/report.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/report.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/table.py b/ingestion/src/metadata/generated/schema/entity/data/table.py index a4f2faa4e68..16cb0d1ab0a 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/table.py +++ b/ingestion/src/metadata/generated/schema/entity/data/table.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/table.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/topic.py b/ingestion/src/metadata/generated/schema/entity/data/topic.py index a84f27817ff..df983dbf347 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/topic.py +++ b/ingestion/src/metadata/generated/schema/entity/data/topic.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/topic.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/feed/__init__.py b/ingestion/src/metadata/generated/schema/entity/feed/__init__.py index a103b390350..aa11f3c6b42 100644 --- a/ingestion/src/metadata/generated/schema/entity/feed/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/feed/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/feed/thread.py b/ingestion/src/metadata/generated/schema/entity/feed/thread.py index f61bcee5e60..de39a495fb7 100644 --- a/ingestion/src/metadata/generated/schema/entity/feed/thread.py +++ b/ingestion/src/metadata/generated/schema/entity/feed/thread.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/feed/thread.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/services/__init__.py b/ingestion/src/metadata/generated/schema/entity/services/__init__.py index a103b390350..aa11f3c6b42 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/services/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/services/dashboardService.py b/ingestion/src/metadata/generated/schema/entity/services/dashboardService.py index 2389d5c400d..cb3ebe28a4f 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/dashboardService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/dashboardService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/services/dashboardService.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/services/databaseService.py b/ingestion/src/metadata/generated/schema/entity/services/databaseService.py index 68b7d15047f..e110ef3639f 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/databaseService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/databaseService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/services/databaseService.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/services/messagingService.py b/ingestion/src/metadata/generated/schema/entity/services/messagingService.py index 247d4ecdb43..fecc3c8b38e 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/messagingService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/messagingService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/services/messagingService.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/services/pipelineService.py b/ingestion/src/metadata/generated/schema/entity/services/pipelineService.py index e1cdbaebdac..564ad3ac4da 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/pipelineService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/pipelineService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/services/pipelineService.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/tags/__init__.py b/ingestion/src/metadata/generated/schema/entity/tags/__init__.py index a103b390350..aa11f3c6b42 100644 --- a/ingestion/src/metadata/generated/schema/entity/tags/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/tags/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py b/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py index f8d6ca34da5..024f71e6ea6 100644 --- a/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py +++ b/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/tags/tagCategory.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/teams/__init__.py b/ingestion/src/metadata/generated/schema/entity/teams/__init__.py index a103b390350..aa11f3c6b42 100644 --- a/ingestion/src/metadata/generated/schema/entity/teams/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/teams/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/teams/team.py b/ingestion/src/metadata/generated/schema/entity/teams/team.py index b8f76f6a718..ed7cc365e38 100644 --- a/ingestion/src/metadata/generated/schema/entity/teams/team.py +++ b/ingestion/src/metadata/generated/schema/entity/teams/team.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/teams/team.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/teams/user.py b/ingestion/src/metadata/generated/schema/entity/teams/user.py index d625b6a44e0..ee240a67b5f 100644 --- a/ingestion/src/metadata/generated/schema/entity/teams/user.py +++ b/ingestion/src/metadata/generated/schema/entity/teams/user.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/teams/user.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/__init__.py b/ingestion/src/metadata/generated/schema/type/__init__.py index a103b390350..aa11f3c6b42 100644 --- a/ingestion/src/metadata/generated/schema/type/__init__.py +++ b/ingestion/src/metadata/generated/schema/type/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 diff --git a/ingestion/src/metadata/generated/schema/type/auditLog.py b/ingestion/src/metadata/generated/schema/type/auditLog.py index 94630d9a324..cc4fa7ed143 100644 --- a/ingestion/src/metadata/generated/schema/type/auditLog.py +++ b/ingestion/src/metadata/generated/schema/type/auditLog.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/auditLog.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/basic.py b/ingestion/src/metadata/generated/schema/type/basic.py index ee200dac721..39f9df8a954 100644 --- a/ingestion/src/metadata/generated/schema/type/basic.py +++ b/ingestion/src/metadata/generated/schema/type/basic.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/basic.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py b/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py index 78473ac1471..051a7e268da 100644 --- a/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py +++ b/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/collectionDescriptor.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/dailyCount.py b/ingestion/src/metadata/generated/schema/type/dailyCount.py index 9179cc5a57d..3ec821a49d4 100644 --- a/ingestion/src/metadata/generated/schema/type/dailyCount.py +++ b/ingestion/src/metadata/generated/schema/type/dailyCount.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/dailyCount.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/entityHistory.py b/ingestion/src/metadata/generated/schema/type/entityHistory.py new file mode 100644 index 00000000000..293a8c96c72 --- /dev/null +++ b/ingestion/src/metadata/generated/schema/type/entityHistory.py @@ -0,0 +1,43 @@ +# generated by datamodel-codegen: +# filename: schema/type/entityHistory.json +# timestamp: 2021-10-31T21:55:34+00:00 + +from __future__ import annotations + +from typing import List, Optional + +from pydantic import BaseModel, Extra, Field, confloat + + +class EntityVersionHistory(BaseModel): + class Config: + extra = Extra.forbid + + entityType: str = Field( + ..., + description='Entity type, such as `database`, `table`, `dashboard`, for which this version history is produced.', + ) + versions: List + + +class EntityVersion(BaseModel): + __root__: confloat(ge=0.1, multiple_of=0.1) = Field( + ..., + description='Metadata version of the entity in the form `Major.Minor`. First version always starts from `0.1` when the entity is created. When the backward compatible changes are made to the entity, only the `Minor` version is incremented - example `1.0` is changed to `1.1`. When backward incompatible changes are made the `Major` version is incremented - example `1.1` to `2.0`.', + ) + + +class ChangeDescription(BaseModel): + class Config: + extra = Extra.forbid + + fieldsAdded: Optional[List[str]] = Field( + None, description='Fields added during the version changes.' + ) + fieldsUpdated: Optional[List[str]] = Field( + None, description='Fields modified during the version changes.' + ) + fieldsDeleted: Optional[List[str]] = Field( + None, description='Fields deleted during the version changes.' + ) + previousVersion: Optional[EntityVersion] = None diff --git a/ingestion/src/metadata/generated/schema/type/entityLineage.py b/ingestion/src/metadata/generated/schema/type/entityLineage.py index cd1e76dcf15..f21529388ea 100644 --- a/ingestion/src/metadata/generated/schema/type/entityLineage.py +++ b/ingestion/src/metadata/generated/schema/type/entityLineage.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/entityLineage.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/entityReference.py b/ingestion/src/metadata/generated/schema/type/entityReference.py index 5c60857f24a..fe4956755e1 100644 --- a/ingestion/src/metadata/generated/schema/type/entityReference.py +++ b/ingestion/src/metadata/generated/schema/type/entityReference.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/entityReference.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/entityUsage.py b/ingestion/src/metadata/generated/schema/type/entityUsage.py index ae4fffe2f4d..03eec4c5192 100644 --- a/ingestion/src/metadata/generated/schema/type/entityUsage.py +++ b/ingestion/src/metadata/generated/schema/type/entityUsage.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/entityUsage.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/jdbcConnection.py b/ingestion/src/metadata/generated/schema/type/jdbcConnection.py index 7158ab81a24..65950bbcdf0 100644 --- a/ingestion/src/metadata/generated/schema/type/jdbcConnection.py +++ b/ingestion/src/metadata/generated/schema/type/jdbcConnection.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/jdbcConnection.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/paging.py b/ingestion/src/metadata/generated/schema/type/paging.py index 001e1eb6ce6..5cebb9bb1f1 100644 --- a/ingestion/src/metadata/generated/schema/type/paging.py +++ b/ingestion/src/metadata/generated/schema/type/paging.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/paging.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/profile.py b/ingestion/src/metadata/generated/schema/type/profile.py index b62f5445491..6f2bf71b071 100644 --- a/ingestion/src/metadata/generated/schema/type/profile.py +++ b/ingestion/src/metadata/generated/schema/type/profile.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/profile.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/schedule.py b/ingestion/src/metadata/generated/schema/type/schedule.py index fa7e2629b8c..b2c63a2cf20 100644 --- a/ingestion/src/metadata/generated/schema/type/schedule.py +++ b/ingestion/src/metadata/generated/schema/type/schedule.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/schedule.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/tagLabel.py b/ingestion/src/metadata/generated/schema/type/tagLabel.py index 0fa23556d34..e549f3d9c20 100644 --- a/ingestion/src/metadata/generated/schema/type/tagLabel.py +++ b/ingestion/src/metadata/generated/schema/type/tagLabel.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/tagLabel.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/usageDetails.py b/ingestion/src/metadata/generated/schema/type/usageDetails.py index 31ae05f95f8..108bfeefa51 100644 --- a/ingestion/src/metadata/generated/schema/type/usageDetails.py +++ b/ingestion/src/metadata/generated/schema/type/usageDetails.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/usageDetails.json -# timestamp: 2021-10-31T08:53:25+00:00 +# timestamp: 2021-10-31T21:55:34+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch.py b/ingestion/src/metadata/ingestion/sink/elasticsearch.py index 7508079d52e..b02194016e8 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch.py @@ -359,14 +359,14 @@ class ElasticsearchSink(Sink): tier = pipeline_tag.tagFQN else: tags.add(pipeline_tag.tagFQN) - tasks: List[Task] = self._get_tasks(pipeline.tasks) + tasks: List[Task] = pipeline.tasks task_names = [] task_descriptions = [] for task in tasks: task_names.append(task.displayName) if task.description is not None: task_descriptions.append(task.description) - if len(task.tags) > 0: + if tags in task and len(task.tags) > 0: for col_tag in task.tags: tags.add(col_tag.tagFQN) diff --git a/ingestion/src/metadata/ingestion/source/sample_data.py b/ingestion/src/metadata/ingestion/source/sample_data.py index 01e169d027c..dcdaa53e88a 100644 --- a/ingestion/src/metadata/ingestion/source/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/sample_data.py @@ -352,7 +352,6 @@ class SampleDataSource(Source): yield from self.ingest_topics() yield from self.ingest_charts() yield from self.ingest_dashboards() - yield from self.ingest_tasks() yield from self.ingest_pipelines() yield from self.ingest_lineage() yield from self.ingest_users() @@ -436,25 +435,14 @@ class SampleDataSource(Source): yield task_ev def ingest_pipelines(self) -> Iterable[Dashboard]: - tasks = self.client.list_tasks("service") - task_dict = {} - for task in tasks: - task_dict[task.name] = task - for pipeline in self.pipelines["pipelines"]: - task_refs = [] - for task in pipeline["tasks"]: - if task in task_dict: - task_refs.append( - EntityReference(id=task_dict[task].id, type="task") - ) pipeline_ev = Pipeline( id=uuid.uuid4(), name=pipeline["name"], displayName=pipeline["displayName"], description=pipeline["description"], pipelineUrl=pipeline["pipelineUrl"], - tasks=task_refs, + tasks=pipeline["tasks"], service=EntityReference( id=self.pipeline_service.id, type="pipelineService" ),