From c44b0bbcea56bc7c6cde2265ed08361d912fbefd Mon Sep 17 00:00:00 2001 From: NiharDoshi99 <51595473+NiharDoshi99@users.noreply.github.com> Date: Wed, 5 Apr 2023 16:03:30 +0530 Subject: [PATCH] adding timeout option for Dagster and Amundsen changes (#10934) * adding timeout option for dagster and amundsen changes * Update ingestion/src/metadata/ingestion/source/pipeline/dagster/connection.py Co-authored-by: Mayur Singal <39544459+ulixius9@users.noreply.github.com> --------- Co-authored-by: Mayur Singal <39544459+ulixius9@users.noreply.github.com> --- .../metadata/ingestion/source/metadata/amundsen/metadata.py | 2 +- .../ingestion/source/pipeline/dagster/connection.py | 1 + .../services/connections/pipeline/dagsterConnection.json | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ingestion/src/metadata/ingestion/source/metadata/amundsen/metadata.py b/ingestion/src/metadata/ingestion/source/metadata/amundsen/metadata.py index cc5c670e5be..8bd44a6cb20 100644 --- a/ingestion/src/metadata/ingestion/source/metadata/amundsen/metadata.py +++ b/ingestion/src/metadata/ingestion/source/metadata/amundsen/metadata.py @@ -216,7 +216,7 @@ class AmundsenSource(Source[Entity]): name=table_entity.name, tableType=table_entity.tableType, description=table_entity.description, - databaseSchema=table_entity.databaseSchema, + databaseSchema=table_entity.databaseSchema.fullyQualifiedName, tags=table_entity.tags, columns=table_entity.columns, owner=user_entity_ref, diff --git a/ingestion/src/metadata/ingestion/source/pipeline/dagster/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/dagster/connection.py index fc2d26f77c9..36938709b18 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/dagster/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/dagster/connection.py @@ -40,6 +40,7 @@ def get_connection(connection: DagsterConnection) -> DagsterGraphQLClient: headers={"Dagster-Cloud-Api-Token": connection.token.get_secret_value()} if connection.token else None, + timeout=connection.timeout, ), ) diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/dagsterConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/dagsterConnection.json index 6428015d092..a116fa07725 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/dagsterConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/dagsterConnection.json @@ -32,6 +32,12 @@ "type": "string", "format": "password" }, + "timeout": { + "title": "Time Out", + "description": "Connection Time Limit Between OM and Dagster Graphql API in second", + "type": "integer", + "default": "1000" + }, "supportsMetadataExtraction": { "title": "Supports Metadata Extraction", "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction"