From 456f10ab634bba50c7304069ea0756e17cb5906f Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Sun, 14 Nov 2021 09:37:14 -0800 Subject: [PATCH] Fix #1180: Ingestion APIs: source type is not getting generated (#1181) --- .../catalog/ingestion/AirflowUtils.java | 14 ++++++++------ .../resources/operations/IngestionResource.java | 1 + .../api/operations/workflows/createIngestion.json | 3 +++ .../schema/operations/workflows/ingestion.json | 9 ++++++--- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/ingestion/AirflowUtils.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/ingestion/AirflowUtils.java index 3e7c6985582..bc5d7152751 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/ingestion/AirflowUtils.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/ingestion/AirflowUtils.java @@ -27,23 +27,25 @@ public final class AirflowUtils { private AirflowUtils() {} - public static OpenMetadataIngestionComponent makeDatasourceComponent(Ingestion ingestion) { + public static OpenMetadataIngestionComponent makeOpenMetadataDatasourceComponent(Ingestion ingestion) { Map dbConfig = new HashMap<>(); dbConfig.put("host_port", ingestion.getConnectorConfig().getHost()); dbConfig.put("username", ingestion.getConnectorConfig().getUsername()); dbConfig.put("password", ingestion.getConnectorConfig().getPassword()); dbConfig.put("database", ingestion.getConnectorConfig().getDatabase()); dbConfig.put("service_name", ingestion.getService().getName()); - dbConfig.put("filter_pattern", ingestion.getConnectorConfig().getIncludeFilterPattern()); + if (ingestion.getConnectorConfig().getIncludeFilterPattern() != null) { + dbConfig.put("filter_pattern", ingestion.getConnectorConfig().getIncludeFilterPattern()); + } return OpenMetadataIngestionComponent.builder() - .type(ingestion.getService().getType()) + .type(ingestion.getIngestionType().value()) .config(dbConfig).build(); } - public static OpenMetadataIngestionComponent makeMetadataSourceComponent(Ingestion ingestion) { + public static OpenMetadataIngestionComponent makeOpenMetadataSourceComponent(Ingestion ingestion) { Map dbConfig = new HashMap<>(); return OpenMetadataIngestionComponent.builder() - .type("metadata") + .type(ingestion.getIngestionType().value()) .config(dbConfig) .build(); } @@ -76,7 +78,7 @@ public final class AirflowUtils { public static OpenMetadataIngestionConfig buildDatabaseIngestion(Ingestion ingestion, AirflowConfiguration airflowConfiguration) { return OpenMetadataIngestionConfig.builder() - .source(makeDatasourceComponent(ingestion)) + .source(makeOpenMetadataDatasourceComponent(ingestion)) .sink(makeOpenMetadataSinkComponent(ingestion)) .metadataServer(makeOpenMetadataConfigComponent(ingestion, airflowConfiguration)).build(); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/IngestionResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/IngestionResource.java index 367ef73bfab..ede98f1aea5 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/IngestionResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/IngestionResource.java @@ -365,6 +365,7 @@ public class IngestionResource { return new Ingestion().withId(UUID.randomUUID()).withName(create.getName()) .withDisplayName(create.getDisplayName()) .withDescription(create.getDescription()) + .withIngestionType(create.getIngestionType()) .withForceDeploy(create.getForceDeploy()) .withConcurrency(create.getConcurrency()) .withPauseWorkflow(create.getPauseWorkflow()) diff --git a/catalog-rest-service/src/main/resources/json/schema/api/operations/workflows/createIngestion.json b/catalog-rest-service/src/main/resources/json/schema/api/operations/workflows/createIngestion.json index db95df57819..4cf25bf0bbe 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/operations/workflows/createIngestion.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/operations/workflows/createIngestion.json @@ -19,6 +19,9 @@ "description": "Description of the workflow.", "type": "string" }, + "ingestionType": { + "$ref": "../../../operations/workflows/ingestion.json#/definitions/ingestionType" + }, "owner": { "description": "Owner of this Ingestion.", "$ref": "../../../type/entityReference.json", diff --git a/catalog-rest-service/src/main/resources/json/schema/operations/workflows/ingestion.json b/catalog-rest-service/src/main/resources/json/schema/operations/workflows/ingestion.json index eed1d934432..f2ef86c06fd 100644 --- a/catalog-rest-service/src/main/resources/json/schema/operations/workflows/ingestion.json +++ b/catalog-rest-service/src/main/resources/json/schema/operations/workflows/ingestion.json @@ -27,19 +27,19 @@ "name": "bigquery" }, { - "name": "bigquery-usage" + "name": "bigquery_usage" }, { "name": "redshift" }, { - "name": "redshift-usage" + "name": "redshift_usage" }, { "name": "snowflake" }, { - "name": "snowflake-usage" + "name": "snowflake_usage" }, { "name": "hive" @@ -171,6 +171,9 @@ "minLength": 1, "maxLength": 128 }, + "ingestionType": { + "$ref": "#/definitions/ingestionType" + }, "tags": { "description": "Tags associated with the Ingestion.", "type": "array",