yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-01 01:56:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #1180: Ingestion APIs: source type is not getting generated (#1181)

Browse Source
This commit is contained in:
Sriharsha Chintalapani 2021-11-14 09:37:14 -08:00 committed by GitHub
parent f5a7c193e3
commit 456f10ab63
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 18 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
catalog-rest-service/src/main/java/org/openmetadata/catalog/ingestion/AirflowUtils.java
View File

@ -27,23 +27,25 @@ public final class AirflowUtils {
private AirflowUtils() {}
public static OpenMetadataIngestionComponent makeDatasourceComponent(Ingestion ingestion) {
public static OpenMetadataIngestionComponent makeOpenMetadataDatasourceComponent(Ingestion ingestion) {
Map<String, Object> dbConfig = new HashMap<>();
dbConfig.put("host_port", ingestion.getConnectorConfig().getHost());
dbConfig.put("username", ingestion.getConnectorConfig().getUsername());
dbConfig.put("password", ingestion.getConnectorConfig().getPassword());
dbConfig.put("database", ingestion.getConnectorConfig().getDatabase());
dbConfig.put("service_name", ingestion.getService().getName());
dbConfig.put("filter_pattern", ingestion.getConnectorConfig().getIncludeFilterPattern());
if (ingestion.getConnectorConfig().getIncludeFilterPattern() != null) {
dbConfig.put("filter_pattern", ingestion.getConnectorConfig().getIncludeFilterPattern());
}
return OpenMetadataIngestionComponent.builder()
.type(ingestion.getService().getType())
.type(ingestion.getIngestionType().value())
.config(dbConfig).build();
}
public static OpenMetadataIngestionComponent makeMetadataSourceComponent(Ingestion ingestion) {
public static OpenMetadataIngestionComponent makeOpenMetadataSourceComponent(Ingestion ingestion) {
Map<String, Object> dbConfig = new HashMap<>();
return OpenMetadataIngestionComponent.builder()
.type("metadata")
.type(ingestion.getIngestionType().value())
.config(dbConfig)
.build();
}
@ -76,7 +78,7 @@ public final class AirflowUtils {
public static OpenMetadataIngestionConfig buildDatabaseIngestion(Ingestion ingestion,
AirflowConfiguration airflowConfiguration) {
return OpenMetadataIngestionConfig.builder()
.source(makeDatasourceComponent(ingestion))
.source(makeOpenMetadataDatasourceComponent(ingestion))
.sink(makeOpenMetadataSinkComponent(ingestion))
.metadataServer(makeOpenMetadataConfigComponent(ingestion, airflowConfiguration)).build();

1
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/IngestionResource.java
View File

@ -365,6 +365,7 @@ public class IngestionResource {
return new Ingestion().withId(UUID.randomUUID()).withName(create.getName())
.withDisplayName(create.getDisplayName())
.withDescription(create.getDescription())
.withIngestionType(create.getIngestionType())
.withForceDeploy(create.getForceDeploy())
.withConcurrency(create.getConcurrency())
.withPauseWorkflow(create.getPauseWorkflow())

3
catalog-rest-service/src/main/resources/json/schema/api/operations/workflows/createIngestion.json
View File

@ -19,6 +19,9 @@
"description": "Description of the workflow.",
"type": "string"
},
"ingestionType": {
"$ref": "../../../operations/workflows/ingestion.json#/definitions/ingestionType"
},
"owner": {
"description": "Owner of this Ingestion.",
"$ref": "../../../type/entityReference.json",

9
catalog-rest-service/src/main/resources/json/schema/operations/workflows/ingestion.json
View File

@ -27,19 +27,19 @@
"name": "bigquery"
},
{
"name": "bigquery-usage"
"name": "bigquery_usage"
},
{
"name": "redshift"
},
{
"name": "redshift-usage"
"name": "redshift_usage"
},
{
"name": "snowflake"
},
{
"name": "snowflake-usage"
"name": "snowflake_usage"
},
{
"name": "hive"
@ -171,6 +171,9 @@
"minLength": 1,
"maxLength": 128
},
"ingestionType": {
"$ref": "#/definitions/ingestionType"
},
"tags": {
"description": "Tags associated with the Ingestion.",
"type": "array",