yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-01 13:48:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #3819 & #3821 - Minor JSON Schema fixes (#3872)

Browse Source 
Fix #3819 & #3821 - Minor JSON Schema fixes (#3872)
This commit is contained in:
Pere Miquel Brull 2022-04-06 09:11:39 +02:00 committed by GitHub
parent 954bbca383
commit b93c4611bb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 22 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json
View File

@ -46,7 +46,7 @@
"enablePolicyTagImport": { "enablePolicyTagImport": {
"description": "Enable importing policy tags of BigQuery into OpenMetadata", "description": "Enable importing policy tags of BigQuery into OpenMetadata",
"type": "boolean", "type": "boolean",
"default": "true" "default": true
}, },
"tagCategoryName": { "tagCategoryName": {
"description": "OpenMetadata Tag category name if enablePolicyTagImport is set to true.", "description": "OpenMetadata Tag category name if enablePolicyTagImport is set to true.",

6
catalog-rest-service/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json
View File

@ -19,12 +19,12 @@
"forceDeploy": { "forceDeploy": {
"description": "Deploy the pipeline by overwriting existing pipeline with the same name.", "description": "Deploy the pipeline by overwriting existing pipeline with the same name.",
"type": "boolean", "type": "boolean",
"default": "false" "default": false
}, },
"pausePipeline": { "pausePipeline": {
"description": "pause the pipeline from running once the deploy is finished successfully.", "description": "pause the pipeline from running once the deploy is finished successfully.",
"type": "boolean", "type": "boolean",
"default": "false" "default": false
}, },
"concurrency": { "concurrency": {
"description": "Concurrency of the Pipeline.", "description": "Concurrency of the Pipeline.",
@ -61,7 +61,7 @@
"pipelineCatchup": { "pipelineCatchup": {
"description": "Run past executions if the start date is in the past.", "description": "Run past executions if the start date is in the past.",
"type": "boolean", "type": "boolean",
"default": "false" "default": false
}, },
"scheduleInterval": { "scheduleInterval": {
"description": "Scheduler Interval for the pipeline in cron format.", "description": "Scheduler Interval for the pipeline in cron format.",

8
catalog-rest-service/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json
View File

@ -7,17 +7,17 @@
"markDeletedTables": { "markDeletedTables": {
"description": "Optional configuration to soft delete tables in OpenMetadata if the source tables are deleted.", "description": "Optional configuration to soft delete tables in OpenMetadata if the source tables are deleted.",
"type": "boolean", "type": "boolean",
"default": "true" "default": true
}, },
"includeViews": { "includeViews": {
"description": "Optional configuration to turn off fetching metadata for views.", "description": "Optional configuration to turn off fetching metadata for views.",
"type": "boolean", "type": "boolean",
"default": "true" "default": true
}, },
"generateSampleData": { "generateSampleData": {
"description": "Option to turn on/off generating sample data during metadata extraction.", "description": "Option to turn on/off generating sample data during metadata extraction.",
"type": "boolean", "type": "boolean",
"default": "true" "default": true
}, },
"sampleDataQuery": { "sampleDataQuery": {
"description": "Sample data extraction query.", "description": "Sample data extraction query.",
@ -27,7 +27,7 @@
"enableDataProfiler": { "enableDataProfiler": {
"description": "Run data profiler as part of this metadata ingestion to get table profile data.", "description": "Run data profiler as part of this metadata ingestion to get table profile data.",
"type": "boolean", "type": "boolean",
"default": "false" "default": false
}, },
"schemaFilterPattern": { "schemaFilterPattern": {
"description": "Regex to only fetch tables or databases that matches the pattern.", "description": "Regex to only fetch tables or databases that matches the pattern.",

3
catalog-rest-service/src/main/resources/json/schema/metadataIngestion/workflow.json
View File

@ -113,6 +113,7 @@
}, },
"sourceConfig": { "sourceConfig": {
"description": "Additional connection configuration.", "description": "Additional connection configuration.",
"javaType": "org.openmetadata.catalog.metadataIngestion.SourceConfig",
"type": "object", "type": "object",
"properties": { "properties": {
"config": { "config": {
@ -275,7 +276,7 @@
"additionalProperties": false "additionalProperties": false
} }
}, },
"anyOf": [ "oneOf": [
{ {
"required": ["name", "openMetadataWorkflowConfig", "source", "sink"] "required": ["name", "openMetadataWorkflowConfig", "source", "sink"]
}, },

8
catalog-rest-service/src/main/resources/json/schema/type/databaseConnectionConfig.json
View File

@ -29,17 +29,17 @@
"includeViews": { "includeViews": {
"description": "optional configuration to turn off fetching metadata for views.", "description": "optional configuration to turn off fetching metadata for views.",
"type": "boolean", "type": "boolean",
"default": "true" "default": true
}, },
"includeTables": { "includeTables": {
"description": "Optional configuration to turn off fetching metadata for tables.", "description": "Optional configuration to turn off fetching metadata for tables.",
"type": "boolean", "type": "boolean",
"default": "true" "default": true
}, },
"generateSampleData": { "generateSampleData": {
"description": "Turn on/off collecting sample data.", "description": "Turn on/off collecting sample data.",
"type": "boolean", "type": "boolean",
"default": "true" "default": true
}, },
"sampleDataQuery": { "sampleDataQuery": {
"description": "query to generate sample data.", "description": "query to generate sample data.",
@ -49,7 +49,7 @@
"enableDataProfiler": { "enableDataProfiler": {
"description": "Run data profiler as part of ingestion to get table profile data.", "description": "Run data profiler as part of ingestion to get table profile data.",
"type": "boolean", "type": "boolean",
"default": "false" "default": false
}, },
"includeFilterPattern": { "includeFilterPattern": {
"description": "Regex to only fetch tables or databases that matches the pattern.", "description": "Regex to only fetch tables or databases that matches the pattern.",

4
catalog-rest-service/src/main/resources/json/schema/type/filterPattern.json
View File

@ -1,13 +1,13 @@
{ {
"$id": "https://open-metadata.org/schema/type/filterPattern.json", "$id": "https://open-metadata.org/schema/type/filterPattern.json",
"$schema": "http://json-schema.org/draft-07/schema#", "$schema": "http://json-schema.org/draft-07/schema#",
"title": "MetadataWorkflow", "title": "FilterPattern",
"description": "OpenMetadata Ingestion Framework definition.", "description": "OpenMetadata Ingestion Framework definition.",
"type": "object", "type": "object",
"definitions": { "definitions": {
"filterPattern": { "filterPattern": {
"type": "object", "type": "object",
"javaType": "org.openmetadata.catalog.metadataingestion.FilterPattern", "javaType": "org.openmetadata.catalog.metadataIngestion.FilterPattern",
"description": "Regex to only fetch dashboards or charts that matches the pattern.", "description": "Regex to only fetch dashboards or charts that matches the pattern.",
"properties": { "properties": {
"includes": { "includes": {

4
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/DatabaseServiceResourceTest.java
View File

@ -36,8 +36,8 @@ import org.openmetadata.catalog.entity.services.DatabaseService;
import org.openmetadata.catalog.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.catalog.entity.services.ingestionPipelines.IngestionPipeline;
import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface; import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface;
import org.openmetadata.catalog.metadataIngestion.DatabaseServiceMetadataPipeline; import org.openmetadata.catalog.metadataIngestion.DatabaseServiceMetadataPipeline;
import org.openmetadata.catalog.metadataingestion.FilterPattern; import org.openmetadata.catalog.metadataIngestion.FilterPattern;
import org.openmetadata.catalog.metadataingestion.SourceConfig; import org.openmetadata.catalog.metadataIngestion.SourceConfig;
import org.openmetadata.catalog.resources.EntityResourceTest; import org.openmetadata.catalog.resources.EntityResourceTest;
import org.openmetadata.catalog.resources.services.database.DatabaseServiceResource.DatabaseServiceList; import org.openmetadata.catalog.resources.services.database.DatabaseServiceResource.DatabaseServiceList;
import org.openmetadata.catalog.resources.services.ingestionpipelines.IngestionPipelineResourceTest; import org.openmetadata.catalog.resources.services.ingestionpipelines.IngestionPipelineResourceTest;

8
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResourceTest.java
View File

@ -45,7 +45,6 @@ import org.openmetadata.catalog.airflow.AirflowConfiguration;
import org.openmetadata.catalog.api.services.CreateDatabaseService; import org.openmetadata.catalog.api.services.CreateDatabaseService;
import org.openmetadata.catalog.api.services.DatabaseConnection; import org.openmetadata.catalog.api.services.DatabaseConnection;
import org.openmetadata.catalog.api.services.ingestionPipelines.CreateIngestionPipeline; import org.openmetadata.catalog.api.services.ingestionPipelines.CreateIngestionPipeline;
import org.openmetadata.catalog.api.services.ingestionPipelines.SourceConfig;
import org.openmetadata.catalog.entity.services.DatabaseService; import org.openmetadata.catalog.entity.services.DatabaseService;
import org.openmetadata.catalog.entity.services.ingestionPipelines.AirflowConfig; import org.openmetadata.catalog.entity.services.ingestionPipelines.AirflowConfig;
import org.openmetadata.catalog.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.catalog.entity.services.ingestionPipelines.IngestionPipeline;
@ -55,8 +54,9 @@ import org.openmetadata.catalog.jdbi3.IngestionPipelineRepository;
import org.openmetadata.catalog.metadataIngestion.DashboardServiceMetadataPipeline; import org.openmetadata.catalog.metadataIngestion.DashboardServiceMetadataPipeline;
import org.openmetadata.catalog.metadataIngestion.DatabaseServiceMetadataPipeline; import org.openmetadata.catalog.metadataIngestion.DatabaseServiceMetadataPipeline;
import org.openmetadata.catalog.metadataIngestion.DatabaseServiceQueryUsagePipeline; import org.openmetadata.catalog.metadataIngestion.DatabaseServiceQueryUsagePipeline;
import org.openmetadata.catalog.metadataIngestion.FilterPattern;
import org.openmetadata.catalog.metadataIngestion.MessagingServiceMetadataPipeline; import org.openmetadata.catalog.metadataIngestion.MessagingServiceMetadataPipeline;
import org.openmetadata.catalog.metadataingestion.FilterPattern; import org.openmetadata.catalog.metadataIngestion.SourceConfig;
import org.openmetadata.catalog.resources.EntityResourceTest; import org.openmetadata.catalog.resources.EntityResourceTest;
import org.openmetadata.catalog.resources.services.DatabaseServiceResourceTest; import org.openmetadata.catalog.resources.services.DatabaseServiceResourceTest;
import org.openmetadata.catalog.services.connections.database.BigQueryConnection; import org.openmetadata.catalog.services.connections.database.BigQueryConnection;
@ -370,7 +370,7 @@ public class IngestionPipelineResourceTest extends EntityResourceTest<IngestionP
.withStartDate(startDate.toString())), .withStartDate(startDate.toString())),
OK, OK,
ADMIN_AUTH_HEADERS); ADMIN_AUTH_HEADERS);
String expectedFQN = EntityUtil.getFQN(SUPERSET_REFERENCE.getName(), ingestion.getName()); String expectedFQN = FullyQualifiedName.build(SUPERSET_REFERENCE.getName(), ingestion.getName());
assertEquals(startDate.toString(), ingestion.getAirflowConfig().getStartDate()); assertEquals(startDate.toString(), ingestion.getAirflowConfig().getStartDate());
assertEquals(pipelineConcurrency, ingestion.getAirflowConfig().getConcurrency()); assertEquals(pipelineConcurrency, ingestion.getAirflowConfig().getConcurrency());
assertEquals(expectedFQN, ingestion.getFullyQualifiedName()); assertEquals(expectedFQN, ingestion.getFullyQualifiedName());
@ -424,7 +424,7 @@ public class IngestionPipelineResourceTest extends EntityResourceTest<IngestionP
.withStartDate(startDate.toString())), .withStartDate(startDate.toString())),
OK, OK,
ADMIN_AUTH_HEADERS); ADMIN_AUTH_HEADERS);
String expectedFQN = EntityUtil.getFQN(KAFKA_REFERENCE.getName(), ingestion.getName()); String expectedFQN = FullyQualifiedName.build(KAFKA_REFERENCE.getName(), ingestion.getName());
assertEquals(startDate.toString(), ingestion.getAirflowConfig().getStartDate()); assertEquals(startDate.toString(), ingestion.getAirflowConfig().getStartDate());
assertEquals(pipelineConcurrency, ingestion.getAirflowConfig().getConcurrency()); assertEquals(pipelineConcurrency, ingestion.getAirflowConfig().getConcurrency());
assertEquals(expectedFQN, ingestion.getFullyQualifiedName()); assertEquals(expectedFQN, ingestion.getFullyQualifiedName());