yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-10 15:59:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix minor column data quality test bugs (#7111)

Browse Source 
* Fixed test name issue + filtered out partition details for non BQ tables

* Exclude non BQ table from partition processing

* Fixed test + formating
This commit is contained in:
Teddy 2022-09-01 13:47:00 +02:00 committed by GitHub
parent e180cb15af
commit 9dbcb3911b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 19 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
ingestion/src/metadata/orm_profiler/api/workflow.py
View File

@ -35,7 +35,10 @@ from metadata.generated.schema.entity.data.table import (
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
OpenMetadataConnection, OpenMetadataConnection,
) )
from metadata.generated.schema.entity.services.databaseService import DatabaseService from metadata.generated.schema.entity.services.databaseService import (
DatabaseService,
DatabaseServiceType,
)
from metadata.generated.schema.entity.services.serviceType import ServiceType from metadata.generated.schema.entity.services.serviceType import ServiceType
from metadata.generated.schema.metadataIngestion.databaseServiceProfilerPipeline import ( from metadata.generated.schema.metadataIngestion.databaseServiceProfilerPipeline import (
DatabaseServiceProfilerPipeline, DatabaseServiceProfilerPipeline,
@ -207,6 +210,9 @@ class ProfilerWorkflow:
Args: Args:
entity: table entity entity: table entity
""" """
# Should remove this with https://github.com/open-metadata/OpenMetadata/issues/5458
if entity.serviceType != DatabaseServiceType.BigQuery:
return None
entity_config: TableConfig = self.get_config_for_entity(entity) entity_config: TableConfig = self.get_config_for_entity(entity)
if entity_config: if entity_config:
return entity_config.partitionConfig return entity_config.partitionConfig

8
ingestion/src/metadata/test_suite/api/workflow.py
View File

@ -30,7 +30,10 @@ from metadata.generated.schema.entity.data.table import IntervalType, Table
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
OpenMetadataConnection, OpenMetadataConnection,
) )
from metadata.generated.schema.entity.services.databaseService import DatabaseService from metadata.generated.schema.entity.services.databaseService import (
DatabaseService,
DatabaseServiceType,
)
from metadata.generated.schema.metadataIngestion.testSuitePipeline import ( from metadata.generated.schema.metadataIngestion.testSuitePipeline import (
TestSuitePipeline, TestSuitePipeline,
) )
@ -195,6 +198,9 @@ class TestSuiteWorkflow:
Args: Args:
entity: table entity entity: table entity
""" """
# Should remove this with https://github.com/open-metadata/OpenMetadata/issues/5458
if entity.serviceType != DatabaseServiceType.BigQuery:
return None
if entity.tablePartition: if entity.tablePartition:
if entity.tablePartition.intervalType in { if entity.tablePartition.intervalType in {
IntervalType.TIME_UNIT, IntervalType.TIME_UNIT,

4
ingestion/src/metadata/test_suite/validations/core.py
View File

@ -133,11 +133,11 @@ validation_enum_registry.add("columnValueStdDevToBeBetween")(
# # Column Session Tests # # Column Session Tests
validation_enum_registry.add("columnValuesToBeNotInSet")(column_values_not_in_set) validation_enum_registry.add("columnValuesToBeNotInSet")(column_values_not_in_set)
validation_enum_registry.add("columnValuesToBeInSet")(column_values_in_set) validation_enum_registry.add("ColumnValuesToBeInSet")(column_values_in_set)
validation_enum_registry.add("columnValuesToMatchRegex")(column_values_to_match_regex) validation_enum_registry.add("columnValuesToMatchRegex")(column_values_to_match_regex)
validation_enum_registry.add("columnValuesToNotMatchRegex")( validation_enum_registry.add("columnValuesToNotMatchRegex")(
column_values_to_not_match_regex column_values_to_not_match_regex
) )
validation_enum_registry.add("columnValuesMissingCountToBeEqual")( validation_enum_registry.add("columnValuesMissingCount")(
column_values_missing_count_to_be_equal column_values_missing_count_to_be_equal
) )

6
ingestion/tests/unit/test_suite/test_validations.py
View File

@ -270,7 +270,7 @@ class testSuiteValidation(unittest.TestCase):
], ],
) )
res = validation_enum_registry.registry["columnValuesToBeInSet"]( res = validation_enum_registry.registry["ColumnValuesToBeInSet"](
test_case=test_case, test_case=test_case,
execution_date=EXECUTION_DATE.timestamp(), execution_date=EXECUTION_DATE.timestamp(),
runner=self.runner, runner=self.runner,
@ -292,7 +292,7 @@ class testSuiteValidation(unittest.TestCase):
], ],
) )
res = validation_enum_registry.registry["columnValuesMissingCountToBeEqual"]( res = validation_enum_registry.registry["columnValuesMissingCount"](
test_case=test_case, test_case=test_case,
execution_date=EXECUTION_DATE.timestamp(), execution_date=EXECUTION_DATE.timestamp(),
runner=self.runner, runner=self.runner,
@ -313,7 +313,7 @@ class testSuiteValidation(unittest.TestCase):
], ],
) )
res = validation_enum_registry.registry["columnValuesMissingCountToBeEqual"]( res = validation_enum_registry.registry["columnValuesMissingCount"](
test_case=test_case, test_case=test_case,
execution_date=EXECUTION_DATE.timestamp(), execution_date=EXECUTION_DATE.timestamp(),
runner=self.runner, runner=self.runner,