yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-12 00:41:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixed ingestion of sample data for failed sample rows (#15879)

Browse Source
This commit is contained in:
Imri Paran 2024-04-15 07:59:27 +02:00 committed by GitHub
parent e91f846f5c
commit 706d1ab97e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 70 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
ingestion/examples/sample_data/tests/testCaseResults.json
View File

@ -594,7 +594,67 @@
} }
] ]
} }
] ],
"failedRowsSample": {
"columns": [
"address_id",
"shop_id",
"first_name",
"last_name",
"address1",
"address2",
"company",
"city",
"region",
"zip",
"country",
"phone"
],
"rows": [
[
"bc35100e-2da5-48bb-bfc8-667dafe66532",
"70424951-bc97-4b20-9ce7-be37c4619361",
"Zachary",
"Brett",
"9054 Maria Circle Apt. 296",
"48348 Victoria Valleys Suite 144",
"Robinson Inc",
"Stephanieport",
"048 Moore Turnpike Apt. 061",
"1001",
"Latvia",
"(381)575-6692"
],
[
"facf92d7-05ea-43d2-ba2a-067d63dee60c",
"a8d30187-1409-4606-9259-322a4f6caf74",
"Amber",
"Albert",
"3170 Warren Orchard Apt. 834",
"3204 Brewer Shoal Suite 324",
"Davila-Snyder",
"Nicoleland",
"023 Paul Course",
"1002",
"Sweden",
"438-959-1151"
],
[
"bab9a506-e23d-4c53-9402-d070e7704376",
"e02e1fac-b650-4db8-8c9d-5fa5edf5d863",
"Heidi",
"Kelly",
"30942 Gonzalez Stravenue",
"3158 Watts Green",
"Moore PLC",
"West Erica",
"6294 Elliott Ville",
"2789",
"Saint Martin",
"(830)112-9566x8681"
]
]
}
} }
] ]
} }

62
ingestion/examples/sample_data/tests/testSuites.json
View File

@ -339,67 +339,7 @@
"value": 96162 "value": 96162
} }
], ],
"resolutions": {}, "resolutions": {}
"sampleFailedRows": {
"columns": [
"address_id",
"shop_id",
"first_name",
"last_name",
"address1",
"address2",
"company",
"city",
"region",
"zip",
"country",
"phone"
],
"rows": [
[
"bc35100e-2da5-48bb-bfc8-667dafe66532",
"70424951-bc97-4b20-9ce7-be37c4619361",
"Zachary",
"Brett",
"9054 Maria Circle Apt. 296",
"48348 Victoria Valleys Suite 144",
"Robinson Inc",
"Stephanieport",
"048 Moore Turnpike Apt. 061",
"1001",
"Latvia",
"(381)575-6692"
],
[
"facf92d7-05ea-43d2-ba2a-067d63dee60c",
"a8d30187-1409-4606-9259-322a4f6caf74",
"Amber",
"Albert",
"3170 Warren Orchard Apt. 834",
"3204 Brewer Shoal Suite 324",
"Davila-Snyder",
"Nicoleland",
"023 Paul Course",
"1002",
"Sweden",
"438-959-1151"
],
[
"bab9a506-e23d-4c53-9402-d070e7704376",
"e02e1fac-b650-4db8-8c9d-5fa5edf5d863",
"Heidi",
"Kelly",
"30942 Gonzalez Stravenue",
"3158 Watts Green",
"Moore PLC",
"West Erica",
"6294 Elliott Ville",
"2789",
"Saint Martin",
"(830)112-9566x8681"
]
]
}
} }
] ]
} }

20
ingestion/src/metadata/ingestion/source/database/sample_data.py
View File

@ -1410,18 +1410,6 @@ class SampleDataSource(
) # type: ignore ) # type: ignore
) )
yield Either(right=test_case_req) yield Either(right=test_case_req)
if test_case.get("sampleFailedRows"):
test_case_entity = self.metadata.get_or_create_test_case(
test_case_fqn=f"{entity_link.get_table_or_column_fqn(test_case['entityLink'])}.{test_case['name']}",
)
self.metadata.ingest_failed_rows_sample(
test_case_entity,
TableData(
rows=test_case["sampleFailedRows"]["rows"],
columns=test_case["sampleFailedRows"]["columns"],
),
)
def ingest_incidents(self) -> Iterable[Either[OMetaTestCaseResolutionStatus]]: def ingest_incidents(self) -> Iterable[Either[OMetaTestCaseResolutionStatus]]:
""" """
@ -1505,6 +1493,14 @@ class SampleDataSource(
test_case_name=case.fullyQualifiedName.__root__, test_case_name=case.fullyQualifiedName.__root__,
) )
yield Either(right=test_case_result_req) yield Either(right=test_case_result_req)
if test_case_results.get("failedRowsSample"):
self.metadata.ingest_failed_rows_sample(
case,
TableData(
rows=test_case_results["failedRowsSample"]["rows"],
columns=test_case_results["failedRowsSample"]["columns"],
),
)
def ingest_data_insights(self) -> Iterable[Either[OMetaDataInsightSample]]: def ingest_data_insights(self) -> Iterable[Either[OMetaDataInsightSample]]:
"""Iterate over all the data insights and ingest them""" """Iterate over all the data insights and ingest them"""