From 706d1ab97e91eaf66cd8d45406ded734f97223df Mon Sep 17 00:00:00 2001 From: Imri Paran Date: Mon, 15 Apr 2024 07:59:27 +0200 Subject: [PATCH] fixed ingestion of sample data for failed sample rows (#15879) --- .../sample_data/tests/testCaseResults.json | 62 ++++++++++++++++++- .../sample_data/tests/testSuites.json | 62 +------------------ .../ingestion/source/database/sample_data.py | 20 +++--- 3 files changed, 70 insertions(+), 74 deletions(-) diff --git a/ingestion/examples/sample_data/tests/testCaseResults.json b/ingestion/examples/sample_data/tests/testCaseResults.json index 06b3d29d8a0..c55d10c95ce 100644 --- a/ingestion/examples/sample_data/tests/testCaseResults.json +++ b/ingestion/examples/sample_data/tests/testCaseResults.json @@ -594,7 +594,67 @@ } ] } - ] + ], + "failedRowsSample": { + "columns": [ + "address_id", + "shop_id", + "first_name", + "last_name", + "address1", + "address2", + "company", + "city", + "region", + "zip", + "country", + "phone" + ], + "rows": [ + [ + "bc35100e-2da5-48bb-bfc8-667dafe66532", + "70424951-bc97-4b20-9ce7-be37c4619361", + "Zachary", + "Brett", + "9054 Maria Circle Apt. 296", + "48348 Victoria Valleys Suite 144", + "Robinson Inc", + "Stephanieport", + "048 Moore Turnpike Apt. 061", + "1001", + "Latvia", + "(381)575-6692" + ], + [ + "facf92d7-05ea-43d2-ba2a-067d63dee60c", + "a8d30187-1409-4606-9259-322a4f6caf74", + "Amber", + "Albert", + "3170 Warren Orchard Apt. 834", + "3204 Brewer Shoal Suite 324", + "Davila-Snyder", + "Nicoleland", + "023 Paul Course", + "1002", + "Sweden", + "438-959-1151" + ], + [ + "bab9a506-e23d-4c53-9402-d070e7704376", + "e02e1fac-b650-4db8-8c9d-5fa5edf5d863", + "Heidi", + "Kelly", + "30942 Gonzalez Stravenue", + "3158 Watts Green", + "Moore PLC", + "West Erica", + "6294 Elliott Ville", + "2789", + "Saint Martin", + "(830)112-9566x8681" + ] + ] + } } ] } \ No newline at end of file diff --git a/ingestion/examples/sample_data/tests/testSuites.json b/ingestion/examples/sample_data/tests/testSuites.json index cfd2e6d7b8c..08a3dbcb96b 100644 --- a/ingestion/examples/sample_data/tests/testSuites.json +++ b/ingestion/examples/sample_data/tests/testSuites.json @@ -339,67 +339,7 @@ "value": 96162 } ], - "resolutions": {}, - "sampleFailedRows": { - "columns": [ - "address_id", - "shop_id", - "first_name", - "last_name", - "address1", - "address2", - "company", - "city", - "region", - "zip", - "country", - "phone" - ], - "rows": [ - [ - "bc35100e-2da5-48bb-bfc8-667dafe66532", - "70424951-bc97-4b20-9ce7-be37c4619361", - "Zachary", - "Brett", - "9054 Maria Circle Apt. 296", - "48348 Victoria Valleys Suite 144", - "Robinson Inc", - "Stephanieport", - "048 Moore Turnpike Apt. 061", - "1001", - "Latvia", - "(381)575-6692" - ], - [ - "facf92d7-05ea-43d2-ba2a-067d63dee60c", - "a8d30187-1409-4606-9259-322a4f6caf74", - "Amber", - "Albert", - "3170 Warren Orchard Apt. 834", - "3204 Brewer Shoal Suite 324", - "Davila-Snyder", - "Nicoleland", - "023 Paul Course", - "1002", - "Sweden", - "438-959-1151" - ], - [ - "bab9a506-e23d-4c53-9402-d070e7704376", - "e02e1fac-b650-4db8-8c9d-5fa5edf5d863", - "Heidi", - "Kelly", - "30942 Gonzalez Stravenue", - "3158 Watts Green", - "Moore PLC", - "West Erica", - "6294 Elliott Ville", - "2789", - "Saint Martin", - "(830)112-9566x8681" - ] - ] - } + "resolutions": {} } ] } diff --git a/ingestion/src/metadata/ingestion/source/database/sample_data.py b/ingestion/src/metadata/ingestion/source/database/sample_data.py index 29b5e255a88..f5255f03586 100644 --- a/ingestion/src/metadata/ingestion/source/database/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/database/sample_data.py @@ -1410,18 +1410,6 @@ class SampleDataSource( ) # type: ignore ) yield Either(right=test_case_req) - if test_case.get("sampleFailedRows"): - test_case_entity = self.metadata.get_or_create_test_case( - test_case_fqn=f"{entity_link.get_table_or_column_fqn(test_case['entityLink'])}.{test_case['name']}", - ) - - self.metadata.ingest_failed_rows_sample( - test_case_entity, - TableData( - rows=test_case["sampleFailedRows"]["rows"], - columns=test_case["sampleFailedRows"]["columns"], - ), - ) def ingest_incidents(self) -> Iterable[Either[OMetaTestCaseResolutionStatus]]: """ @@ -1505,6 +1493,14 @@ class SampleDataSource( test_case_name=case.fullyQualifiedName.__root__, ) yield Either(right=test_case_result_req) + if test_case_results.get("failedRowsSample"): + self.metadata.ingest_failed_rows_sample( + case, + TableData( + rows=test_case_results["failedRowsSample"]["rows"], + columns=test_case_results["failedRowsSample"]["columns"], + ), + ) def ingest_data_insights(self) -> Iterable[Either[OMetaDataInsightSample]]: """Iterate over all the data insights and ingest them"""