yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-07 15:04:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): catch errors when profiling for sample values (#6194)

Browse Source
This commit is contained in:
Mayuri Nehate 2022-10-15 05:45:56 +05:30 committed by GitHub
parent 43f7b568e3
commit 3fc6364979
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py
View File

@ -460,16 +460,28 @@ class _SingleDatasetProfiler(BasicDatasetProfilerBase):
def _get_dataset_column_sample_values( def _get_dataset_column_sample_values(
self, column_profile: DatasetFieldProfileClass, column: str self, column_profile: DatasetFieldProfileClass, column: str
) -> None: ) -> None:
if self.config.include_field_sample_values: if not self.config.include_field_sample_values:
return
try:
# TODO do this without GE # TODO do this without GE
self.dataset.set_config_value("interactive_evaluation", True) self.dataset.set_config_value("interactive_evaluation", True)
res = self.dataset.expect_column_values_to_be_in_set( res = self.dataset.expect_column_values_to_be_in_set(
column, [], result_format="SUMMARY" column, [], result_format="SUMMARY"
).result ).result
column_profile.sampleValues = [ column_profile.sampleValues = [
str(v) for v in res["partial_unexpected_list"] str(v) for v in res["partial_unexpected_list"]
] ]
except Exception as e:
logger.debug(
f"Caught exception while attempting to get sample values for column {column}. {e}"
)
self.report.report_warning(
"Profiling - Unable to get column sample values",
f"{self.dataset_name}.{column}",
)
def generate_dataset_profile( # noqa: C901 (complexity) def generate_dataset_profile( # noqa: C901 (complexity)
self, self,