yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-03 04:10:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): s3 - add check for 0 rows in profiling (#5219)

Browse Source
This commit is contained in:
Jordan Wolinsky 2022-06-23 06:01:59 -04:00 committed by GitHub
parent 1086614129
commit 2be1e64cde
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
metadata-ingestion/src/datahub/ingestion/source/s3/profiling.py
View File

@ -273,7 +273,8 @@ class _SingleTableProfiler:
)
column_null_counts = null_counts.toPandas().T[0].to_dict()
column_null_fractions = {
c: column_null_counts[c] / self.row_count for c in self.columns_to_profile
c: column_null_counts[c] / self.row_count if self.row_count != 0 else 0
for c in self.columns_to_profile
}
column_nonnull_counts = {
c: self.row_count - column_null_counts[c] for c in self.columns_to_profile