yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-26 17:34:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix: Filter Datalake empty files (#8677)

Browse Source
This commit is contained in:
Milan Bariya 2022-11-13 22:09:29 +05:30 committed by GitHub
parent a20945e13f
commit ebfb872e16
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 15 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ingestion/src/metadata/ingestion/source/database/datalake.py
View File

@ -301,6 +301,7 @@ class DatalakeSource(DatabaseServiceSource):
data_frame = self.get_gcs_files(key=table_name, bucket_name=schema_name)
if isinstance(self.service_connection.configSource, S3Config):
data_frame = self.get_s3_files(key=table_name, bucket_name=schema_name)
if not data_frame.empty:
columns = self.get_columns(data_frame)
table_request = CreateTableRequest(
name=table_name,