yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-01 21:57:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

usage fixes (#5805)

Browse Source 
Co-authored-by: Onkar Ravgan <onkarravgan@Onkars-MacBook-Pro.local>
This commit is contained in:
Onkar Ravgan 2022-06-30 21:33:01 +05:30 committed by GitHub
parent cb0fd3d811
commit 380fdf18cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
ingestion/src/metadata/ingestion/stage/table_usage.py
View File

@ -11,6 +11,7 @@
import json import json
import os import os
import shutil
import traceback import traceback
from metadata.config.common import ConfigModel from metadata.config.common import ConfigModel
@ -47,6 +48,9 @@ class TableUsageStage(Stage[QueryParserData]):
isdir = os.path.isdir(self.config.filename) isdir = os.path.isdir(self.config.filename)
if not isdir: if not isdir:
os.mkdir(self.config.filename) os.mkdir(self.config.filename)
else:
shutil.rmtree(self.config.filename)
os.mkdir(self.config.filename)
self.wrote_something = False self.wrote_something = False
@classmethod @classmethod
@ -55,7 +59,7 @@ class TableUsageStage(Stage[QueryParserData]):
return cls(config, metadata_config) return cls(config, metadata_config)
def _add_sql_query(self, record, table): def _add_sql_query(self, record, table):
if self.table_queries.get(table): if self.table_queries.get((table, record.date)):
self.table_queries[(table, record.date)].append(SqlQuery(query=record.sql)) self.table_queries[(table, record.date)].append(SqlQuery(query=record.sql))
else: else:
self.table_queries[(table, record.date)] = [SqlQuery(query=record.sql)] self.table_queries[(table, record.date)] = [SqlQuery(query=record.sql)]
@ -64,6 +68,7 @@ class TableUsageStage(Stage[QueryParserData]):
if not data or not data.parsedData: if not data or not data.parsedData:
return return
self.table_usage = {} self.table_usage = {}
self.table_queries = {}
for record in data.parsedData: for record in data.parsedData:
if record is None: if record is None:
continue continue