From 380fdf18cd94e1378d2cc204c87b4597adab95aa Mon Sep 17 00:00:00 2001 From: Onkar Ravgan Date: Thu, 30 Jun 2022 21:33:01 +0530 Subject: [PATCH] usage fixes (#5805) Co-authored-by: Onkar Ravgan --- ingestion/src/metadata/ingestion/stage/table_usage.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ingestion/src/metadata/ingestion/stage/table_usage.py b/ingestion/src/metadata/ingestion/stage/table_usage.py index a326221c544..40a81f1a592 100644 --- a/ingestion/src/metadata/ingestion/stage/table_usage.py +++ b/ingestion/src/metadata/ingestion/stage/table_usage.py @@ -11,6 +11,7 @@ import json import os +import shutil import traceback from metadata.config.common import ConfigModel @@ -47,6 +48,9 @@ class TableUsageStage(Stage[QueryParserData]): isdir = os.path.isdir(self.config.filename) if not isdir: os.mkdir(self.config.filename) + else: + shutil.rmtree(self.config.filename) + os.mkdir(self.config.filename) self.wrote_something = False @classmethod @@ -55,7 +59,7 @@ class TableUsageStage(Stage[QueryParserData]): return cls(config, metadata_config) def _add_sql_query(self, record, table): - if self.table_queries.get(table): + if self.table_queries.get((table, record.date)): self.table_queries[(table, record.date)].append(SqlQuery(query=record.sql)) else: self.table_queries[(table, record.date)] = [SqlQuery(query=record.sql)] @@ -64,6 +68,7 @@ class TableUsageStage(Stage[QueryParserData]): if not data or not data.parsedData: return self.table_usage = {} + self.table_queries = {} for record in data.parsedData: if record is None: continue