yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-26 17:37:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/snowflake): fix usage enum bug (#8649)

Browse Source
This commit is contained in:
Harshal Sheth 2023-08-17 10:27:19 -07:00 committed by GitHub
parent c0addf6eef
commit 130f908a49
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 10 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py
View File

@ -1,5 +1,6 @@
from typing import List, Optional
from datahub.configuration.time_window_config import BucketDuration
from datahub.ingestion.source.snowflake.constants import SnowflakeObjectDomain
from datahub.ingestion.source.snowflake.snowflake_config import DEFAULT_TABLES_DENY_LIST
@ -575,14 +576,17 @@ class SnowflakeQuery:
def usage_per_object_per_time_bucket_for_time_window(
start_time_millis: int,
end_time_millis: int,
time_bucket_size: str,
time_bucket_size: BucketDuration,
use_base_objects: bool,
top_n_queries: int,
include_top_n_queries: bool,
) -> str:
if not include_top_n_queries:
top_n_queries = 0
assert time_bucket_size == "DAY" or time_bucket_size == "HOUR"
assert (
time_bucket_size == BucketDuration.DAY
or time_bucket_size == BucketDuration.HOUR
)
objects_column = (
"BASE_OBJECTS_ACCESSED" if use_base_objects else "DIRECT_OBJECTS_ACCESSED"
)
@ -629,7 +633,7 @@ class SnowflakeQuery:
SELECT
object_name,
ANY_VALUE(object_domain) AS object_domain,
DATE_TRUNC('{time_bucket_size}', CONVERT_TIMEZONE('UTC', query_start_time)) AS bucket_start_time,
DATE_TRUNC('{time_bucket_size.value}', CONVERT_TIMEZONE('UTC', query_start_time)) AS bucket_start_time,
count(distinct(query_id)) AS total_queries,
count( distinct(user_name) ) AS total_users
FROM
@ -644,7 +648,7 @@ class SnowflakeQuery:
SELECT
object_name,
column_name,
DATE_TRUNC('{time_bucket_size}', CONVERT_TIMEZONE('UTC', query_start_time)) AS bucket_start_time,
DATE_TRUNC('{time_bucket_size.value}', CONVERT_TIMEZONE('UTC', query_start_time)) AS bucket_start_time,
count(distinct(query_id)) AS total_queries
FROM
field_access_history
@ -658,7 +662,7 @@ class SnowflakeQuery:
(
SELECT
object_name,
DATE_TRUNC('{time_bucket_size}', CONVERT_TIMEZONE('UTC', query_start_time)) AS bucket_start_time,
DATE_TRUNC('{time_bucket_size.value}', CONVERT_TIMEZONE('UTC', query_start_time)) AS bucket_start_time,
count(distinct(query_id)) AS total_queries,
user_name,
ANY_VALUE(users.email) AS user_email
@ -677,7 +681,7 @@ class SnowflakeQuery:
(
SELECT
object_name,
DATE_TRUNC('{time_bucket_size}', CONVERT_TIMEZONE('UTC', query_start_time)) AS bucket_start_time,
DATE_TRUNC('{time_bucket_size.value}', CONVERT_TIMEZONE('UTC', query_start_time)) AS bucket_start_time,
query_history.query_text AS query_text,
count(distinct(access_history.query_id)) AS total_queries
FROM

1
metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py
View File

@ -356,7 +356,6 @@ class SnowflakeUsageExtractor(
def _get_operation_aspect_work_unit(
self, event: SnowflakeJoinedAccessEvent, discovered_datasets: List[str]
) -> Iterable[MetadataWorkUnit]:
if event.query_start_time and event.query_type:
start_time = event.query_start_time
query_type = event.query_type