yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-27 15:38:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

issue-3420: debug-logger-added (#3434)

Browse Source
This commit is contained in:
codingwithabhi 2022-03-15 19:13:31 +05:30 committed by GitHub
parent 3e92693d34
commit 55c65b5c53
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 15 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ingestion/src/metadata/ingestion/source/snowflake_usage.py
View File

@ -126,6 +126,7 @@ class SnowflakeUsageSource(Source[TableQuery]):
sql=row["query_text"],
service_name=self.config.service_name,
)
logger.debug(f"Parsed Query: {row['query_text']}")
if row["schema_name"] is not None:
self.report.scanned(f"{row['database_name']}.{row['schema_name']}")
else:

24
ingestion/src/metadata/utils/helpers.py
View File

@ -222,17 +222,21 @@ def create_lineage(from_table, to_table, query_info, metadata):
def ingest_lineage(query_info, metadata_config):
from sqllineage.runner import LineageRunner
result = LineageRunner(query_info["sql"])
metadata = OpenMetadata(metadata_config)
for intermediate_table in result.intermediate_tables:
for source_table in result.source_tables:
create_lineage(source_table, intermediate_table, query_info, metadata)
for target_table in result.target_tables:
create_lineage(intermediate_table, target_table, query_info, metadata)
if not result.intermediate_tables:
for target_table in result.target_tables:
try:
result = LineageRunner(query_info["sql"])
metadata = OpenMetadata(metadata_config)
for intermediate_table in result.intermediate_tables:
for source_table in result.source_tables:
create_lineage(source_table, target_table, query_info, metadata)
create_lineage(source_table, intermediate_table, query_info, metadata)
for target_table in result.target_tables:
create_lineage(intermediate_table, target_table, query_info, metadata)
if not result.intermediate_tables:
for target_table in result.target_tables:
for source_table in result.source_tables:
create_lineage(source_table, target_table, query_info, metadata)
except Exception as err:
logger.error(str(err))
def get_raw_extract_iter(alchemy_helper) -> Iterable[Dict[str, Any]]: