yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-06-27 05:03:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest): add num_queries_used_in_lineage counter (#12336)

Browse Source
This commit is contained in:
Harshal Sheth 2025-01-15 14:58:47 -08:00 committed by GitHub
parent a3c7a336fa
commit 96cfa4662d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -85,6 +85,7 @@ metadata-service/plugin/src/test/resources/sample-plugins/**
smoke-test/rollback-reports smoke-test/rollback-reports
coverage*.xml coverage*.xml
.vercel .vercel
.envrc
# A long series of binary directories we should ignore # A long series of binary directories we should ignore
datahub-frontend/bin/main/ datahub-frontend/bin/main/

2
metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py
View File

@ -284,6 +284,7 @@ class SqlAggregatorReport(Report):
# Queries. # Queries.
num_queries_entities_generated: int = 0 num_queries_entities_generated: int = 0
num_queries_used_in_lineage: Optional[int] = None
num_queries_skipped_due_to_filters: int = 0 num_queries_skipped_due_to_filters: int = 0
# Usage-related. # Usage-related.
@ -1200,6 +1201,7 @@ class SqlParsingAggregator(Closeable):
queries_generated: Set[QueryId] = set() queries_generated: Set[QueryId] = set()
yield from self._gen_lineage_mcps(queries_generated) yield from self._gen_lineage_mcps(queries_generated)
self.report.num_queries_used_in_lineage = len(queries_generated)
yield from self._gen_usage_statistics_mcps() yield from self._gen_usage_statistics_mcps()
yield from self._gen_operation_mcps(queries_generated) yield from self._gen_operation_mcps(queries_generated)
yield from self._gen_remaining_queries(queries_generated) yield from self._gen_remaining_queries(queries_generated)