yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-09 16:03:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/redash): Limit size of RedashSourceReport (#9873)

Browse Source
This commit is contained in:
Andrew Jones 2024-02-20 20:48:43 -05:00 committed by GitHub
parent e4bc915c78
commit 66871cb81d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
metadata-ingestion/src/datahub/ingestion/source/redash.py
View File

@ -39,6 +39,7 @@ from datahub.metadata.schema_classes import (
ChartTypeClass, ChartTypeClass,
DashboardInfoClass, DashboardInfoClass,
) )
from datahub.utilities.lossy_collections import LossyDict, LossyList
from datahub.utilities.perf_timer import PerfTimer from datahub.utilities.perf_timer import PerfTimer
from datahub.utilities.sql_parser import SQLParser from datahub.utilities.sql_parser import SQLParser
@ -282,7 +283,7 @@ class RedashConfig(ConfigModel):
@dataclass @dataclass
class RedashSourceReport(SourceReport): class RedashSourceReport(SourceReport):
items_scanned: int = 0 items_scanned: int = 0
filtered: List[str] = field(default_factory=list) filtered: LossyList[str] = field(default_factory=LossyList)
queries_problem_parsing: Set[str] = field(default_factory=set) queries_problem_parsing: Set[str] = field(default_factory=set)
queries_no_dataset: Set[str] = field(default_factory=set) queries_no_dataset: Set[str] = field(default_factory=set)
charts_no_input: Set[str] = field(default_factory=set) charts_no_input: Set[str] = field(default_factory=set)
@ -295,7 +296,7 @@ class RedashSourceReport(SourceReport):
) )
max_page_dashboards: Optional[int] = field(default=None) max_page_dashboards: Optional[int] = field(default=None)
api_page_limit: Optional[float] = field(default=None) api_page_limit: Optional[float] = field(default=None)
timing: Dict[str, int] = field(default_factory=dict) timing: LossyDict[str, int] = field(default_factory=LossyDict)
def report_item_scanned(self) -> None: def report_item_scanned(self) -> None:
self.items_scanned += 1 self.items_scanned += 1