yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-31 21:03:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): unity - Removing unneeded dependency for sqlalchemy from unity-catalog to fix connector setup (#6379)

Browse Source
This commit is contained in:
Tamas Nemeth 2022-11-09 15:57:17 +01:00 committed by GitHub
parent 44cfc29af0
commit ef5c7128da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 22 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
metadata-ingestion/src/datahub/ingestion/source/unity/report.py
View File

@ -1,12 +1,15 @@
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import Dict from typing import Dict
from datahub.ingestion.source.sql.sql_common import SQLSourceReport from datahub.ingestion.source.state.stale_entity_removal_handler import (
StaleEntityRemovalSourceReport,
)
from datahub.utilities.lossy_collections import LossyList
from datahub.utilities.stats_collections import TopKDict from datahub.utilities.stats_collections import TopKDict
@dataclass @dataclass
class UnityCatalogReport(SQLSourceReport): class UnityCatalogReport(StaleEntityRemovalSourceReport):
scanned_metastore: int = 0 scanned_metastore: int = 0
scanned_catalog: int = 0 scanned_catalog: int = 0
scanned_schema: int = 0 scanned_schema: int = 0
@ -14,6 +17,9 @@ class UnityCatalogReport(SQLSourceReport):
num_catalogs_to_scan: Dict[str, int] = field(default_factory=TopKDict) num_catalogs_to_scan: Dict[str, int] = field(default_factory=TopKDict)
num_schemas_to_scan: Dict[str, int] = field(default_factory=TopKDict) num_schemas_to_scan: Dict[str, int] = field(default_factory=TopKDict)
num_tables_to_scan: Dict[str, int] = field(default_factory=TopKDict) num_tables_to_scan: Dict[str, int] = field(default_factory=TopKDict)
tables_scanned: int = 0
views_scanned: int = 0
filtered: LossyList[str] = field(default_factory=LossyList)
def increment_scanned_metastore(self, count: int = 1) -> None: def increment_scanned_metastore(self, count: int = 1) -> None:
self.scanned_metastore = self.scanned_metastore + count self.scanned_metastore = self.scanned_metastore + count
@ -26,3 +32,17 @@ class UnityCatalogReport(SQLSourceReport):
def increment_scanned_table(self, count: int = 1) -> None: def increment_scanned_table(self, count: int = 1) -> None:
self.scanned_table = self.scanned_table + count self.scanned_table = self.scanned_table + count
def report_dropped(self, ent_name: str) -> None:
self.filtered.append(ent_name)
def report_entity_scanned(self, name: str, ent_type: str = "table") -> None:
"""
Entity could be a view or a table
"""
if ent_type == "table":
self.tables_scanned += 1
elif ent_type == "view":
self.views_scanned += 1
else:
raise KeyError(f"Unknown entity {ent_type}.")