yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-09 16:03:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingestion): add ability to exclude archived mode reports. (#14978)

Browse Source 
Co-authored-by: Tamas Nemeth <treff7es@gmail.com>
This commit is contained in:
Neha Gslab 2025-10-16 16:28:58 +05:30 committed by GitHub
parent 10c28be20c
commit 1934ed45e8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
metadata-ingestion/src/datahub/ingestion/source/mode.py
View File

@ -214,6 +214,10 @@ class ModeConfig(
description="Number of items per page for paginated API requests.", description="Number of items per page for paginated API requests.",
) )
exclude_archived: bool = Field(
default=False, description="Exclude archived reports"
)
@validator("connect_uri") @validator("connect_uri")
def remove_trailing_slash(cls, v): def remove_trailing_slash(cls, v):
return config_clean.remove_trailing_slashes(v) return config_clean.remove_trailing_slashes(v)
@ -1473,6 +1477,15 @@ class ModeSource(StatefulIngestionSourceBase):
logger.debug( logger.debug(
f"Read {len(reports_page)} reports records from workspace {self.workspace_uri} space {space_token}" f"Read {len(reports_page)} reports records from workspace {self.workspace_uri} space {space_token}"
) )
if self.config.exclude_archived:
logger.debug(
f"Excluding archived reports since exclude_archived: {self.config.exclude_archived}"
)
reports_page = [
report
for report in reports_page
if not report.get("archived", False)
]
yield reports_page yield reports_page
except ModeRequestError as e: except ModeRequestError as e:
if isinstance(e, HTTPError) and e.response.status_code == 404: if isinstance(e, HTTPError) and e.response.status_code == 404: