From a3ca8b6e6697f9a1786d9c633cd64dc6a70fb074 Mon Sep 17 00:00:00 2001 From: Onkar Ravgan Date: Tue, 5 Sep 2023 11:01:28 +0530 Subject: [PATCH] Added project filter pattern to dashboard entity (#12925) --- ingestion/examples/workflows/metabase.yaml | 1 + ingestion/examples/workflows/powerbi.yaml | 4 ++ ingestion/examples/workflows/tableau.yaml | 1 + .../source/dashboard/dashboard_service.py | 25 +++++++- .../source/dashboard/metabase/metadata.py | 16 +++--- .../source/dashboard/powerbi/metadata.py | 57 +++++++++++++++---- .../source/dashboard/tableau/metadata.py | 17 +++++- ingestion/src/metadata/utils/filters.py | 15 +++++ .../unit/topology/dashboard/test_metabase.py | 2 + .../dashboard/domo-dashboard/yaml.md | 9 +++ .../connectors/dashboard/looker/yaml.md | 9 +++ .../connectors/dashboard/metabase/yaml.md | 9 +++ .../connectors/dashboard/mode/yaml.md | 9 +++ .../connectors/dashboard/powerbi/yaml.md | 9 +++ .../connectors/dashboard/qliksense/yaml.md | 9 +++ .../connectors/dashboard/quicksight/yaml.md | 9 +++ .../connectors/dashboard/redash/yaml.md | 9 +++ .../connectors/dashboard/superset/yaml.md | 9 +++ .../connectors/dashboard/tableau/yaml.md | 9 +++ .../dashboardServiceMetadataPipeline.json | 4 ++ 20 files changed, 210 insertions(+), 22 deletions(-) diff --git a/ingestion/examples/workflows/metabase.yaml b/ingestion/examples/workflows/metabase.yaml index fbf247d9e61..08738c63bfe 100644 --- a/ingestion/examples/workflows/metabase.yaml +++ b/ingestion/examples/workflows/metabase.yaml @@ -12,6 +12,7 @@ source: type: DashboardMetadata dashboardFilterPattern: {} chartFilterPattern: {} + projectFilterPattern: {} sink: type: metadata-rest config: {} diff --git a/ingestion/examples/workflows/powerbi.yaml b/ingestion/examples/workflows/powerbi.yaml index f5f8acbd751..a5a02c00916 100644 --- a/ingestion/examples/workflows/powerbi.yaml +++ b/ingestion/examples/workflows/powerbi.yaml @@ -25,6 +25,10 @@ source: includes: - Supplier Quality Analysis Sample - "Customer" + projectFilterPattern: + includes: + - Supplier Quality Analysis Sample + - "Customer" sink: type: metadata-rest config: {} diff --git a/ingestion/examples/workflows/tableau.yaml b/ingestion/examples/workflows/tableau.yaml index d1918e70254..007c022c29c 100644 --- a/ingestion/examples/workflows/tableau.yaml +++ b/ingestion/examples/workflows/tableau.yaml @@ -24,6 +24,7 @@ source: type: DashboardMetadata dashboardFilterPattern: {} chartFilterPattern: {} + projectFilterPattern: {} sink: type: metadata-rest config: {} diff --git a/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py b/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py index ff73a177b7a..d7167dc9907 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py @@ -63,7 +63,7 @@ from metadata.ingestion.models.topology import ( from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.connections import get_connection, get_test_connection_fn from metadata.utils import fqn -from metadata.utils.filters import filter_by_dashboard +from metadata.utils.filters import filter_by_dashboard, filter_by_project from metadata.utils.logger import ingestion_logger logger = ingestion_logger() @@ -447,6 +447,18 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC): try: dashboard_details = self.get_dashboard_details(dashboard) + self.context.project_name = ( # pylint: disable=assignment-from-none + self.get_project_name(dashboard_details=dashboard_details) + ) + if self.context.project_name and filter_by_project( + self.source_config.projectFilterPattern, + self.context.project_name, + ): + self.status.filter( + self.context.project_name, + "Project / Workspace Filtered Out", + ) + continue except Exception as exc: logger.debug(traceback.format_exc()) logger.warning( @@ -501,3 +513,14 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC): return None return database_schema_name + + def get_project_name( # pylint: disable=unused-argument, useless-return + self, dashboard_details: Any + ) -> Optional[str]: + """ + Get the project / workspace / folder / collection name of the dashboard + """ + logger.debug( + f"Projects are not supported for {self.service_connection.type.name}" + ) + return None diff --git a/ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py index 8dfa6c35a58..ae382b9296a 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py @@ -11,7 +11,7 @@ """Metabase source module""" import traceback -from typing import Iterable, List, Optional +from typing import Any, Iterable, List, Optional from metadata.generated.schema.api.data.createChart import CreateChartRequest from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest @@ -100,17 +100,17 @@ class MetabaseSource(DashboardServiceSource): """ return self.client.get_dashboard_details(dashboard.id) - def _get_collection_name(self, collection_id: Optional[str]) -> Optional[str]: + def get_project_name(self, dashboard_details: Any) -> Optional[str]: """ - Method to search the dataset using id in the workspace dict + Method to get the project name by searching the dataset using id in the workspace dict """ try: - if collection_id: + if dashboard_details.collection_id: collection_name = next( ( collection.name for collection in self.collections - if collection.id == collection_id + if collection.id == dashboard_details.collection_id ), None, ) @@ -118,7 +118,7 @@ class MetabaseSource(DashboardServiceSource): except Exception as exc: # pylint: disable=broad-except logger.debug(traceback.format_exc()) logger.warning( - f"Error fetching the collection details for [{collection_id}]: {exc}" + f"Error fetching the collection details for [{dashboard_details.collection_id}]: {exc}" ) return None @@ -138,9 +138,7 @@ class MetabaseSource(DashboardServiceSource): sourceUrl=dashboard_url, displayName=dashboard_details.name, description=dashboard_details.description, - project=self._get_collection_name( - collection_id=dashboard_details.collection_id - ), + project=self.context.project_name, charts=[ fqn.build( self.metadata, diff --git a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py index bd172770bd8..d7e6881baab 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py @@ -44,6 +44,7 @@ from metadata.ingestion.api.steps import InvalidSourceException from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource from metadata.ingestion.source.dashboard.powerbi.models import ( Dataset, + Group, PowerBIDashboard, PowerBIReport, PowerBiTable, @@ -54,6 +55,7 @@ from metadata.utils.filters import ( filter_by_chart, filter_by_dashboard, filter_by_datamodel, + filter_by_project, ) from metadata.utils.helpers import clean_uri from metadata.utils.logger import ingestion_logger @@ -80,12 +82,32 @@ class PowerbiSource(DashboardServiceSource): def prepare(self): if self.service_connection.useAdminApis: - self.get_admin_workspace_data() + groups = self.get_admin_workspace_data() else: - self.get_org_workspace_data() + groups = self.get_org_workspace_data() + if groups: + self.workspace_data = self.get_filtered_workspaces(groups) return super().prepare() - def get_org_workspace_data(self): + def get_filtered_workspaces(self, groups: List[Group]) -> List[Group]: + """ + Method to get the workspaces filtered by project filter pattern + """ + filtered_groups = [] + for group in groups: + if filter_by_project( + self.source_config.projectFilterPattern, + group.name, + ): + self.status.filter( + group.name, + "Workspace Filtered Out", + ) + continue + filtered_groups.append(group) + return filtered_groups + + def get_org_workspace_data(self) -> Optional[List[Group]]: """ fetch all the group workspace ids """ @@ -121,12 +143,13 @@ class PowerbiSource(DashboardServiceSource): ) or [] ) - self.workspace_data = groups + return groups - def get_admin_workspace_data(self): + def get_admin_workspace_data(self) -> Optional[List[Group]]: """ fetch all the workspace ids """ + groups = [] workspaces = self.client.fetch_all_workspaces() if workspaces: workspace_id_list = [workspace.id for workspace in workspaces] @@ -155,7 +178,7 @@ class PowerbiSource(DashboardServiceSource): response = self.client.fetch_workspace_scan_result( scan_id=workspace_scan.id ) - self.workspace_data.extend( + groups.extend( [ active_workspace for active_workspace in response.workspaces @@ -166,7 +189,8 @@ class PowerbiSource(DashboardServiceSource): logger.error("Error in fetching dashboards and charts") count += 1 else: - logger.error("Unable to fetch any Powerbi workspaces") + logger.error("Unable to fetch any PowerBI workspaces") + return groups or None @classmethod def create(cls, config_dict, metadata_config: OpenMetadataConnection): @@ -180,7 +204,7 @@ class PowerbiSource(DashboardServiceSource): def get_dashboard(self) -> Any: """ - Method to iterate through dashboard lists filter dashbaords & yield dashboard details + Method to iterate through dashboard lists filter dashboards & yield dashboard details """ for workspace in self.workspace_data: self.context.workspace = workspace @@ -369,7 +393,7 @@ class PowerbiSource(DashboardServiceSource): workspace_id=self.context.workspace.id, dashboard_id=dashboard_details.id, ), - project=str(self.context.workspace.name), + project=self.get_project_name(dashboard_details=dashboard_details), displayName=dashboard_details.displayName, dashboardType=DashboardType.Dashboard, charts=[ @@ -391,7 +415,7 @@ class PowerbiSource(DashboardServiceSource): workspace_id=self.context.workspace.id, dashboard_id=dashboard_details.id, ), - project=str(self.context.workspace.name), + project=self.get_project_name(dashboard_details=dashboard_details), displayName=dashboard_details.name, service=self.context.dashboard_service.fullyQualifiedName.__root__, ) @@ -668,3 +692,16 @@ class PowerbiSource(DashboardServiceSource): return next(iter(workspace_names), None) return None + + def get_project_name(self, dashboard_details: Any) -> Optional[str]: + """ + Get the project / workspace / folder / collection name of the dashboard + """ + try: + return str(self.context.workspace.name) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning( + f"Error fetching project name for {dashboard_details.id}: {exc}" + ) + return None diff --git a/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py index b1629b38948..4f860474b49 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py @@ -12,7 +12,7 @@ Tableau source module """ import traceback -from typing import Iterable, List, Optional, Set +from typing import Any, Iterable, List, Optional, Set from metadata.generated.schema.api.data.createChart import CreateChartRequest from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest @@ -215,7 +215,7 @@ class TableauSource(DashboardServiceSource): name=dashboard_details.id, displayName=dashboard_details.name, description=dashboard_details.description, - project=dashboard_details.project.name, + project=self.get_project_name(dashboard_details=dashboard_details), charts=[ fqn.build( self.metadata, @@ -460,3 +460,16 @@ class TableauSource(DashboardServiceSource): logger.debug(traceback.format_exc()) logger.warning(f"Error to yield datamodel column: {exc}") return datasource_columns + + def get_project_name(self, dashboard_details: Any) -> Optional[str]: + """ + Get the project / workspace / folder / collection name of the dashboard + """ + try: + return dashboard_details.project.name + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning( + f"Error fetching project name for {dashboard_details.id}: {exc}" + ) + return None diff --git a/ingestion/src/metadata/utils/filters.py b/ingestion/src/metadata/utils/filters.py index 24e6621aced..fa9fc268d95 100644 --- a/ingestion/src/metadata/utils/filters.py +++ b/ingestion/src/metadata/utils/filters.py @@ -240,6 +240,21 @@ def filter_by_datamodel( return _filter(datamodel_filter_pattern, datamodel_name) +def filter_by_project( + project_filter_pattern: Optional[FilterPattern], project_name: str +) -> bool: + """ + Return True if the project needs to be filtered, False otherwise + + Include takes precedence over exclude + + :param project_filter_pattern: Model defining project filtering logic + :param project_name: project name + :return: True for filtering, False otherwise + """ + return _filter(project_filter_pattern, project_name) + + def filter_by_search_index( search_index_filter_pattern: Optional[FilterPattern], search_index_name: str ) -> bool: diff --git a/ingestion/tests/unit/topology/dashboard/test_metabase.py b/ingestion/tests/unit/topology/dashboard/test_metabase.py index 547a62907a2..93f1c798b1e 100644 --- a/ingestion/tests/unit/topology/dashboard/test_metabase.py +++ b/ingestion/tests/unit/topology/dashboard/test_metabase.py @@ -169,6 +169,7 @@ EXPECTED_DASHBOARD = [ sourceUrl="http://metabase.com/dashboard/1-test-db", charts=[], service=FullyQualifiedEntityName(__root__="mock_metabase"), + project="Test Collection", ) ] @@ -227,6 +228,7 @@ class MetabaseUnitTest(TestCase): ) self.metabase.client = SimpleNamespace() self.metabase.context.__dict__["dashboard_service"] = MOCK_DASHBOARD_SERVICE + self.metabase.context.__dict__["project_name"] = "Test Collection" def test_dashboard_name(self): assert ( diff --git a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/domo-dashboard/yaml.md b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/domo-dashboard/yaml.md index 2328b45255c..a93957fb58a 100644 --- a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/domo-dashboard/yaml.md +++ b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/domo-dashboard/yaml.md @@ -12,6 +12,7 @@ slug: /connectors/dashboard/domo-dashboard/yaml | Owners | {% icon iconName="check" /%} | | Tags | {% icon iconName="cross" /%} | | Datamodels | {% icon iconName="cross" /%} | +| Projects | {% icon iconName="cross" /%} | | Lineage | {% icon iconName="cross" /%} | In this section, we provide guides and references to use the DomoDashboard connector. @@ -101,6 +102,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it. - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard". +- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project". - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten. - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion. - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion. @@ -167,6 +169,13 @@ source: # excludes: # - chart3 # - chart4 + # projectFilterPattern: + # includes: + # - project1 + # - project2 + # excludes: + # - project3 + # - project4 ``` ```yaml {% srNumber=7 %} sink: diff --git a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/looker/yaml.md b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/looker/yaml.md index 43f14736c78..e50b2ccfab7 100644 --- a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/looker/yaml.md +++ b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/looker/yaml.md @@ -12,6 +12,7 @@ slug: /connectors/dashboard/looker/yaml | Owners | {% icon iconName="check" /%} | | Tags | {% icon iconName="cross" /%} | | Datamodels | {% icon iconName="check" /%} | +| Projects | {% icon iconName="cross" /%} | | Lineage | {% icon iconName="check" /%} | In this section, we provide guides and references to use the Looker connector. @@ -124,6 +125,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it. - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard". +- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project". - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten. - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion. - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion. @@ -191,6 +193,13 @@ source: # excludes: # - chart3 # - chart4 + # projectFilterPattern: + # includes: + # - project1 + # - project2 + # excludes: + # - project3 + # - project4 ``` ```yaml {% srNumber=6 %} diff --git a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/metabase/yaml.md b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/metabase/yaml.md index 7b537f94591..aa05a1ae3b4 100644 --- a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/metabase/yaml.md +++ b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/metabase/yaml.md @@ -12,6 +12,7 @@ slug: /connectors/dashboard/metabase/yaml | Owners | {% icon iconName="cross" /%} | | Tags | {% icon iconName="cross" /%} | | Datamodels | {% icon iconName="cross" /%} | +| Projects | {% icon iconName="check" /%} | | Lineage | {% icon iconName="check" /%} | In this section, we provide guides and references to use the Metabase connector. @@ -90,6 +91,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it. - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard". +- **projectFilterPattern**: Filter the Metabase dashboards and charts by projects (In case of Metabase, projects corresponds to Collections). Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project". - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten. - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion. - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion. @@ -150,6 +152,13 @@ source: # excludes: # - chart3 # - chart4 + # projectFilterPattern: + # includes: + # - project1 + # - project2 + # excludes: + # - project3 + # - project4 ``` ```yaml {% srNumber=5 %} diff --git a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/mode/yaml.md b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/mode/yaml.md index 00bf84149f3..9af5ab1b0f4 100644 --- a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/mode/yaml.md +++ b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/mode/yaml.md @@ -12,6 +12,7 @@ slug: /connectors/dashboard/mode/yaml | Owners | {% icon iconName="cross" /%} | | Tags | {% icon iconName="cross" /%} | | Datamodels | {% icon iconName="cross" /%} | +| Projects | {% icon iconName="cross" /%} | | Lineage | {% icon iconName="check" /%} | In this section, we provide guides and references to use the Mode connector. @@ -111,6 +112,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it. - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard". +- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project". - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten. - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion. - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion. @@ -173,6 +175,13 @@ source: # excludes: # - chart3 # - chart4 + # projectFilterPattern: + # includes: + # - project1 + # - project2 + # excludes: + # - project3 + # - project4 ``` ```yaml {% srNumber=6 %} sink: diff --git a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/powerbi/yaml.md b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/powerbi/yaml.md index ac9e1231e4b..2601d3b9c08 100644 --- a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/powerbi/yaml.md +++ b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/powerbi/yaml.md @@ -12,6 +12,7 @@ slug: /connectors/dashboard/powerbi/yaml | Owners | {% icon iconName="cross" /%} | | Tags | {% icon iconName="cross" /%} | | Datamodels | {% icon iconName="check" /%} | +| Projects | {% icon iconName="check" /%} | | Lineage | {% icon iconName="check" /%} | In this section, we provide guides and references to use the PowerBI connector. @@ -206,6 +207,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it. - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard". +- **projectFilterPattern**: Filter the PowerBI dashboards, reports, tiles and data sources by projects(In case of PowerBI, projects correspond to workspaces). Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project". - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten. - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion. - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion. @@ -281,6 +283,13 @@ source: # excludes: # - chart3 # - chart4 + # projectFilterPattern: + # includes: + # - project1 + # - project2 + # excludes: + # - project3 + # - project4 ``` ```yaml {% srNumber=10 %} sink: diff --git a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/qliksense/yaml.md b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/qliksense/yaml.md index fba2546dedc..9e73edd907d 100644 --- a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/qliksense/yaml.md +++ b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/qliksense/yaml.md @@ -12,6 +12,7 @@ slug: /connectors/dashboard/qliksense/yaml | Owners | {% icon iconName="cross" /%} | | Tags | {% icon iconName="cross" /%} | | Datamodels | {% icon iconName="check" /%} | +| Projects | {% icon iconName="cross" /%} | | Lineage | {% icon iconName="check" /%} | In this section, we provide guides and references to use the PowerBI connector. @@ -145,6 +146,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it. - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard". +- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project". - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten. - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion. - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion. @@ -221,6 +223,13 @@ source: # excludes: # - chart3 # - chart4 + # projectFilterPattern: + # includes: + # - project1 + # - project2 + # excludes: + # - project3 + # - project4 ``` ```yaml {% srNumber=8 %} sink: diff --git a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/quicksight/yaml.md b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/quicksight/yaml.md index a706fe66855..ac3caafc204 100644 --- a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/quicksight/yaml.md +++ b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/quicksight/yaml.md @@ -12,6 +12,7 @@ slug: /connectors/dashboard/quicksight/yaml | Owners | {% icon iconName="cross" /%} | | Tags | {% icon iconName="cross" /%} | | Datamodels | {% icon iconName="cross" /%} | +| Projects | {% icon iconName="cross" /%} | | Lineage | {% icon iconName="check" /%} | In this section, we provide guides and references to use the QuickSight connector. @@ -145,6 +146,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it. - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard". +- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project". - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten. - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion. - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion. @@ -212,6 +214,13 @@ source: # excludes: # - chart3 # - chart4 + # projectFilterPattern: + # includes: + # - project1 + # - project2 + # excludes: + # - project3 + # - project4 ```yaml {% srNumber=6 %} sink: diff --git a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/redash/yaml.md b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/redash/yaml.md index 72cd71e3ab7..e2e9f2d36c0 100644 --- a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/redash/yaml.md +++ b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/redash/yaml.md @@ -12,6 +12,7 @@ slug: /connectors/dashboard/redash/yaml | Owners | {% icon iconName="check" /%} | | Tags | {% icon iconName="check" /%} | | Datamodels | {% icon iconName="cross" /%} | +| Projects | {% icon iconName="cross" /%} | | Lineage | {% icon iconName="check" /%} | In this section, we provide guides and references to use the Redash connector. @@ -92,6 +93,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it. - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard". +- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project". - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten. - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion. - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion. @@ -154,6 +156,13 @@ source: # excludes: # - chart3 # - chart4 + # projectFilterPattern: + # includes: + # - project1 + # - project2 + # excludes: + # - project3 + # - project4 ``` ```yaml {% srNumber=6 %} sink: diff --git a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/superset/yaml.md b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/superset/yaml.md index 82dc03b7848..07323c663d2 100644 --- a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/superset/yaml.md +++ b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/superset/yaml.md @@ -12,6 +12,7 @@ slug: /connectors/dashboard/superset/yaml | Owners | {% icon iconName="check" /%} | | Tags | {% icon iconName="cross" /%} | | Datamodels | {% icon iconName="cross" /%} | +| Projects | {% icon iconName="cross" /%} | | Lineage | {% icon iconName="check" /%} | In this section, we provide guides and references to use the Superset connector. @@ -141,6 +142,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it. - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard". +- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project". - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten. - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion. - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion. @@ -218,6 +220,13 @@ source: # excludes: # - chart3 # - chart4 + # projectFilterPattern: + # includes: + # - project1 + # - project2 + # excludes: + # - project3 + # - project4 ``` ```yaml {% srNumber=5 %} sink: diff --git a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/tableau/yaml.md b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/tableau/yaml.md index d4a0d6cd566..7df99bf4ad5 100644 --- a/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/tableau/yaml.md +++ b/openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/tableau/yaml.md @@ -12,6 +12,7 @@ slug: /connectors/dashboard/tableau/yaml | Owners | {% icon iconName="check" /%} | | Tags | {% icon iconName="check" /%} | | Datamodels | {% icon iconName="check" /%} | +| Projects | {% icon iconName="check" /%} | | Lineage | {% icon iconName="check" /%} | In this section, we provide guides and references to use the Tableau connector. @@ -129,6 +130,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it. - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard". +- **projectFilterPattern**: Filter the tableau dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project". - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten. - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion. - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion. @@ -218,6 +220,13 @@ source: # excludes: # - datamodel3 # - datamodel4 + # projectFilterPattern: + # includes: + # - project1 + # - project2 + # excludes: + # - project3 + # - project4 ``` ```yaml {% srNumber=9 %} sink: diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json index 0f5a1bc85ee..a13e04bee42 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json @@ -30,6 +30,10 @@ "description": "Regex exclude or include data models that matches the pattern.", "$ref": "../type/filterPattern.json#/definitions/filterPattern" }, + "projectFilterPattern": { + "description": "Regex to exclude or include projects that matches the pattern.", + "$ref": "../type/filterPattern.json#/definitions/filterPattern" + }, "dbServiceNames": { "title": "Database Service Names List", "description": "List of Database Service Names for creation of lineage",