Added project filter pattern to dashboard entity (#12925)

ingestion/examples/workflows/metabase.yaml

@@ -12,6 +12,7 @@ source:
       type: DashboardMetadata
       dashboardFilterPattern: {}
       chartFilterPattern: {}
+      projectFilterPattern: {}
 sink:
   type: metadata-rest
   config: {}

ingestion/examples/workflows/powerbi.yaml

@@ -25,6 +25,10 @@ source:
         includes:
           - Supplier Quality Analysis Sample
           - "Customer"
+      projectFilterPattern:
+        includes:
+          - Supplier Quality Analysis Sample
+          - "Customer"
 sink:
   type: metadata-rest
   config: {}

ingestion/examples/workflows/tableau.yaml

@@ -24,6 +24,7 @@ source:
       type: DashboardMetadata
       dashboardFilterPattern: {}
       chartFilterPattern: {}
+      projectFilterPattern: {}
 sink:
   type: metadata-rest
   config: {}

ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py

@@ -63,7 +63,7 @@ from metadata.ingestion.models.topology import (
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
 from metadata.utils import fqn
-from metadata.utils.filters import filter_by_dashboard
+from metadata.utils.filters import filter_by_dashboard, filter_by_project
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -447,6 +447,18 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
 
             try:
                 dashboard_details = self.get_dashboard_details(dashboard)
+                self.context.project_name = (  # pylint: disable=assignment-from-none
+                    self.get_project_name(dashboard_details=dashboard_details)
+                )
+                if self.context.project_name and filter_by_project(
+                    self.source_config.projectFilterPattern,
+                    self.context.project_name,
+                ):
+                    self.status.filter(
+                        self.context.project_name,
+                        "Project / Workspace Filtered Out",
+                    )
+                    continue
             except Exception as exc:
                 logger.debug(traceback.format_exc())
                 logger.warning(
@@ -501,3 +513,14 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
             return None
 
         return database_schema_name
+
+    def get_project_name(  # pylint: disable=unused-argument, useless-return
+        self, dashboard_details: Any
+    ) -> Optional[str]:
+        """
+        Get the project / workspace / folder / collection name of the dashboard
+        """
+        logger.debug(
+            f"Projects are not supported for {self.service_connection.type.name}"
+        )
+        return None

ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py

@@ -11,7 +11,7 @@
 """Metabase source module"""
 
 import traceback
-from typing import Iterable, List, Optional
+from typing import Any, Iterable, List, Optional
 
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
@@ -100,17 +100,17 @@ class MetabaseSource(DashboardServiceSource):
         """
         return self.client.get_dashboard_details(dashboard.id)
 
-    def _get_collection_name(self, collection_id: Optional[str]) -> Optional[str]:
+    def get_project_name(self, dashboard_details: Any) -> Optional[str]:
         """
-        Method to search the dataset using id in the workspace dict
+        Method to get the project name by searching the dataset using id in the workspace dict
         """
         try:
-            if collection_id:
+            if dashboard_details.collection_id:
                 collection_name = next(
                     (
                         collection.name
                         for collection in self.collections
-                        if collection.id == collection_id
+                        if collection.id == dashboard_details.collection_id
                     ),
                     None,
                 )
@@ -118,7 +118,7 @@ class MetabaseSource(DashboardServiceSource):
         except Exception as exc:  # pylint: disable=broad-except
             logger.debug(traceback.format_exc())
             logger.warning(
-                f"Error fetching the collection details for [{collection_id}]: {exc}"
+                f"Error fetching the collection details for [{dashboard_details.collection_id}]: {exc}"
             )
         return None
 
@@ -138,9 +138,7 @@ class MetabaseSource(DashboardServiceSource):
                 sourceUrl=dashboard_url,
                 displayName=dashboard_details.name,
                 description=dashboard_details.description,
-                project=self._get_collection_name(
+                project=self.context.project_name,
-                    collection_id=dashboard_details.collection_id
-                ),
                 charts=[
                     fqn.build(
                         self.metadata,

ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -44,6 +44,7 @@ from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
 from metadata.ingestion.source.dashboard.powerbi.models import (
     Dataset,
+    Group,
     PowerBIDashboard,
     PowerBIReport,
     PowerBiTable,
@@ -54,6 +55,7 @@ from metadata.utils.filters import (
     filter_by_chart,
     filter_by_dashboard,
     filter_by_datamodel,
+    filter_by_project,
 )
 from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ingestion_logger
@@ -80,12 +82,32 @@ class PowerbiSource(DashboardServiceSource):
 
     def prepare(self):
         if self.service_connection.useAdminApis:
-            self.get_admin_workspace_data()
+            groups = self.get_admin_workspace_data()
         else:
-            self.get_org_workspace_data()
+            groups = self.get_org_workspace_data()
+        if groups:
+            self.workspace_data = self.get_filtered_workspaces(groups)
         return super().prepare()
 
-    def get_org_workspace_data(self):
+    def get_filtered_workspaces(self, groups: List[Group]) -> List[Group]:
+        """
+        Method to get the workspaces filtered by project filter pattern
+        """
+        filtered_groups = []
+        for group in groups:
+            if filter_by_project(
+                self.source_config.projectFilterPattern,
+                group.name,
+            ):
+                self.status.filter(
+                    group.name,
+                    "Workspace Filtered Out",
+                )
+                continue
+            filtered_groups.append(group)
+        return filtered_groups
+
+    def get_org_workspace_data(self) -> Optional[List[Group]]:
         """
         fetch all the group workspace ids
         """
@@ -121,12 +143,13 @@ class PowerbiSource(DashboardServiceSource):
                     )
                     or []
                 )
-        self.workspace_data = groups
+        return groups
 
-    def get_admin_workspace_data(self):
+    def get_admin_workspace_data(self) -> Optional[List[Group]]:
         """
         fetch all the workspace ids
         """
+        groups = []
         workspaces = self.client.fetch_all_workspaces()
         if workspaces:
             workspace_id_list = [workspace.id for workspace in workspaces]
@@ -155,7 +178,7 @@ class PowerbiSource(DashboardServiceSource):
                     response = self.client.fetch_workspace_scan_result(
                         scan_id=workspace_scan.id
                     )
-                    self.workspace_data.extend(
+                    groups.extend(
                         [
                             active_workspace
                             for active_workspace in response.workspaces
@@ -166,7 +189,8 @@ class PowerbiSource(DashboardServiceSource):
                     logger.error("Error in fetching dashboards and charts")
                 count += 1
         else:
-            logger.error("Unable to fetch any Powerbi workspaces")
+            logger.error("Unable to fetch any PowerBI workspaces")
+        return groups or None
 
     @classmethod
     def create(cls, config_dict, metadata_config: OpenMetadataConnection):
@@ -180,7 +204,7 @@ class PowerbiSource(DashboardServiceSource):
 
     def get_dashboard(self) -> Any:
         """
-        Method to iterate through dashboard lists filter dashbaords & yield dashboard details
+        Method to iterate through dashboard lists filter dashboards & yield dashboard details
         """
         for workspace in self.workspace_data:
             self.context.workspace = workspace
@@ -369,7 +393,7 @@ class PowerbiSource(DashboardServiceSource):
                         workspace_id=self.context.workspace.id,
                         dashboard_id=dashboard_details.id,
                     ),
-                    project=str(self.context.workspace.name),
+                    project=self.get_project_name(dashboard_details=dashboard_details),
                     displayName=dashboard_details.displayName,
                     dashboardType=DashboardType.Dashboard,
                     charts=[
@@ -391,7 +415,7 @@ class PowerbiSource(DashboardServiceSource):
                         workspace_id=self.context.workspace.id,
                         dashboard_id=dashboard_details.id,
                     ),
-                    project=str(self.context.workspace.name),
+                    project=self.get_project_name(dashboard_details=dashboard_details),
                     displayName=dashboard_details.name,
                     service=self.context.dashboard_service.fullyQualifiedName.__root__,
                 )
@@ -668,3 +692,16 @@ class PowerbiSource(DashboardServiceSource):
             return next(iter(workspace_names), None)
 
         return None
+
+    def get_project_name(self, dashboard_details: Any) -> Optional[str]:
+        """
+        Get the project / workspace / folder / collection name of the dashboard
+        """
+        try:
+            return str(self.context.workspace.name)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Error fetching project name for {dashboard_details.id}: {exc}"
+            )
+        return None

ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -12,7 +12,7 @@
 Tableau source module
 """
 import traceback
-from typing import Iterable, List, Optional, Set
+from typing import Any, Iterable, List, Optional, Set
 
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
@@ -215,7 +215,7 @@ class TableauSource(DashboardServiceSource):
                 name=dashboard_details.id,
                 displayName=dashboard_details.name,
                 description=dashboard_details.description,
-                project=dashboard_details.project.name,
+                project=self.get_project_name(dashboard_details=dashboard_details),
                 charts=[
                     fqn.build(
                         self.metadata,
@@ -460,3 +460,16 @@ class TableauSource(DashboardServiceSource):
                 logger.debug(traceback.format_exc())
                 logger.warning(f"Error to yield datamodel column: {exc}")
         return datasource_columns
+
+    def get_project_name(self, dashboard_details: Any) -> Optional[str]:
+        """
+        Get the project / workspace / folder / collection name of the dashboard
+        """
+        try:
+            return dashboard_details.project.name
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Error fetching project name for {dashboard_details.id}: {exc}"
+            )
+        return None

ingestion/src/metadata/utils/filters.py

@@ -240,6 +240,21 @@ def filter_by_datamodel(
     return _filter(datamodel_filter_pattern, datamodel_name)
 
 
+def filter_by_project(
+    project_filter_pattern: Optional[FilterPattern], project_name: str
+) -> bool:
+    """
+    Return True if the project needs to be filtered, False otherwise
+
+    Include takes precedence over exclude
+
+    :param project_filter_pattern: Model defining project filtering logic
+    :param project_name: project name
+    :return: True for filtering, False otherwise
+    """
+    return _filter(project_filter_pattern, project_name)
+
+
 def filter_by_search_index(
     search_index_filter_pattern: Optional[FilterPattern], search_index_name: str
 ) -> bool:

ingestion/tests/unit/topology/dashboard/test_metabase.py

@@ -169,6 +169,7 @@ EXPECTED_DASHBOARD = [
         sourceUrl="http://metabase.com/dashboard/1-test-db",
         charts=[],
         service=FullyQualifiedEntityName(__root__="mock_metabase"),
+        project="Test Collection",
     )
 ]
 
@@ -227,6 +228,7 @@ class MetabaseUnitTest(TestCase):
         )
         self.metabase.client = SimpleNamespace()
         self.metabase.context.__dict__["dashboard_service"] = MOCK_DASHBOARD_SERVICE
+        self.metabase.context.__dict__["project_name"] = "Test Collection"
 
     def test_dashboard_name(self):
         assert (

openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/domo-dashboard/yaml.md

@@ -12,6 +12,7 @@ slug: /connectors/dashboard/domo-dashboard/yaml
 | Owners     | {% icon iconName="check" /%} |
 | Tags       | {% icon iconName="cross" /%} |
 | Datamodels | {% icon iconName="cross" /%} |
+| Projects   | {% icon iconName="cross" /%} |
 | Lineage    | {% icon iconName="cross" /%} |
 
 In this section, we provide guides and references to use the DomoDashboard connector.
@@ -101,6 +102,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
 
 - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
 - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
+- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
 - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
 - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
 - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@@ -167,6 +169,13 @@ source:
       #   excludes:
       #     - chart3
       #     - chart4
+      # projectFilterPattern:
+      #   includes:
+      #     - project1
+      #     - project2
+      #   excludes:
+      #     - project3
+      #     - project4
 ```
 ```yaml {% srNumber=7 %}
 sink:

openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/looker/yaml.md

@@ -12,6 +12,7 @@ slug: /connectors/dashboard/looker/yaml
 | Owners     | {% icon iconName="check" /%} |
 | Tags       | {% icon iconName="cross" /%} |
 | Datamodels | {% icon iconName="check" /%} |
+| Projects   | {% icon iconName="cross" /%} |
 | Lineage    | {% icon iconName="check" /%} |
 
 In this section, we provide guides and references to use the Looker connector.
@@ -124,6 +125,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
 
 - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
 - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
+- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
 - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
 - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
 - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@@ -191,6 +193,13 @@ source:
       #   excludes:
       #     - chart3
       #     - chart4
+      # projectFilterPattern:
+      #   includes:
+      #     - project1
+      #     - project2
+      #   excludes:
+      #     - project3
+      #     - project4
 
 ```
 ```yaml {% srNumber=6 %}

openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/metabase/yaml.md

@@ -12,6 +12,7 @@ slug: /connectors/dashboard/metabase/yaml
 | Owners     | {% icon iconName="cross" /%} |
 | Tags       | {% icon iconName="cross" /%} |
 | Datamodels | {% icon iconName="cross" /%} |
+| Projects   | {% icon iconName="check" /%} |
 | Lineage    | {% icon iconName="check" /%} |
 
 In this section, we provide guides and references to use the Metabase connector.
@@ -90,6 +91,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
 
 - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
 - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
+- **projectFilterPattern**: Filter the Metabase dashboards and charts by projects (In case of Metabase, projects corresponds to Collections). Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
 - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
 - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
 - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@@ -150,6 +152,13 @@ source:
       #   excludes:
       #     - chart3
       #     - chart4
+      # projectFilterPattern:
+      #   includes:
+      #     - project1
+      #     - project2
+      #   excludes:
+      #     - project3
+      #     - project4
 
 ```
 ```yaml {% srNumber=5 %}

openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/mode/yaml.md

@@ -12,6 +12,7 @@ slug: /connectors/dashboard/mode/yaml
 | Owners     | {% icon iconName="cross" /%} |
 | Tags       | {% icon iconName="cross" /%} |
 | Datamodels | {% icon iconName="cross" /%} |
+| Projects   | {% icon iconName="cross" /%} |
 | Lineage    | {% icon iconName="check" /%} |
 
 In this section, we provide guides and references to use the Mode connector.
@@ -111,6 +112,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
 
 - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
 - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
+- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
 - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
 - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
 - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@@ -173,6 +175,13 @@ source:
       #   excludes:
       #     - chart3
       #     - chart4
+      # projectFilterPattern:
+      #   includes:
+      #     - project1
+      #     - project2
+      #   excludes:
+      #     - project3
+      #     - project4
 ```
 ```yaml {% srNumber=6 %}
 sink:

openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/powerbi/yaml.md

@@ -12,6 +12,7 @@ slug: /connectors/dashboard/powerbi/yaml
 | Owners     | {% icon iconName="cross" /%} |
 | Tags       | {% icon iconName="cross" /%} |
 | Datamodels | {% icon iconName="check" /%} |
+| Projects   | {% icon iconName="check" /%} |
 | Lineage    | {% icon iconName="check" /%} |
 
 In this section, we provide guides and references to use the PowerBI connector.
@@ -206,6 +207,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
 
 - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
 - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
+- **projectFilterPattern**: Filter the PowerBI dashboards, reports, tiles and data sources by projects(In case of PowerBI, projects correspond to workspaces). Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
 - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
 - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
 - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@@ -281,6 +283,13 @@ source:
       #   excludes:
       #     - chart3
       #     - chart4
+      # projectFilterPattern:
+      #   includes:
+      #     - project1
+      #     - project2
+      #   excludes:
+      #     - project3
+      #     - project4
 ```
 ```yaml {% srNumber=10 %}
 sink:

openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/qliksense/yaml.md

@@ -12,6 +12,7 @@ slug: /connectors/dashboard/qliksense/yaml
 | Owners     | {% icon iconName="cross" /%} |
 | Tags       | {% icon iconName="cross" /%} |
 | Datamodels | {% icon iconName="check" /%} |
+| Projects   | {% icon iconName="cross" /%} |
 | Lineage    | {% icon iconName="check" /%} |
 
 In this section, we provide guides and references to use the PowerBI connector.
@@ -145,6 +146,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
 
 - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
 - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
+- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
 - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
 - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
 - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@@ -221,6 +223,13 @@ source:
       #   excludes:
       #     - chart3
       #     - chart4
+      # projectFilterPattern:
+      #   includes:
+      #     - project1
+      #     - project2
+      #   excludes:
+      #     - project3
+      #     - project4
 ```
 ```yaml {% srNumber=8 %}
 sink:

openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/quicksight/yaml.md

@@ -12,6 +12,7 @@ slug: /connectors/dashboard/quicksight/yaml
 | Owners     | {% icon iconName="cross" /%} |
 | Tags       | {% icon iconName="cross" /%} |
 | Datamodels | {% icon iconName="cross" /%} |
+| Projects   | {% icon iconName="cross" /%} |
 | Lineage    | {% icon iconName="check" /%} |
 
 In this section, we provide guides and references to use the QuickSight connector.
@@ -145,6 +146,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
 
 - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
 - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
+- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
 - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
 - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
 - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@@ -212,6 +214,13 @@ source:
       #   excludes:
       #     - chart3
       #     - chart4
+      # projectFilterPattern:
+      #   includes:
+      #     - project1
+      #     - project2
+      #   excludes:
+      #     - project3
+      #     - project4
 
 ```yaml {% srNumber=6 %}
 sink:

openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/redash/yaml.md

@@ -12,6 +12,7 @@ slug: /connectors/dashboard/redash/yaml
 | Owners     | {% icon iconName="check" /%} |
 | Tags       | {% icon iconName="check" /%} |
 | Datamodels | {% icon iconName="cross" /%} |
+| Projects   | {% icon iconName="cross" /%} |
 | Lineage    | {% icon iconName="check" /%} |
 
 In this section, we provide guides and references to use the Redash connector.
@@ -92,6 +93,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
 
 - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
 - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
+- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
 - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
 - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
 - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@@ -154,6 +156,13 @@ source:
       #   excludes:
       #     - chart3
       #     - chart4
+      # projectFilterPattern:
+      #   includes:
+      #     - project1
+      #     - project2
+      #   excludes:
+      #     - project3
+      #     - project4
 ```
 ```yaml {% srNumber=6 %}
 sink:

openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/superset/yaml.md

@@ -12,6 +12,7 @@ slug: /connectors/dashboard/superset/yaml
 | Owners     | {% icon iconName="check" /%} |
 | Tags       | {% icon iconName="cross" /%} |
 | Datamodels | {% icon iconName="cross" /%} |
+| Projects   | {% icon iconName="cross" /%} |
 | Lineage    | {% icon iconName="check" /%} |
 
 In this section, we provide guides and references to use the Superset connector.
@@ -141,6 +142,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
 
 - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
 - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
+- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
 - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
 - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
 - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@@ -218,6 +220,13 @@ source:
       #   excludes:
       #     - chart3
       #     - chart4
+      # projectFilterPattern:
+      #   includes:
+      #     - project1
+      #     - project2
+      #   excludes:
+      #     - project3
+      #     - project4
 ```
 ```yaml {% srNumber=5 %}
 sink:

openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/tableau/yaml.md

@@ -12,6 +12,7 @@ slug: /connectors/dashboard/tableau/yaml
 | Owners     | {% icon iconName="check" /%} |
 | Tags       | {% icon iconName="check" /%} |
 | Datamodels | {% icon iconName="check" /%} |
+| Projects   | {% icon iconName="check" /%} |
 | Lineage    | {% icon iconName="check" /%} |
 
 In this section, we provide guides and references to use the Tableau connector.
@@ -129,6 +130,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
 
 - **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
 - **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
+- **projectFilterPattern**: Filter the tableau dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
 - **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
 - **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
 - **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@@ -218,6 +220,13 @@ source:
       #   excludes:
       #     - datamodel3
       #     - datamodel4
+      # projectFilterPattern:
+      #   includes:
+      #     - project1
+      #     - project2
+      #   excludes:
+      #     - project3
+      #     - project4
 ```
 ```yaml {% srNumber=9 %}
 sink:

openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json

@@ -30,6 +30,10 @@
       "description": "Regex exclude or include data models that matches the pattern.",
       "$ref": "../type/filterPattern.json#/definitions/filterPattern"
     },
+    "projectFilterPattern": {
+      "description": "Regex to exclude or include projects that matches the pattern.",
+      "$ref": "../type/filterPattern.json#/definitions/filterPattern"
+    },
     "dbServiceNames": {
       "title": "Database Service Names List",
       "description": "List of Database Service Names for creation of lineage",