yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-11 16:58:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added project filter pattern to dashboard entity (#12925)

Browse Source
This commit is contained in:
Onkar Ravgan 2023-09-05 11:01:28 +05:30 committed by GitHub
parent dc8e59eba8
commit a3ca8b6e66
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 210 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ingestion/examples/workflows/metabase.yaml
View File

@ -12,6 +12,7 @@ source:
type: DashboardMetadata
dashboardFilterPattern: {}
chartFilterPattern: {}
projectFilterPattern: {}
sink:
type: metadata-rest
config: {}

4
ingestion/examples/workflows/powerbi.yaml
View File

@ -25,6 +25,10 @@ source:
includes:
- Supplier Quality Analysis Sample
- "Customer"
projectFilterPattern:
includes:
- Supplier Quality Analysis Sample
- "Customer"
sink:
type: metadata-rest
config: {}

1
ingestion/examples/workflows/tableau.yaml
View File

@ -24,6 +24,7 @@ source:
type: DashboardMetadata
dashboardFilterPattern: {}
chartFilterPattern: {}
projectFilterPattern: {}
sink:
type: metadata-rest
config: {}

25
ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py
View File

@ -63,7 +63,7 @@ from metadata.ingestion.models.topology import (
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
from metadata.utils import fqn
from metadata.utils.filters import filter_by_dashboard
from metadata.utils.filters import filter_by_dashboard, filter_by_project
from metadata.utils.logger import ingestion_logger
logger = ingestion_logger()
@ -447,6 +447,18 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
try:
dashboard_details = self.get_dashboard_details(dashboard)
self.context.project_name = ( # pylint: disable=assignment-from-none
self.get_project_name(dashboard_details=dashboard_details)
)
if self.context.project_name and filter_by_project(
self.source_config.projectFilterPattern,
self.context.project_name,
):
self.status.filter(
self.context.project_name,
"Project / Workspace Filtered Out",
)
continue
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(
@ -501,3 +513,14 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
return None
return database_schema_name
def get_project_name( # pylint: disable=unused-argument, useless-return
self, dashboard_details: Any
) -> Optional[str]:
"""
Get the project / workspace / folder / collection name of the dashboard
"""
logger.debug(
f"Projects are not supported for {self.service_connection.type.name}"
)
return None

16
ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py
View File

@ -11,7 +11,7 @@
"""Metabase source module"""
import traceback
from typing import Iterable, List, Optional
from typing import Any, Iterable, List, Optional
from metadata.generated.schema.api.data.createChart import CreateChartRequest
from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
@ -100,17 +100,17 @@ class MetabaseSource(DashboardServiceSource):
"""
return self.client.get_dashboard_details(dashboard.id)
def _get_collection_name(self, collection_id: Optional[str]) -> Optional[str]:
def get_project_name(self, dashboard_details: Any) -> Optional[str]:
"""
Method to search the dataset using id in the workspace dict
Method to get the project name by searching the dataset using id in the workspace dict
"""
try:
if collection_id:
if dashboard_details.collection_id:
collection_name = next(
(
collection.name
for collection in self.collections
if collection.id == collection_id
if collection.id == dashboard_details.collection_id
),
None,
)
@ -118,7 +118,7 @@ class MetabaseSource(DashboardServiceSource):
except Exception as exc: # pylint: disable=broad-except
logger.debug(traceback.format_exc())
logger.warning(
f"Error fetching the collection details for [{collection_id}]: {exc}"
f"Error fetching the collection details for [{dashboard_details.collection_id}]: {exc}"
)
return None
@ -138,9 +138,7 @@ class MetabaseSource(DashboardServiceSource):
sourceUrl=dashboard_url,
displayName=dashboard_details.name,
description=dashboard_details.description,
project=self._get_collection_name(
collection_id=dashboard_details.collection_id
),
project=self.context.project_name,
charts=[
fqn.build(
self.metadata,

57
ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py
View File

@ -44,6 +44,7 @@ from metadata.ingestion.api.steps import InvalidSourceException
from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
from metadata.ingestion.source.dashboard.powerbi.models import (
Dataset,
Group,
PowerBIDashboard,
PowerBIReport,
PowerBiTable,
@ -54,6 +55,7 @@ from metadata.utils.filters import (
filter_by_chart,
filter_by_dashboard,
filter_by_datamodel,
filter_by_project,
)
from metadata.utils.helpers import clean_uri
from metadata.utils.logger import ingestion_logger
@ -80,12 +82,32 @@ class PowerbiSource(DashboardServiceSource):
def prepare(self):
if self.service_connection.useAdminApis:
self.get_admin_workspace_data()
groups = self.get_admin_workspace_data()
else:
self.get_org_workspace_data()
groups = self.get_org_workspace_data()
if groups:
self.workspace_data = self.get_filtered_workspaces(groups)
return super().prepare()
def get_org_workspace_data(self):
def get_filtered_workspaces(self, groups: List[Group]) -> List[Group]:
"""
Method to get the workspaces filtered by project filter pattern
"""
filtered_groups = []
for group in groups:
if filter_by_project(
self.source_config.projectFilterPattern,
group.name,
):
self.status.filter(
group.name,
"Workspace Filtered Out",
)
continue
filtered_groups.append(group)
return filtered_groups
def get_org_workspace_data(self) -> Optional[List[Group]]:
"""
fetch all the group workspace ids
"""
@ -121,12 +143,13 @@ class PowerbiSource(DashboardServiceSource):
)
or []
)
self.workspace_data = groups
return groups
def get_admin_workspace_data(self):
def get_admin_workspace_data(self) -> Optional[List[Group]]:
"""
fetch all the workspace ids
"""
groups = []
workspaces = self.client.fetch_all_workspaces()
if workspaces:
workspace_id_list = [workspace.id for workspace in workspaces]
@ -155,7 +178,7 @@ class PowerbiSource(DashboardServiceSource):
response = self.client.fetch_workspace_scan_result(
scan_id=workspace_scan.id
)
self.workspace_data.extend(
groups.extend(
[
active_workspace
for active_workspace in response.workspaces
@ -166,7 +189,8 @@ class PowerbiSource(DashboardServiceSource):
logger.error("Error in fetching dashboards and charts")
count += 1
else:
logger.error("Unable to fetch any Powerbi workspaces")
logger.error("Unable to fetch any PowerBI workspaces")
return groups or None
@classmethod
def create(cls, config_dict, metadata_config: OpenMetadataConnection):
@ -180,7 +204,7 @@ class PowerbiSource(DashboardServiceSource):
def get_dashboard(self) -> Any:
"""
Method to iterate through dashboard lists filter dashbaords & yield dashboard details
Method to iterate through dashboard lists filter dashboards & yield dashboard details
"""
for workspace in self.workspace_data:
self.context.workspace = workspace
@ -369,7 +393,7 @@ class PowerbiSource(DashboardServiceSource):
workspace_id=self.context.workspace.id,
dashboard_id=dashboard_details.id,
),
project=str(self.context.workspace.name),
project=self.get_project_name(dashboard_details=dashboard_details),
displayName=dashboard_details.displayName,
dashboardType=DashboardType.Dashboard,
charts=[
@ -391,7 +415,7 @@ class PowerbiSource(DashboardServiceSource):
workspace_id=self.context.workspace.id,
dashboard_id=dashboard_details.id,
),
project=str(self.context.workspace.name),
project=self.get_project_name(dashboard_details=dashboard_details),
displayName=dashboard_details.name,
service=self.context.dashboard_service.fullyQualifiedName.__root__,
)
@ -668,3 +692,16 @@ class PowerbiSource(DashboardServiceSource):
return next(iter(workspace_names), None)
return None
def get_project_name(self, dashboard_details: Any) -> Optional[str]:
"""
Get the project / workspace / folder / collection name of the dashboard
"""
try:
return str(self.context.workspace.name)
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(
f"Error fetching project name for {dashboard_details.id}: {exc}"
)
return None

17
ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py
View File

@ -12,7 +12,7 @@
Tableau source module
"""
import traceback
from typing import Iterable, List, Optional, Set
from typing import Any, Iterable, List, Optional, Set
from metadata.generated.schema.api.data.createChart import CreateChartRequest
from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
@ -215,7 +215,7 @@ class TableauSource(DashboardServiceSource):
name=dashboard_details.id,
displayName=dashboard_details.name,
description=dashboard_details.description,
project=dashboard_details.project.name,
project=self.get_project_name(dashboard_details=dashboard_details),
charts=[
fqn.build(
self.metadata,
@ -460,3 +460,16 @@ class TableauSource(DashboardServiceSource):
logger.debug(traceback.format_exc())
logger.warning(f"Error to yield datamodel column: {exc}")
return datasource_columns
def get_project_name(self, dashboard_details: Any) -> Optional[str]:
"""
Get the project / workspace / folder / collection name of the dashboard
"""
try:
return dashboard_details.project.name
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(
f"Error fetching project name for {dashboard_details.id}: {exc}"
)
return None

15
ingestion/src/metadata/utils/filters.py
View File

@ -240,6 +240,21 @@ def filter_by_datamodel(
return _filter(datamodel_filter_pattern, datamodel_name)
def filter_by_project(
project_filter_pattern: Optional[FilterPattern], project_name: str
) -> bool:
"""
Return True if the project needs to be filtered, False otherwise
Include takes precedence over exclude
:param project_filter_pattern: Model defining project filtering logic
:param project_name: project name
:return: True for filtering, False otherwise
"""
return _filter(project_filter_pattern, project_name)
def filter_by_search_index(
search_index_filter_pattern: Optional[FilterPattern], search_index_name: str
) -> bool:

2
ingestion/tests/unit/topology/dashboard/test_metabase.py
View File

@ -169,6 +169,7 @@ EXPECTED_DASHBOARD = [
sourceUrl="http://metabase.com/dashboard/1-test-db",
charts=[],
service=FullyQualifiedEntityName(__root__="mock_metabase"),
project="Test Collection",
)
]
@ -227,6 +228,7 @@ class MetabaseUnitTest(TestCase):
)
self.metabase.client = SimpleNamespace()
self.metabase.context.__dict__["dashboard_service"] = MOCK_DASHBOARD_SERVICE
self.metabase.context.__dict__["project_name"] = "Test Collection"
def test_dashboard_name(self):
assert (

9
openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/domo-dashboard/yaml.md
View File

@ -12,6 +12,7 @@ slug: /connectors/dashboard/domo-dashboard/yaml
| Owners | {% icon iconName="check" /%} |
| Tags | {% icon iconName="cross" /%} |
| Datamodels | {% icon iconName="cross" /%} |
| Projects | {% icon iconName="cross" /%} |
| Lineage | {% icon iconName="cross" /%} |
In this section, we provide guides and references to use the DomoDashboard connector.
@ -101,6 +102,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
- **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
- **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
- **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
- **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
- **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@ -167,6 +169,13 @@ source:
# excludes:
# - chart3
# - chart4
# projectFilterPattern:
# includes:
# - project1
# - project2
# excludes:
# - project3
# - project4
```
```yaml {% srNumber=7 %}
sink:

9
openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/looker/yaml.md
View File

@ -12,6 +12,7 @@ slug: /connectors/dashboard/looker/yaml
| Owners | {% icon iconName="check" /%} |
| Tags | {% icon iconName="cross" /%} |
| Datamodels | {% icon iconName="check" /%} |
| Projects | {% icon iconName="cross" /%} |
| Lineage | {% icon iconName="check" /%} |
In this section, we provide guides and references to use the Looker connector.
@ -124,6 +125,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
- **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
- **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
- **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
- **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
- **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@ -191,6 +193,13 @@ source:
# excludes:
# - chart3
# - chart4
# projectFilterPattern:
# includes:
# - project1
# - project2
# excludes:
# - project3
# - project4
```
```yaml {% srNumber=6 %}

9
openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/metabase/yaml.md
View File

@ -12,6 +12,7 @@ slug: /connectors/dashboard/metabase/yaml
| Owners | {% icon iconName="cross" /%} |
| Tags | {% icon iconName="cross" /%} |
| Datamodels | {% icon iconName="cross" /%} |
| Projects | {% icon iconName="check" /%} |
| Lineage | {% icon iconName="check" /%} |
In this section, we provide guides and references to use the Metabase connector.
@ -90,6 +91,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
- **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
- **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
- **projectFilterPattern**: Filter the Metabase dashboards and charts by projects (In case of Metabase, projects corresponds to Collections). Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
- **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
- **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
- **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@ -150,6 +152,13 @@ source:
# excludes:
# - chart3
# - chart4
# projectFilterPattern:
# includes:
# - project1
# - project2
# excludes:
# - project3
# - project4
```
```yaml {% srNumber=5 %}

9
openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/mode/yaml.md
View File

@ -12,6 +12,7 @@ slug: /connectors/dashboard/mode/yaml
| Owners | {% icon iconName="cross" /%} |
| Tags | {% icon iconName="cross" /%} |
| Datamodels | {% icon iconName="cross" /%} |
| Projects | {% icon iconName="cross" /%} |
| Lineage | {% icon iconName="check" /%} |
In this section, we provide guides and references to use the Mode connector.
@ -111,6 +112,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
- **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
- **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
- **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
- **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
- **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@ -173,6 +175,13 @@ source:
# excludes:
# - chart3
# - chart4
# projectFilterPattern:
# includes:
# - project1
# - project2
# excludes:
# - project3
# - project4
```
```yaml {% srNumber=6 %}
sink:

9
openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/powerbi/yaml.md
View File

@ -12,6 +12,7 @@ slug: /connectors/dashboard/powerbi/yaml
| Owners | {% icon iconName="cross" /%} |
| Tags | {% icon iconName="cross" /%} |
| Datamodels | {% icon iconName="check" /%} |
| Projects | {% icon iconName="check" /%} |
| Lineage | {% icon iconName="check" /%} |
In this section, we provide guides and references to use the PowerBI connector.
@ -206,6 +207,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
- **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
- **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
- **projectFilterPattern**: Filter the PowerBI dashboards, reports, tiles and data sources by projects(In case of PowerBI, projects correspond to workspaces). Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
- **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
- **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
- **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@ -281,6 +283,13 @@ source:
# excludes:
# - chart3
# - chart4
# projectFilterPattern:
# includes:
# - project1
# - project2
# excludes:
# - project3
# - project4
```
```yaml {% srNumber=10 %}
sink:

9
openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/qliksense/yaml.md
View File

@ -12,6 +12,7 @@ slug: /connectors/dashboard/qliksense/yaml
| Owners | {% icon iconName="cross" /%} |
| Tags | {% icon iconName="cross" /%} |
| Datamodels | {% icon iconName="check" /%} |
| Projects | {% icon iconName="cross" /%} |
| Lineage | {% icon iconName="check" /%} |
In this section, we provide guides and references to use the PowerBI connector.
@ -145,6 +146,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
- **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
- **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
- **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
- **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
- **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@ -221,6 +223,13 @@ source:
# excludes:
# - chart3
# - chart4
# projectFilterPattern:
# includes:
# - project1
# - project2
# excludes:
# - project3
# - project4
```
```yaml {% srNumber=8 %}
sink:

9
openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/quicksight/yaml.md
View File

@ -12,6 +12,7 @@ slug: /connectors/dashboard/quicksight/yaml
| Owners | {% icon iconName="cross" /%} |
| Tags | {% icon iconName="cross" /%} |
| Datamodels | {% icon iconName="cross" /%} |
| Projects | {% icon iconName="cross" /%} |
| Lineage | {% icon iconName="check" /%} |
In this section, we provide guides and references to use the QuickSight connector.
@ -145,6 +146,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
- **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
- **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
- **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
- **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
- **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@ -212,6 +214,13 @@ source:
# excludes:
# - chart3
# - chart4
# projectFilterPattern:
# includes:
# - project1
# - project2
# excludes:
# - project3
# - project4
```yaml {% srNumber=6 %}
sink:

9
openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/redash/yaml.md
View File

@ -12,6 +12,7 @@ slug: /connectors/dashboard/redash/yaml
| Owners | {% icon iconName="check" /%} |
| Tags | {% icon iconName="check" /%} |
| Datamodels | {% icon iconName="cross" /%} |
| Projects | {% icon iconName="cross" /%} |
| Lineage | {% icon iconName="check" /%} |
In this section, we provide guides and references to use the Redash connector.
@ -92,6 +93,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
- **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
- **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
- **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
- **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
- **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@ -154,6 +156,13 @@ source:
# excludes:
# - chart3
# - chart4
# projectFilterPattern:
# includes:
# - project1
# - project2
# excludes:
# - project3
# - project4
```
```yaml {% srNumber=6 %}
sink:

9
openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/superset/yaml.md
View File

@ -12,6 +12,7 @@ slug: /connectors/dashboard/superset/yaml
| Owners | {% icon iconName="check" /%} |
| Tags | {% icon iconName="cross" /%} |
| Datamodels | {% icon iconName="cross" /%} |
| Projects | {% icon iconName="cross" /%} |
| Lineage | {% icon iconName="check" /%} |
In this section, we provide guides and references to use the Superset connector.
@ -141,6 +142,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
- **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
- **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
- **projectFilterPattern**: Filter the dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
- **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
- **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
- **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@ -218,6 +220,13 @@ source:
# excludes:
# - chart3
# - chart4
# projectFilterPattern:
# includes:
# - project1
# - project2
# excludes:
# - project3
# - project4
```
```yaml {% srNumber=5 %}
sink:

9
openmetadata-docs/content/v1.2.0-SNAPSHOT/connectors/dashboard/tableau/yaml.md
View File

@ -12,6 +12,7 @@ slug: /connectors/dashboard/tableau/yaml
| Owners | {% icon iconName="check" /%} |
| Tags | {% icon iconName="check" /%} |
| Datamodels | {% icon iconName="check" /%} |
| Projects | {% icon iconName="check" /%} |
| Lineage | {% icon iconName="check" /%} |
In this section, we provide guides and references to use the Tableau connector.
@ -129,6 +130,7 @@ The `sourceConfig` is defined [here](https://github.com/open-metadata/OpenMetada
- **dbServiceNames**: Database Service Names for ingesting lineage if the source supports it.
- **dashboardFilterPattern**, **chartFilterPattern**, **dataModelFilterPattern**: Note that all of them support regex as include or exclude. E.g., "My dashboard, My dash.*, .*Dashboard".
- **projectFilterPattern**: Filter the tableau dashboards, charts and data sources by projects. Note that all of them support regex as include or exclude. E.g., "My project, My proj.*, .*Project".
- **includeOwners**: Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.
- **includeTags**: Set the 'Include Tags' toggle to control whether to include tags in metadata ingestion.
- **includeDataModels**: Set the 'Include Data Models' toggle to control whether to include tags as part of metadata ingestion.
@ -218,6 +220,13 @@ source:
# excludes:
# - datamodel3
# - datamodel4
# projectFilterPattern:
# includes:
# - project1
# - project2
# excludes:
# - project3
# - project4
```
```yaml {% srNumber=9 %}
sink:

4
openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json
View File

@ -30,6 +30,10 @@
"description": "Regex exclude or include data models that matches the pattern.",
"$ref": "../type/filterPattern.json#/definitions/filterPattern"
},
"projectFilterPattern": {
"description": "Regex to exclude or include projects that matches the pattern.",
"$ref": "../type/filterPattern.json#/definitions/filterPattern"
},
"dbServiceNames": {
"title": "Database Service Names List",
"description": "List of Database Service Names for creation of lineage",