yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-16 13:07:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

multiple-database-service-names-added-in-dashboard (#6591)

Browse Source 
* multiple-database-service-names-added-in-dashboard

* code-formatted

* code-smell-removed

* code-formatted

* improved-dbserviceName-handling-in-dashboard

* fix checks failing

Co-authored-by: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com>
This commit is contained in:
Abhishek Pandey 2022-08-08 22:21:28 +05:30 committed by GitHub
parent d18d1c548e
commit a08201e383
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 57 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
catalog-rest-service/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json
View File

@ -25,10 +25,10 @@
"description": "Regex exclude tables or databases that matches the pattern.",
"$ref": "../type/filterPattern.json#/definitions/filterPattern"
},
"dbServiceName": {
"title": "Database Service Name",
"description": "Database Service Name for creation of lineage",
"type": "string"
"dbServiceNames": {
"title": "Database Service Name List",
"description": "List of Database Service Name for creation of lineage",
"type": "array"
}
},
"additionalProperties": false

2
ingestion/examples/workflows/metabase.yaml
View File

@ -9,7 +9,7 @@ source:
hostPort: http://hostPort
sourceConfig:
config:
dbServiceName: Database Service Name to create Lineage
dbServiceNames: list of database service name to create Lineage
dashboardFilterPattern: {}
chartFilterPattern: {}
sink:

2
ingestion/examples/workflows/mode.yaml
View File

@ -20,7 +20,7 @@ source:
includes:
- Supplier Quality Analysis Sample
- "Customer"
dbServiceName: local_redshift
dbServiceNames: [local_redshift]
sink:
type: metadata-rest
config: {}

8
ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py
View File

@ -167,7 +167,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
@abstractmethod
def yield_dashboard_lineage_details(
self, dashboard_details: Any
self, dashboard_details: Any, db_service_name: str
) -> Optional[Iterable[AddLineageRequest]]:
"""
Get lineage between dashboard and data sources
@ -205,8 +205,10 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
"""
Yields lineage if config is enabled
"""
if self.source_config.dbServiceName:
yield from self.yield_dashboard_lineage_details(dashboard_details) or []
for db_service_name in self.source_config.dbServiceNames or []:
yield from self.yield_dashboard_lineage_details(
dashboard_details, db_service_name
) or []
def yield_tag(self, *args, **kwargs) -> Optional[Iterable[OMetaTagAndCategory]]:
"""

4
ingestion/src/metadata/ingestion/source/dashboard/looker.py
View File

@ -182,7 +182,7 @@ class LookerSource(DashboardServiceSource):
return dashboard_sources
def yield_dashboard_lineage_details(
self, dashboard_details: LookerDashboard
self, dashboard_details: LookerDashboard, db_service_name
) -> Optional[Iterable[AddLineageRequest]]:
"""
Get lineage between charts and data sources.
@ -212,7 +212,7 @@ class LookerSource(DashboardServiceSource):
from_fqn = fqn.build(
self.metadata,
entity_type=Table,
service_name=self.source_config.dbServiceName,
service_name=db_service_name,
database_name=source_elements["database"],
schema_name=source_elements["database_schema"],
table_name=source_elements["table"],

8
ingestion/src/metadata/ingestion/source/dashboard/metabase.py
View File

@ -173,14 +173,14 @@ class MetabaseSource(DashboardServiceSource):
continue
def yield_dashboard_lineage_details(
self, dashboard_details: dict
self, dashboard_details: dict, db_service_name
) -> Optional[Iterable[AddLineageRequest]]:
"""Get lineage method
Args:
dashboard_details
"""
if not self.source_config.dbServiceName:
if not db_service_name:
return
chart_list, dashboard_name = (
dashboard_details["ordered_cards"],
@ -216,7 +216,7 @@ class MetabaseSource(DashboardServiceSource):
from_entities = search_table_entities(
metadata=self.metadata,
database=database["details"]["dbname"],
service_name=self.source_config.dbServiceName,
service_name=db_service_name,
database_schema=database_schema_name,
table=table,
)
@ -256,7 +256,7 @@ class MetabaseSource(DashboardServiceSource):
from_fqn = fqn.build(
self.metadata,
entity_type=Table,
service_name=self.source_config.dbServiceName,
service_name=db_service_name,
database_name=table["db"]["details"]["dbname"],
schema_name=table.get("schema"),
table_name=table.get("display_name"),

4
ingestion/src/metadata/ingestion/source/dashboard/mode.py
View File

@ -116,7 +116,7 @@ class ModeSource(DashboardServiceSource):
)
def yield_dashboard_lineage_details(
self, dashboard_details: dict
self, dashboard_details: dict, db_service_name: str
) -> Optional[Iterable[AddLineageRequest]]:
"""Get lineage method
@ -147,7 +147,7 @@ class ModeSource(DashboardServiceSource):
from_entities = search_table_entities(
metadata=self.metadata,
database=data_source.get(mode_client.DATABASE),
service_name=self.source_config.dbServiceName,
service_name=db_service_name,
database_schema=database_schema_name,
table=table,
)

4
ingestion/src/metadata/ingestion/source/dashboard/powerbi.py
View File

@ -98,7 +98,7 @@ class PowerbiSource(DashboardServiceSource):
)
def yield_dashboard_lineage_details(
self, dashboard_details: dict
self, dashboard_details: dict, db_service_name: str
) -> Optional[Iterable[AddLineageRequest]]:
"""
Get lineage between dashboard and data sources
@ -119,7 +119,7 @@ class PowerbiSource(DashboardServiceSource):
from_fqn = fqn.build(
self.metadata,
entity_type=Database,
service_name=self.source_config.dbServiceName,
service_name=db_service_name,
database_name=database_name,
)
from_entity = self.metadata.get_by_name(

4
ingestion/src/metadata/ingestion/source/dashboard/redash.py
View File

@ -112,7 +112,7 @@ class RedashSource(DashboardServiceSource):
self.status.scanned(dashboard_details["name"])
def yield_dashboard_lineage_details(
self, dashboard_details: dict
self, dashboard_details: dict, db_service_name: str
) -> Optional[Iterable[AddLineageRequest]]:
"""
Get lineage between dashboard and data sources
@ -133,7 +133,7 @@ class RedashSource(DashboardServiceSource):
table_entities = search_table_entities(
metadata=self.metadata,
database=None,
service_name=self.source_config.dbServiceName,
service_name=db_service_name,
database_schema=database_schema,
table=table_name,
)

14
ingestion/src/metadata/ingestion/source/dashboard/superset.py
View File

@ -185,7 +185,7 @@ class SupersetSource(DashboardServiceSource):
return []
def yield_dashboard_lineage_details(
self, dashboard_details: dict
self, dashboard_details: dict, db_service_name: str
) -> Optional[Iterable[AddLineageRequest]]:
"""
Get lineage between dashboard and data sources
@ -193,7 +193,9 @@ class SupersetSource(DashboardServiceSource):
for chart_id in self._get_charts_of_dashboard(dashboard_details):
chart_json = self.all_charts.get(chart_id)
datasource_fqn = (
self._get_datasource_fqn(chart_json.get("datasource_id"))
self._get_datasource_fqn(
chart_json.get("datasource_id"), db_service_name
)
if chart_json.get("datasource_id")
else None
)
@ -258,8 +260,10 @@ class SupersetSource(DashboardServiceSource):
)
yield chart
def _get_datasource_fqn(self, datasource_id: str) -> Optional[str]:
if not self.source_config.dbServiceName:
def _get_datasource_fqn(
self, datasource_id: str, db_service_name: str
) -> Optional[str]:
if not db_service_name:
return
try:
datasource_json = self.client.fetch_datasource(datasource_id)
@ -272,7 +276,7 @@ class SupersetSource(DashboardServiceSource):
table_name=datasource_json["result"]["table_name"],
schema_name=datasource_json["result"]["schema"],
database_name=database_json["result"]["parameters"]["database"],
service_name=self.source_config.dbServiceName,
service_name=db_service_name,
)
return dataset_fqn
except KeyError:

4
ingestion/src/metadata/ingestion/source/dashboard/tableau.py
View File

@ -213,7 +213,7 @@ class TableauSource(DashboardServiceSource):
)
def yield_dashboard_lineage_details(
self, dashboard_details: dict
self, dashboard_details: dict, db_service_name: str
) -> Optional[Iterable[AddLineageRequest]]:
"""
Get lineage between dashboard and data sources
@ -246,7 +246,7 @@ class TableauSource(DashboardServiceSource):
from_fqn = fqn.build(
self.metadata,
entity_type=Table,
service_name=self.source_config.dbServiceName,
service_name=db_service_name,
schema_name=schema_name,
table_name=table_name,
database_name=None,

2
openmetadata-docs/ingestion/connectors/metabase/ingestion.yaml
View File

@ -7,7 +7,7 @@ source:
username: <username>
password: <password>
hostPort: <hostPort>
dbServiceName: <dbServiceName>
dbServiceNames: [<dbServiceName>]
sourceConfig:
config:
dashboardFilterPattern: <dashboard name regex list>

2
openmetadata-docs/ingestion/connectors/superset/ingestion.yaml
View File

@ -6,7 +6,7 @@ source:
hostPort: http://localhost:8080
username: admin
password: admin
dbServiceName: local_mysql
dbServiceNames: [local_mysql]
type: Superset
sourceConfig:
config:

10
openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx
View File

@ -91,8 +91,8 @@ const AddIngestion = ({
const [ingestSampleData, setIngestSampleData] = useState(
(data?.sourceConfig.config as ConfigClass)?.generateSampleData ?? true
);
const [databaseServiceName, setDatabaseServiceName] = useState(
(data?.sourceConfig.config as ConfigClass)?.dbServiceName ?? ''
const [databaseServiceNames, setDatabaseServiceNames] = useState(
(data?.sourceConfig.config as ConfigClass)?.dbServiceNames ?? []
);
const [description, setDescription] = useState(data?.description ?? '');
const [repeatFrequency, setRepeatFrequency] = useState(
@ -418,7 +418,7 @@ const AddIngestion = ({
dashboardFilterPattern,
showDashboardFilter
),
dbServiceName: databaseServiceName,
dbServiceNames: databaseServiceNames,
type: ConfigType.DashboardMetadata,
};
}
@ -611,12 +611,12 @@ const AddIngestion = ({
chartFilterPattern={chartFilterPattern}
dashboardFilterPattern={dashboardFilterPattern}
databaseFilterPattern={databaseFilterPattern}
databaseServiceName={databaseServiceName}
databaseServiceNames={databaseServiceNames}
description={description}
enableDebugLog={enableDebugLog}
getExcludeValue={getExcludeValue}
getIncludeValue={getIncludeValue}
handleDatasetServiceName={(val) => setDatabaseServiceName(val)}
handleDatasetServiceName={(val) => setDatabaseServiceNames(val)}
handleDescription={(val) => setDescription(val)}
handleEnableDebugLog={() => setEnableDebugLog((pre) => !pre)}
handleIncludeLineage={() => setIncludeLineage((pre) => !pre)}

2
openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx
View File

@ -92,7 +92,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = {
handleEnableDebugLog: jest.fn(),
ingestSampleData: false,
handleIngestSampleData: jest.fn(),
databaseServiceName: '',
databaseServiceNames: [''],
handleDatasetServiceName: jest.fn(),
threadCount: 5,
handleThreadCount: jest.fn(),

16
openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx
View File

@ -28,7 +28,7 @@ import { ConfigureIngestionProps } from '../addIngestion.interface';
const ConfigureIngestion = ({
ingestionName,
description = '',
databaseServiceName,
databaseServiceNames,
databaseFilterPattern,
dashboardFilterPattern,
schemaFilterPattern,
@ -255,6 +255,16 @@ const ConfigureIngestion = ({
);
};
const handleDashBoardServiceNames = (inputValue: string) => {
const separator = ',';
const databaseNames = inputValue.includes(separator)
? inputValue.split(separator)
: Array(inputValue);
if (databaseNames) handleDatasetServiceName(databaseNames);
};
const getDashboardDBServiceName = () => {
return (
<Field>
@ -270,8 +280,8 @@ const ConfigureIngestion = ({
id="name"
name="name"
type="text"
value={databaseServiceName}
onChange={(e) => handleDatasetServiceName(e.target.value)}
value={databaseServiceNames}
onChange={(e) => handleDashBoardServiceNames(e.target.value)}
/>
{getSeparator('')}
</Field>

4
openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts
View File

@ -57,7 +57,7 @@ export interface AddIngestionProps {
export interface ConfigureIngestionProps {
ingestionName: string;
description?: string;
databaseServiceName: string;
databaseServiceNames: string[];
serviceCategory: ServiceCategory;
databaseFilterPattern: FilterPattern;
dashboardFilterPattern: FilterPattern;
@ -86,7 +86,7 @@ export interface ConfigureIngestionProps {
stageFileLocation: string;
resultLimit: number;
handleIngestionName: (value: string) => void;
handleDatasetServiceName: (value: string) => void;
handleDatasetServiceName: (value: string[]) => void;
handleDescription?: (value: string) => void;
handleIncludeLineage: () => void;
handleIncludeView: () => void;