From a08201e383c1ae288a6b5c8c7ec40bddfcefb8ab Mon Sep 17 00:00:00 2001 From: Abhishek Pandey Date: Mon, 8 Aug 2022 22:21:28 +0530 Subject: [PATCH] multiple-database-service-names-added-in-dashboard (#6591) * multiple-database-service-names-added-in-dashboard * code-formatted * code-smell-removed * code-formatted * improved-dbserviceName-handling-in-dashboard * fix checks failing Co-authored-by: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com> --- .../dashboardServiceMetadataPipeline.json | 8 ++++---- ingestion/examples/workflows/metabase.yaml | 2 +- ingestion/examples/workflows/mode.yaml | 2 +- .../source/dashboard/dashboard_service.py | 8 +++++--- .../ingestion/source/dashboard/looker.py | 4 ++-- .../ingestion/source/dashboard/metabase.py | 8 ++++---- .../metadata/ingestion/source/dashboard/mode.py | 4 ++-- .../ingestion/source/dashboard/powerbi.py | 4 ++-- .../ingestion/source/dashboard/redash.py | 4 ++-- .../ingestion/source/dashboard/superset.py | 14 +++++++++----- .../ingestion/source/dashboard/tableau.py | 4 ++-- .../ingestion/connectors/metabase/ingestion.yaml | 2 +- .../ingestion/connectors/superset/ingestion.yaml | 2 +- .../AddIngestion/AddIngestion.component.tsx | 10 +++++----- .../Steps/ConfigureIngestion.test.tsx | 2 +- .../AddIngestion/Steps/ConfigureIngestion.tsx | 16 +++++++++++++--- .../AddIngestion/addIngestion.interface.ts | 4 ++-- 17 files changed, 57 insertions(+), 41 deletions(-) diff --git a/catalog-rest-service/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json b/catalog-rest-service/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json index d6487142cc6..9c07ad4ffc6 100644 --- a/catalog-rest-service/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json +++ b/catalog-rest-service/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json @@ -25,10 +25,10 @@ "description": "Regex exclude tables or databases that matches the pattern.", "$ref": "../type/filterPattern.json#/definitions/filterPattern" }, - "dbServiceName": { - "title": "Database Service Name", - "description": "Database Service Name for creation of lineage", - "type": "string" + "dbServiceNames": { + "title": "Database Service Name List", + "description": "List of Database Service Name for creation of lineage", + "type": "array" } }, "additionalProperties": false diff --git a/ingestion/examples/workflows/metabase.yaml b/ingestion/examples/workflows/metabase.yaml index 7696854c361..78f70e7d2cb 100644 --- a/ingestion/examples/workflows/metabase.yaml +++ b/ingestion/examples/workflows/metabase.yaml @@ -9,7 +9,7 @@ source: hostPort: http://hostPort sourceConfig: config: - dbServiceName: Database Service Name to create Lineage + dbServiceNames: list of database service name to create Lineage dashboardFilterPattern: {} chartFilterPattern: {} sink: diff --git a/ingestion/examples/workflows/mode.yaml b/ingestion/examples/workflows/mode.yaml index 238b8ca9010..9c1b8d52fd3 100644 --- a/ingestion/examples/workflows/mode.yaml +++ b/ingestion/examples/workflows/mode.yaml @@ -20,7 +20,7 @@ source: includes: - Supplier Quality Analysis Sample - "Customer" - dbServiceName: local_redshift + dbServiceNames: [local_redshift] sink: type: metadata-rest config: {} diff --git a/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py b/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py index a729c20a7c7..3e0fdf317fc 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py @@ -167,7 +167,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC): @abstractmethod def yield_dashboard_lineage_details( - self, dashboard_details: Any + self, dashboard_details: Any, db_service_name: str ) -> Optional[Iterable[AddLineageRequest]]: """ Get lineage between dashboard and data sources @@ -205,8 +205,10 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC): """ Yields lineage if config is enabled """ - if self.source_config.dbServiceName: - yield from self.yield_dashboard_lineage_details(dashboard_details) or [] + for db_service_name in self.source_config.dbServiceNames or []: + yield from self.yield_dashboard_lineage_details( + dashboard_details, db_service_name + ) or [] def yield_tag(self, *args, **kwargs) -> Optional[Iterable[OMetaTagAndCategory]]: """ diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker.py b/ingestion/src/metadata/ingestion/source/dashboard/looker.py index 30f6aa671d2..4a25eb46a0e 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/looker.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/looker.py @@ -182,7 +182,7 @@ class LookerSource(DashboardServiceSource): return dashboard_sources def yield_dashboard_lineage_details( - self, dashboard_details: LookerDashboard + self, dashboard_details: LookerDashboard, db_service_name ) -> Optional[Iterable[AddLineageRequest]]: """ Get lineage between charts and data sources. @@ -212,7 +212,7 @@ class LookerSource(DashboardServiceSource): from_fqn = fqn.build( self.metadata, entity_type=Table, - service_name=self.source_config.dbServiceName, + service_name=db_service_name, database_name=source_elements["database"], schema_name=source_elements["database_schema"], table_name=source_elements["table"], diff --git a/ingestion/src/metadata/ingestion/source/dashboard/metabase.py b/ingestion/src/metadata/ingestion/source/dashboard/metabase.py index 693bef2992f..d20021b7e5f 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/metabase.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/metabase.py @@ -173,14 +173,14 @@ class MetabaseSource(DashboardServiceSource): continue def yield_dashboard_lineage_details( - self, dashboard_details: dict + self, dashboard_details: dict, db_service_name ) -> Optional[Iterable[AddLineageRequest]]: """Get lineage method Args: dashboard_details """ - if not self.source_config.dbServiceName: + if not db_service_name: return chart_list, dashboard_name = ( dashboard_details["ordered_cards"], @@ -216,7 +216,7 @@ class MetabaseSource(DashboardServiceSource): from_entities = search_table_entities( metadata=self.metadata, database=database["details"]["dbname"], - service_name=self.source_config.dbServiceName, + service_name=db_service_name, database_schema=database_schema_name, table=table, ) @@ -256,7 +256,7 @@ class MetabaseSource(DashboardServiceSource): from_fqn = fqn.build( self.metadata, entity_type=Table, - service_name=self.source_config.dbServiceName, + service_name=db_service_name, database_name=table["db"]["details"]["dbname"], schema_name=table.get("schema"), table_name=table.get("display_name"), diff --git a/ingestion/src/metadata/ingestion/source/dashboard/mode.py b/ingestion/src/metadata/ingestion/source/dashboard/mode.py index ef56959cb02..0d525bb0ec4 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/mode.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/mode.py @@ -116,7 +116,7 @@ class ModeSource(DashboardServiceSource): ) def yield_dashboard_lineage_details( - self, dashboard_details: dict + self, dashboard_details: dict, db_service_name: str ) -> Optional[Iterable[AddLineageRequest]]: """Get lineage method @@ -147,7 +147,7 @@ class ModeSource(DashboardServiceSource): from_entities = search_table_entities( metadata=self.metadata, database=data_source.get(mode_client.DATABASE), - service_name=self.source_config.dbServiceName, + service_name=db_service_name, database_schema=database_schema_name, table=table, ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/powerbi.py b/ingestion/src/metadata/ingestion/source/dashboard/powerbi.py index e91261b01d7..0ec59a66339 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi.py @@ -98,7 +98,7 @@ class PowerbiSource(DashboardServiceSource): ) def yield_dashboard_lineage_details( - self, dashboard_details: dict + self, dashboard_details: dict, db_service_name: str ) -> Optional[Iterable[AddLineageRequest]]: """ Get lineage between dashboard and data sources @@ -119,7 +119,7 @@ class PowerbiSource(DashboardServiceSource): from_fqn = fqn.build( self.metadata, entity_type=Database, - service_name=self.source_config.dbServiceName, + service_name=db_service_name, database_name=database_name, ) from_entity = self.metadata.get_by_name( diff --git a/ingestion/src/metadata/ingestion/source/dashboard/redash.py b/ingestion/src/metadata/ingestion/source/dashboard/redash.py index 7ecb7f90318..4e83f72d3a1 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/redash.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/redash.py @@ -112,7 +112,7 @@ class RedashSource(DashboardServiceSource): self.status.scanned(dashboard_details["name"]) def yield_dashboard_lineage_details( - self, dashboard_details: dict + self, dashboard_details: dict, db_service_name: str ) -> Optional[Iterable[AddLineageRequest]]: """ Get lineage between dashboard and data sources @@ -133,7 +133,7 @@ class RedashSource(DashboardServiceSource): table_entities = search_table_entities( metadata=self.metadata, database=None, - service_name=self.source_config.dbServiceName, + service_name=db_service_name, database_schema=database_schema, table=table_name, ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset.py b/ingestion/src/metadata/ingestion/source/dashboard/superset.py index 491de220d1b..3ddb0584415 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset.py @@ -185,7 +185,7 @@ class SupersetSource(DashboardServiceSource): return [] def yield_dashboard_lineage_details( - self, dashboard_details: dict + self, dashboard_details: dict, db_service_name: str ) -> Optional[Iterable[AddLineageRequest]]: """ Get lineage between dashboard and data sources @@ -193,7 +193,9 @@ class SupersetSource(DashboardServiceSource): for chart_id in self._get_charts_of_dashboard(dashboard_details): chart_json = self.all_charts.get(chart_id) datasource_fqn = ( - self._get_datasource_fqn(chart_json.get("datasource_id")) + self._get_datasource_fqn( + chart_json.get("datasource_id"), db_service_name + ) if chart_json.get("datasource_id") else None ) @@ -258,8 +260,10 @@ class SupersetSource(DashboardServiceSource): ) yield chart - def _get_datasource_fqn(self, datasource_id: str) -> Optional[str]: - if not self.source_config.dbServiceName: + def _get_datasource_fqn( + self, datasource_id: str, db_service_name: str + ) -> Optional[str]: + if not db_service_name: return try: datasource_json = self.client.fetch_datasource(datasource_id) @@ -272,7 +276,7 @@ class SupersetSource(DashboardServiceSource): table_name=datasource_json["result"]["table_name"], schema_name=datasource_json["result"]["schema"], database_name=database_json["result"]["parameters"]["database"], - service_name=self.source_config.dbServiceName, + service_name=db_service_name, ) return dataset_fqn except KeyError: diff --git a/ingestion/src/metadata/ingestion/source/dashboard/tableau.py b/ingestion/src/metadata/ingestion/source/dashboard/tableau.py index b96cbacac80..29f7aacbf90 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/tableau.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/tableau.py @@ -213,7 +213,7 @@ class TableauSource(DashboardServiceSource): ) def yield_dashboard_lineage_details( - self, dashboard_details: dict + self, dashboard_details: dict, db_service_name: str ) -> Optional[Iterable[AddLineageRequest]]: """ Get lineage between dashboard and data sources @@ -246,7 +246,7 @@ class TableauSource(DashboardServiceSource): from_fqn = fqn.build( self.metadata, entity_type=Table, - service_name=self.source_config.dbServiceName, + service_name=db_service_name, schema_name=schema_name, table_name=table_name, database_name=None, diff --git a/openmetadata-docs/ingestion/connectors/metabase/ingestion.yaml b/openmetadata-docs/ingestion/connectors/metabase/ingestion.yaml index caf1a5964ed..5f23f65a236 100644 --- a/openmetadata-docs/ingestion/connectors/metabase/ingestion.yaml +++ b/openmetadata-docs/ingestion/connectors/metabase/ingestion.yaml @@ -7,7 +7,7 @@ source: username: password: hostPort: - dbServiceName: + dbServiceNames: [] sourceConfig: config: dashboardFilterPattern: diff --git a/openmetadata-docs/ingestion/connectors/superset/ingestion.yaml b/openmetadata-docs/ingestion/connectors/superset/ingestion.yaml index ac598743d7e..af433565a3e 100644 --- a/openmetadata-docs/ingestion/connectors/superset/ingestion.yaml +++ b/openmetadata-docs/ingestion/connectors/superset/ingestion.yaml @@ -6,7 +6,7 @@ source: hostPort: http://localhost:8080 username: admin password: admin - dbServiceName: local_mysql + dbServiceNames: [local_mysql] type: Superset sourceConfig: config: diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx index 9aa90204c56..ec4de20039e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx @@ -91,8 +91,8 @@ const AddIngestion = ({ const [ingestSampleData, setIngestSampleData] = useState( (data?.sourceConfig.config as ConfigClass)?.generateSampleData ?? true ); - const [databaseServiceName, setDatabaseServiceName] = useState( - (data?.sourceConfig.config as ConfigClass)?.dbServiceName ?? '' + const [databaseServiceNames, setDatabaseServiceNames] = useState( + (data?.sourceConfig.config as ConfigClass)?.dbServiceNames ?? [] ); const [description, setDescription] = useState(data?.description ?? ''); const [repeatFrequency, setRepeatFrequency] = useState( @@ -418,7 +418,7 @@ const AddIngestion = ({ dashboardFilterPattern, showDashboardFilter ), - dbServiceName: databaseServiceName, + dbServiceNames: databaseServiceNames, type: ConfigType.DashboardMetadata, }; } @@ -611,12 +611,12 @@ const AddIngestion = ({ chartFilterPattern={chartFilterPattern} dashboardFilterPattern={dashboardFilterPattern} databaseFilterPattern={databaseFilterPattern} - databaseServiceName={databaseServiceName} + databaseServiceNames={databaseServiceNames} description={description} enableDebugLog={enableDebugLog} getExcludeValue={getExcludeValue} getIncludeValue={getIncludeValue} - handleDatasetServiceName={(val) => setDatabaseServiceName(val)} + handleDatasetServiceName={(val) => setDatabaseServiceNames(val)} handleDescription={(val) => setDescription(val)} handleEnableDebugLog={() => setEnableDebugLog((pre) => !pre)} handleIncludeLineage={() => setIncludeLineage((pre) => !pre)} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx index 3c0be517947..eeb9f2a1a12 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx @@ -92,7 +92,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = { handleEnableDebugLog: jest.fn(), ingestSampleData: false, handleIngestSampleData: jest.fn(), - databaseServiceName: '', + databaseServiceNames: [''], handleDatasetServiceName: jest.fn(), threadCount: 5, handleThreadCount: jest.fn(), diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx index e4d03a4f805..cbab9f3ae50 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx @@ -28,7 +28,7 @@ import { ConfigureIngestionProps } from '../addIngestion.interface'; const ConfigureIngestion = ({ ingestionName, description = '', - databaseServiceName, + databaseServiceNames, databaseFilterPattern, dashboardFilterPattern, schemaFilterPattern, @@ -255,6 +255,16 @@ const ConfigureIngestion = ({ ); }; + const handleDashBoardServiceNames = (inputValue: string) => { + const separator = ','; + + const databaseNames = inputValue.includes(separator) + ? inputValue.split(separator) + : Array(inputValue); + + if (databaseNames) handleDatasetServiceName(databaseNames); + }; + const getDashboardDBServiceName = () => { return ( @@ -270,8 +280,8 @@ const ConfigureIngestion = ({ id="name" name="name" type="text" - value={databaseServiceName} - onChange={(e) => handleDatasetServiceName(e.target.value)} + value={databaseServiceNames} + onChange={(e) => handleDashBoardServiceNames(e.target.value)} /> {getSeparator('')} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts index 2051d1e5b25..f556986d177 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts @@ -57,7 +57,7 @@ export interface AddIngestionProps { export interface ConfigureIngestionProps { ingestionName: string; description?: string; - databaseServiceName: string; + databaseServiceNames: string[]; serviceCategory: ServiceCategory; databaseFilterPattern: FilterPattern; dashboardFilterPattern: FilterPattern; @@ -86,7 +86,7 @@ export interface ConfigureIngestionProps { stageFileLocation: string; resultLimit: number; handleIngestionName: (value: string) => void; - handleDatasetServiceName: (value: string) => void; + handleDatasetServiceName: (value: string[]) => void; handleDescription?: (value: string) => void; handleIncludeLineage: () => void; handleIncludeView: () => void;