multiple-database-service-names-added-in-dashboard (#6591)

* multiple-database-service-names-added-in-dashboard

* code-formatted

* code-smell-removed

* code-formatted

* improved-dbserviceName-handling-in-dashboard

* fix checks failing

Co-authored-by: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com>

catalog-rest-service/src/main/resources/json/schema/metadataIngestion/dashboardServiceMetadataPipeline.json

@@ -25,10 +25,10 @@
       "description": "Regex exclude tables or databases that matches the pattern.",
       "$ref": "../type/filterPattern.json#/definitions/filterPattern"
     },
-    "dbServiceName": {
+    "dbServiceNames": {
-      "title": "Database Service Name",
+      "title": "Database Service Name List",
-      "description": "Database Service Name for creation of lineage",
+      "description": "List of Database Service Name for creation of lineage",
-      "type": "string"
+      "type": "array"
     }
   },
   "additionalProperties": false

ingestion/examples/workflows/metabase.yaml

@@ -9,7 +9,7 @@ source:
       hostPort: http://hostPort
   sourceConfig:
     config:
-      dbServiceName: Database Service Name to create Lineage
+      dbServiceNames: list of database service name to create Lineage
       dashboardFilterPattern: {}
       chartFilterPattern: {}
 sink:

ingestion/examples/workflows/mode.yaml

@@ -20,7 +20,7 @@ source:
         includes:
           - Supplier Quality Analysis Sample
           - "Customer"
-      dbServiceName: local_redshift
+      dbServiceNames: [local_redshift]
 sink:
   type: metadata-rest
   config: {}

ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py

@@ -167,7 +167,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
 
     @abstractmethod
     def yield_dashboard_lineage_details(
-        self, dashboard_details: Any
+        self, dashboard_details: Any, db_service_name: str
     ) -> Optional[Iterable[AddLineageRequest]]:
         """
         Get lineage between dashboard and data sources
@@ -205,8 +205,10 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
         """
         Yields lineage if config is enabled
         """
-        if self.source_config.dbServiceName:
+        for db_service_name in self.source_config.dbServiceNames or []:
-            yield from self.yield_dashboard_lineage_details(dashboard_details) or []
+            yield from self.yield_dashboard_lineage_details(
+                dashboard_details, db_service_name
+            ) or []
 
     def yield_tag(self, *args, **kwargs) -> Optional[Iterable[OMetaTagAndCategory]]:
         """

ingestion/src/metadata/ingestion/source/dashboard/looker.py

@@ -182,7 +182,7 @@ class LookerSource(DashboardServiceSource):
         return dashboard_sources
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: LookerDashboard
+        self, dashboard_details: LookerDashboard, db_service_name
     ) -> Optional[Iterable[AddLineageRequest]]:
         """
         Get lineage between charts and data sources.
@@ -212,7 +212,7 @@ class LookerSource(DashboardServiceSource):
                 from_fqn = fqn.build(
                     self.metadata,
                     entity_type=Table,
-                    service_name=self.source_config.dbServiceName,
+                    service_name=db_service_name,
                     database_name=source_elements["database"],
                     schema_name=source_elements["database_schema"],
                     table_name=source_elements["table"],

ingestion/src/metadata/ingestion/source/dashboard/metabase.py

@@ -173,14 +173,14 @@ class MetabaseSource(DashboardServiceSource):
                 continue
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: dict
+        self, dashboard_details: dict, db_service_name
     ) -> Optional[Iterable[AddLineageRequest]]:
         """Get lineage method
 
         Args:
             dashboard_details
         """
-        if not self.source_config.dbServiceName:
+        if not db_service_name:
             return
         chart_list, dashboard_name = (
             dashboard_details["ordered_cards"],
@@ -216,7 +216,7 @@ class MetabaseSource(DashboardServiceSource):
                         from_entities = search_table_entities(
                             metadata=self.metadata,
                             database=database["details"]["dbname"],
-                            service_name=self.source_config.dbServiceName,
+                            service_name=db_service_name,
                             database_schema=database_schema_name,
                             table=table,
                         )
@@ -256,7 +256,7 @@ class MetabaseSource(DashboardServiceSource):
                         from_fqn = fqn.build(
                             self.metadata,
                             entity_type=Table,
-                            service_name=self.source_config.dbServiceName,
+                            service_name=db_service_name,
                             database_name=table["db"]["details"]["dbname"],
                             schema_name=table.get("schema"),
                             table_name=table.get("display_name"),

ingestion/src/metadata/ingestion/source/dashboard/mode.py

@@ -116,7 +116,7 @@ class ModeSource(DashboardServiceSource):
         )
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: dict
+        self, dashboard_details: dict, db_service_name: str
     ) -> Optional[Iterable[AddLineageRequest]]:
         """Get lineage method
 
@@ -147,7 +147,7 @@ class ModeSource(DashboardServiceSource):
                     from_entities = search_table_entities(
                         metadata=self.metadata,
                         database=data_source.get(mode_client.DATABASE),
-                        service_name=self.source_config.dbServiceName,
+                        service_name=db_service_name,
                         database_schema=database_schema_name,
                         table=table,
                     )

ingestion/src/metadata/ingestion/source/dashboard/powerbi.py

@@ -98,7 +98,7 @@ class PowerbiSource(DashboardServiceSource):
         )
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: dict
+        self, dashboard_details: dict, db_service_name: str
     ) -> Optional[Iterable[AddLineageRequest]]:
         """
         Get lineage between dashboard and data sources
@@ -119,7 +119,7 @@ class PowerbiSource(DashboardServiceSource):
                         from_fqn = fqn.build(
                             self.metadata,
                             entity_type=Database,
-                            service_name=self.source_config.dbServiceName,
+                            service_name=db_service_name,
                             database_name=database_name,
                         )
                         from_entity = self.metadata.get_by_name(

ingestion/src/metadata/ingestion/source/dashboard/redash.py

@@ -112,7 +112,7 @@ class RedashSource(DashboardServiceSource):
         self.status.scanned(dashboard_details["name"])
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: dict
+        self, dashboard_details: dict, db_service_name: str
     ) -> Optional[Iterable[AddLineageRequest]]:
         """
         Get lineage between dashboard and data sources
@@ -133,7 +133,7 @@ class RedashSource(DashboardServiceSource):
                 table_entities = search_table_entities(
                     metadata=self.metadata,
                     database=None,
-                    service_name=self.source_config.dbServiceName,
+                    service_name=db_service_name,
                     database_schema=database_schema,
                     table=table_name,
                 )

ingestion/src/metadata/ingestion/source/dashboard/superset.py

@@ -185,7 +185,7 @@ class SupersetSource(DashboardServiceSource):
         return []
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: dict
+        self, dashboard_details: dict, db_service_name: str
     ) -> Optional[Iterable[AddLineageRequest]]:
         """
         Get lineage between dashboard and data sources
@@ -193,7 +193,9 @@ class SupersetSource(DashboardServiceSource):
         for chart_id in self._get_charts_of_dashboard(dashboard_details):
             chart_json = self.all_charts.get(chart_id)
             datasource_fqn = (
-                self._get_datasource_fqn(chart_json.get("datasource_id"))
+                self._get_datasource_fqn(
+                    chart_json.get("datasource_id"), db_service_name
+                )
                 if chart_json.get("datasource_id")
                 else None
             )
@@ -258,8 +260,10 @@ class SupersetSource(DashboardServiceSource):
             )
             yield chart
 
-    def _get_datasource_fqn(self, datasource_id: str) -> Optional[str]:
+    def _get_datasource_fqn(
-        if not self.source_config.dbServiceName:
+        self, datasource_id: str, db_service_name: str
+    ) -> Optional[str]:
+        if not db_service_name:
             return
         try:
             datasource_json = self.client.fetch_datasource(datasource_id)
@@ -272,7 +276,7 @@ class SupersetSource(DashboardServiceSource):
                 table_name=datasource_json["result"]["table_name"],
                 schema_name=datasource_json["result"]["schema"],
                 database_name=database_json["result"]["parameters"]["database"],
-                service_name=self.source_config.dbServiceName,
+                service_name=db_service_name,
             )
             return dataset_fqn
         except KeyError:

ingestion/src/metadata/ingestion/source/dashboard/tableau.py

@@ -213,7 +213,7 @@ class TableauSource(DashboardServiceSource):
         )
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: dict
+        self, dashboard_details: dict, db_service_name: str
     ) -> Optional[Iterable[AddLineageRequest]]:
         """
         Get lineage between dashboard and data sources
@@ -246,7 +246,7 @@ class TableauSource(DashboardServiceSource):
                 from_fqn = fqn.build(
                     self.metadata,
                     entity_type=Table,
-                    service_name=self.source_config.dbServiceName,
+                    service_name=db_service_name,
                     schema_name=schema_name,
                     table_name=table_name,
                     database_name=None,

openmetadata-docs/ingestion/connectors/metabase/ingestion.yaml

@@ -7,7 +7,7 @@ source:
       username: <username>
       password: <password>
       hostPort: <hostPort>
-      dbServiceName: <dbServiceName>
+      dbServiceNames: [<dbServiceName>]
   sourceConfig:
     config:
       dashboardFilterPattern: <dashboard name regex list>

openmetadata-docs/ingestion/connectors/superset/ingestion.yaml

@@ -6,7 +6,7 @@ source:
       hostPort: http://localhost:8080
       username: admin
       password: admin
-      dbServiceName: local_mysql
+      dbServiceNames: [local_mysql]
       type: Superset
   sourceConfig:
     config:

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx

@@ -91,8 +91,8 @@ const AddIngestion = ({
   const [ingestSampleData, setIngestSampleData] = useState(
     (data?.sourceConfig.config as ConfigClass)?.generateSampleData ?? true
   );
-  const [databaseServiceName, setDatabaseServiceName] = useState(
+  const [databaseServiceNames, setDatabaseServiceNames] = useState(
-    (data?.sourceConfig.config as ConfigClass)?.dbServiceName ?? ''
+    (data?.sourceConfig.config as ConfigClass)?.dbServiceNames ?? []
   );
   const [description, setDescription] = useState(data?.description ?? '');
   const [repeatFrequency, setRepeatFrequency] = useState(
@@ -418,7 +418,7 @@ const AddIngestion = ({
             dashboardFilterPattern,
             showDashboardFilter
           ),
-          dbServiceName: databaseServiceName,
+          dbServiceNames: databaseServiceNames,
           type: ConfigType.DashboardMetadata,
         };
       }
@@ -611,12 +611,12 @@ const AddIngestion = ({
             chartFilterPattern={chartFilterPattern}
             dashboardFilterPattern={dashboardFilterPattern}
             databaseFilterPattern={databaseFilterPattern}
-            databaseServiceName={databaseServiceName}
+            databaseServiceNames={databaseServiceNames}
             description={description}
             enableDebugLog={enableDebugLog}
             getExcludeValue={getExcludeValue}
             getIncludeValue={getIncludeValue}
-            handleDatasetServiceName={(val) => setDatabaseServiceName(val)}
+            handleDatasetServiceName={(val) => setDatabaseServiceNames(val)}
             handleDescription={(val) => setDescription(val)}
             handleEnableDebugLog={() => setEnableDebugLog((pre) => !pre)}
             handleIncludeLineage={() => setIncludeLineage((pre) => !pre)}

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx

@@ -92,7 +92,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = {
   handleEnableDebugLog: jest.fn(),
   ingestSampleData: false,
   handleIngestSampleData: jest.fn(),
-  databaseServiceName: '',
+  databaseServiceNames: [''],
   handleDatasetServiceName: jest.fn(),
   threadCount: 5,
   handleThreadCount: jest.fn(),

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx

@@ -28,7 +28,7 @@ import { ConfigureIngestionProps } from '../addIngestion.interface';
 const ConfigureIngestion = ({
   ingestionName,
   description = '',
-  databaseServiceName,
+  databaseServiceNames,
   databaseFilterPattern,
   dashboardFilterPattern,
   schemaFilterPattern,
@@ -255,6 +255,16 @@ const ConfigureIngestion = ({
     );
   };
 
+  const handleDashBoardServiceNames = (inputValue: string) => {
+    const separator = ',';
+
+    const databaseNames = inputValue.includes(separator)
+      ? inputValue.split(separator)
+      : Array(inputValue);
+
+    if (databaseNames) handleDatasetServiceName(databaseNames);
+  };
+
   const getDashboardDBServiceName = () => {
     return (
       <Field>
@@ -270,8 +280,8 @@ const ConfigureIngestion = ({
           id="name"
           name="name"
           type="text"
-          value={databaseServiceName}
+          value={databaseServiceNames}
-          onChange={(e) => handleDatasetServiceName(e.target.value)}
+          onChange={(e) => handleDashBoardServiceNames(e.target.value)}
         />
         {getSeparator('')}
       </Field>

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts

@@ -57,7 +57,7 @@ export interface AddIngestionProps {
 export interface ConfigureIngestionProps {
   ingestionName: string;
   description?: string;
-  databaseServiceName: string;
+  databaseServiceNames: string[];
   serviceCategory: ServiceCategory;
   databaseFilterPattern: FilterPattern;
   dashboardFilterPattern: FilterPattern;
@@ -86,7 +86,7 @@ export interface ConfigureIngestionProps {
   stageFileLocation: string;
   resultLimit: number;
   handleIngestionName: (value: string) => void;
-  handleDatasetServiceName: (value: string) => void;
+  handleDatasetServiceName: (value: string[]) => void;
   handleDescription?: (value: string) => void;
   handleIncludeLineage: () => void;
   handleIncludeView: () => void;