Fixes #7044 Handle database filter in ingestion pipeline when connection specifies a database from UI side #7044 (#10766)

* Fixes Handle databse filter in ingestion pipeline when connection specifies a database #7044 * fix: Handle databse filter in ingestion pipeline when connection specifies a database #7044 * sync-translation * fixed unit test --------- Co-authored-by: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com>
2025-12-24 22:18:41 +00:00 · 2023-04-05 11:01:19 +05:30 · 2023-04-05 11:01:19 +05:30 · d5a072d83b
commit d5a072d83b
parent abb3c8259f
14 changed files with 58 additions and 10 deletions
--- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx
@ -12,6 +12,7 @@
 */

 import { LOADING_STATE } from 'enums/common.enum';
+import { Connection } from 'generated/api/services/createDatabaseService';
 import { isEmpty, isUndefined, omit, trim } from 'lodash';
 import React, {
  Reducer,
@ -131,9 +132,12 @@ const AddIngestion = ({
    () => getSourceTypeFromConfig(configData as DbtConfig | undefined),
    [configData]
  );
+  const { database, ingestAllDatabases } = serviceData.connection
+    .config as Connection;

  const initialState: AddIngestionState = useMemo(
    () => ({
+      database,
      saveState: 'initial',
      showDeployModal: false,
      ingestionName:
@ -147,7 +151,12 @@ const AddIngestion = ({
        data?.airflowConfig.scheduleInterval ??
        getIngestionFrequency(pipelineType),
      showDashboardFilter: !isUndefined(sourceConfig?.dashboardFilterPattern),
-      showDatabaseFilter: !isUndefined(sourceConfig?.databaseFilterPattern),
+      showDatabaseFilter: Boolean(
+        database || sourceConfig?.databaseFilterPattern
+      ),
+      isDatabaseFilterDisabled: ingestAllDatabases
+        ? !ingestAllDatabases
+        : Boolean(database),
      showSchemaFilter: !isUndefined(sourceConfig?.schemaFilterPattern),
      showTableFilter: !isUndefined(sourceConfig?.tableFilterPattern),
      showTopicFilter: !isUndefined(sourceConfig?.topicFilterPattern),
@ -164,8 +173,12 @@ const AddIngestion = ({
        : undefined,
      dashboardFilterPattern:
        sourceConfig?.dashboardFilterPattern ?? INITIAL_FILTER_PATTERN,
-      databaseFilterPattern:
-        sourceConfig?.databaseFilterPattern ?? INITIAL_FILTER_PATTERN,
+      databaseFilterPattern: isUndefined(database)
+        ? sourceConfig?.databaseFilterPattern ?? INITIAL_FILTER_PATTERN
+        : {
+            includes: [database],
+            excludes: [],
+          },
      markAllDeletedTables: isDatabaseService
        ? Boolean(sourceConfig?.markAllDeletedTables ?? false)
        : undefined,
--- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.test.tsx
@ -16,7 +16,6 @@ import React from 'react';
 import { FormSubmitType } from '../../enums/form.enum';
 import { ServiceCategory } from '../../enums/service.enum';
 import { PipelineType } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
-import { DataObj } from '../../interface/service.interface';
 import AddIngestion from './AddIngestion.component';
 import { AddIngestionProps } from './addIngestion.interface';

@ -25,7 +24,13 @@ const mockAddIngestionProps: AddIngestionProps = {
  setActiveIngestionStep: jest.fn(),
  serviceData: {
    name: 'serviceName',
-  } as DataObj,
+    connection: {
+      config: {
+        database: 'testDb',
+        ingestAllDatabases: false,
+      },
+    },
+  },
  handleCancelClick: jest.fn(),
  serviceCategory: ServiceCategory.DASHBOARD_SERVICES,
  onAddIngestionSave: jest.fn(),
--- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx
@ -46,6 +46,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = {
  serviceCategory: ServiceCategory.DATABASE_SERVICES,
  onChange: jest.fn(),
  data: {
+    showDatabaseFilterField: true,
    ingestionName: '',
    databaseFilterPattern: {
      includes: [],
--- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx
@ -591,8 +591,15 @@ const ConfigureIngestion = ({
            handleShowFilter(value, ShowFilter.showDatabaseFilter)
          }
          includePattern={databaseFilterPattern?.includes ?? []}
+          includePatternExtraInfo={
+            data.database
+              ? t('message.include-database-filter-extra-information')
+              : undefined
+          }
+          isDisabled={data.isDatabaseFilterDisabled}
          type={FilterPatternEnum.DATABASE}
        />
+
        <FilterPattern
          checked={showSchemaFilter}
          excludePattern={schemaFilterPattern?.excludes ?? []}
--- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts
@ -11,7 +11,7 @@
 *  limitations under the License.
 */

-import { LoadingState } from 'Models';
+import { LoadingState, ServicesUpdateRequest } from 'Models';
 import { FilterPatternEnum } from '../../enums/filterPattern.enum';
 import { FormSubmitType } from '../../enums/form.enum';
 import { ServiceCategory } from '../../enums/service.enum';
@ -27,7 +27,7 @@ import {
  PipelineType,
 } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
 import { DbtPipelineClass } from '../../generated/metadataIngestion/dbtPipeline';
-import { DataObj } from '../../interface/service.interface';
+
 import {
  DBT_SOURCES,
  GCS_CONFIG,
@ -41,7 +41,7 @@ export interface AddIngestionProps {
  status: FormSubmitType;
  data?: IngestionPipeline;
  serviceCategory: ServiceCategory;
-  serviceData: DataObj;
+  serviceData: ServicesUpdateRequest;
  showSuccessScreen?: boolean;
  showDeployButton?: boolean;
  setActiveIngestionStep: (step: number) => void;
@ -94,8 +94,10 @@ export type ModifiedDbtConfig = DbtConfig &

 export interface AddIngestionState {
  chartFilterPattern: FilterPattern;
+  database?: string;
  dashboardFilterPattern: FilterPattern;
  databaseFilterPattern: FilterPattern;
+  isDatabaseFilterDisabled: boolean;
  databaseServiceNames: string[];
  dbtClassificationName: string;
  dbtUpdateDescriptions: boolean;
--- a/openmetadata-ui/src/main/resources/ui/src/components/AddService/AddService.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AddService/AddService.component.tsx
@ -29,7 +29,7 @@ import { delimiterRegex, nameWithSpace } from '../../constants/regex.constants';
 import { FormSubmitType } from '../../enums/form.enum';
 import { ServiceCategory } from '../../enums/service.enum';
 import { PipelineType } from '../../generated/entity/services/ingestionPipelines/ingestionPipeline';
-import { ConfigData, DataObj } from '../../interface/service.interface';
+import { ConfigData } from '../../interface/service.interface';
 import { getCurrentUserId, isUrlFriendlyName } from '../../utils/CommonUtils';
 import { getAddServicePath, getSettingPath } from '../../utils/RouterUtils';
 import {
@ -333,7 +333,7 @@ const AddService = ({
            isIngestionDeployed={isIngestionDeployed}
            pipelineType={PipelineType.Metadata}
            serviceCategory={serviceCategory}
-            serviceData={newServiceData as DataObj}
+            serviceData={newServiceData}
            setActiveIngestionStep={(step) => setActiveIngestionStep(step)}
            showDeployButton={showDeployButton}
            status={FormSubmitType.ADD}
--- a/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/FilterPattern.tsx
@ -25,12 +25,14 @@ import { FilterPatternProps } from './filterPattern.interface';

 const FilterPattern = ({
  showSeparator = true,
+  isDisabled = false,
  checked,
  includePattern,
  excludePattern,
  handleChecked,
  getIncludeValue,
  getExcludeValue,
+  includePatternExtraInfo,
  type,
 }: FilterPatternProps) => {
  const includeFilterChangeHandler = (
@ -55,6 +57,7 @@ const FilterPattern = ({
            checked={checked}
            className="m-r-sm filter-pattern-checkbox"
            data-testid={`${type}-filter-pattern-checkbox`}
+            disabled={isDisabled}
            id={`${type}FilterPatternCheckbox`}
            name={`${type}FilterPatternCheckbox`}
            onChange={(e) => handleChecked(e.target.checked)}
@ -101,11 +104,19 @@ const FilterPattern = ({
            <Input
              className="m-t-xss"
              data-testid={`filter-pattern-includes-${type}`}
+              disabled={isDisabled}
              placeholder={t('message.list-of-strings-regex-patterns-csv')}
              type="text"
              value={includePattern}
              onChange={includeFilterChangeHandler}
            />
+            {includePatternExtraInfo && (
+              <Typography.Text
+                className="text-grey-muted m-t-xss m-b-xss"
+                data-testid="filter-pattern-include-info">
+                {includePatternExtraInfo}
+              </Typography.Text>
+            )}
          </Field>
          <Field>
            <label className="d-flex flex-col">{t('label.exclude')}:</label>
@ -120,6 +131,7 @@ const FilterPattern = ({
            <Input
              className="m-t-xss"
              data-testid={`filter-pattern-excludes-${type}`}
+              disabled={isDisabled}
              placeholder={t('message.list-of-strings-regex-patterns-csv')}
              type="text"
              value={excludePattern}
--- a/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/filterPattern.interface.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/FilterPattern/filterPattern.interface.ts
@ -15,6 +15,8 @@ import { FilterPatternEnum } from '../../../enums/filterPattern.enum';

 export interface FilterPatternProps {
  checked: boolean;
+  isDisabled?: boolean;
+  includePatternExtraInfo?: string;
  showSeparator?: boolean;
  handleChecked: (e: boolean) => void;
  includePattern: Array<string> | undefined;
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json
@ -1022,6 +1022,7 @@
    "has-been-created-successfully": "has been created successfully",
    "in-this-database": "In this Database",
    "include-assets-message": "Optional configuration to toggle the tags ingestion.",
+    "include-database-filter-extra-information": "Database which was added while creating service.",
    "include-lineage-message": "Configuration to turn off fetching lineage from pipelines.",
    "ingest-sample-data-for-entity": "Extract sample data from each {{entity}}.",
    "ingestion-bot-cant-be-deleted": "You cannot delete the ingestion bot.",
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json
@ -1022,6 +1022,7 @@
    "has-been-created-successfully": "has been created successfully",
    "in-this-database": "In this Database",
    "include-assets-message": "Optional configuration to toggle the tags ingestion.",
+    "include-database-filter-extra-information": "Database which was added while creating service.",
    "include-lineage-message": "Configuration to turn off fetching lineage from pipelines.",
    "ingest-sample-data-for-entity": "Extract sample data from each {{entity}}.",
    "ingestion-bot-cant-be-deleted": "You cannot delete the ingestion bot.",
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json
@ -1022,6 +1022,7 @@
    "has-been-created-successfully": "has been created successfully",
    "in-this-database": "In this Database",
    "include-assets-message": "Enable extracting {{assets}} from the data source.",
+    "include-database-filter-extra-information": "Database which was added while creating service.",
    "include-lineage-message": "Configuration to turn off fetching lineage from pipelines.",
    "ingest-sample-data-for-entity": "Extract sample data from each {{entity}}.",
    "ingestion-bot-cant-be-deleted": "Vous ne pouvez pas supprimer l'agent numérique d'ingestion.",
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json
@ -1022,6 +1022,7 @@
    "has-been-created-successfully": "正常に作成されました",
    "in-this-database": "In this Database",
    "include-assets-message": "データソースから{{assets}}の抽出を有効にする。",
+    "include-database-filter-extra-information": "Database which was added while creating service.",
    "include-lineage-message": "Configuration to turn off fetching lineage from pipelines.",
    "ingest-sample-data-for-entity": "{{entity}}からサンプルデータを抽出",
    "ingestion-bot-cant-be-deleted": "You cannot delete the ingestion bot.",
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json
@ -1022,6 +1022,7 @@
    "has-been-created-successfully": "foi criado com sucesso",
    "in-this-database": "Neste banco de dados",
    "include-assets-message": "Habilitar extração de {{assets}} da fonte de dados.",
+    "include-database-filter-extra-information": "Database which was added while creating service.",
    "include-lineage-message": "Configuração para desligar a busca de linhagem nos pipelines.",
    "ingest-sample-data-for-entity": "Extrair dados de exemplo de cada {{entity}}.",
    "ingestion-bot-cant-be-deleted": "Você não pode excluir o robô de ingestão.",
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json
@ -1022,6 +1022,7 @@
    "has-been-created-successfully": "has been created successfully",
    "in-this-database": "In this Database",
    "include-assets-message": "Enable extracting {{assets}} from the data source.",
+    "include-database-filter-extra-information": "Database which was added while creating service.",
    "include-lineage-message": "Configuration to turn off fetching lineage from pipelines.",
    "ingest-sample-data-for-entity": "Extract sample data from each {{entity}}.",
    "ingestion-bot-cant-be-deleted": "You can not delete the ingestion bot.",