Fix: #16380 support TestSuite redeploy option from UI (#16721)

* Fix: #16380 support TestSuite redeploy option from UI * move pipelines to settings page * fix * add playwright * update playwright to use dynamic data instead sample data
2025-08-30 20:06:19 +00:00 · 2024-06-27 16:48:20 +05:30 · 2024-06-27 16:48:20 +05:30 · ce6870d906
commit ce6870d906
parent f744705c41
17 changed files with 174 additions and 13 deletions
--- a/openmetadata-ui/src/main/resources/ui/playwright/constant/settings.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/constant/settings.ts
@ -66,6 +66,7 @@ export enum GlobalSettingOptions {
  APPLICATIONS = 'apps',
  OM_HEALTH = 'om-health',
  APPEARANCE = 'appearance',
+  DATA_OBSERVABILITY = 'dataObservability',
 }

 export const SETTINGS_OPTIONS_PATH = {
@ -103,6 +104,10 @@ export const SETTINGS_OPTIONS_PATH = {
    GlobalSettingsMenuCategory.SERVICES,
    `${GlobalSettingsMenuCategory.SERVICES}.${GlobalSettingOptions.METADATA}`,
  ],
+  [GlobalSettingOptions.DATA_OBSERVABILITY]: [
+    GlobalSettingsMenuCategory.SERVICES,
+    `${GlobalSettingsMenuCategory.SERVICES}.${GlobalSettingOptions.DATA_OBSERVABILITY}`,
+  ],

  // Applications

--- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/TestSuitePipelineRedeploy.spec.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/TestSuitePipelineRedeploy.spec.ts
@ -0,0 +1,71 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+import test, { expect } from '@playwright/test';
+import { GlobalSettingOptions } from '../../constant/settings';
+import { TableClass } from '../../support/entity/TableClass';
+import { createNewPage, redirectToHomePage } from '../../utils/common';
+import { settingClick } from '../../utils/sidebar';
+
+// use the admin user to login
+test.use({ storageState: 'playwright/.auth/admin.json' });
+
+const table1 = new TableClass();
+const table2 = new TableClass();
+
+test.describe('Bulk Re-Deploy pipelines ', () => {
+  test.beforeAll('Setup pre-requests', async ({ browser }) => {
+    const { afterAction, apiContext } = await createNewPage(browser);
+
+    await table1.create(apiContext);
+    await table2.create(apiContext);
+
+    await table1.createTestSuiteAndPipelines(apiContext);
+    await table2.createTestSuiteAndPipelines(apiContext);
+
+    await afterAction();
+  });
+
+  test.afterAll('Clean up', async ({ browser }) => {
+    const { afterAction, apiContext } = await createNewPage(browser);
+
+    await table1.delete(apiContext);
+    await table2.delete(apiContext);
+
+    await afterAction();
+  });
+
+  test.beforeEach('Visit home page', async ({ page }) => {
+    await redirectToHomePage(page);
+  });
+
+  test('Re-deploy all test-suite ingestion pipelines', async ({ page }) => {
+    await settingClick(page, GlobalSettingOptions.DATA_OBSERVABILITY);
+
+    await expect(
+      page.getByRole('button', { name: 'Re Deploy' })
+    ).not.toBeEnabled();
+    await expect(page.locator('.ant-table-container')).toBeVisible();
+
+    await page.getByRole('checkbox').first().click();
+
+    await expect(page.getByRole('button', { name: 'Re Deploy' })).toBeEnabled();
+
+    await page.getByRole('button', { name: 'Re Deploy' }).click();
+
+    await expect(
+      page.getByText('Pipelines Re Deploy Successfully')
+    ).toBeVisible();
+  });
+
+  // TODO: Add test to verify the re-deployed pipelines for Database, Dashboard and other entities
+});
--- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TableClass.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TableClass.ts
@ -158,6 +158,42 @@ export class TableClass extends EntityClass {
    });
  }

+  async createTestSuiteAndPipelines(apiContext: APIRequestContext) {
+    if (!this.entityResponseData) {
+      return Promise.reject('Entity not created');
+    }
+
+    const testSuiteData = await apiContext
+      .post('/api/v1/dataQuality/testSuites/executable', {
+        data: {
+          name: `pw-test-suite-${uuid()}`,
+          executableEntityReference:
+            this.entityResponseData['fullyQualifiedName'],
+          description: 'Playwright test suite for table',
+        },
+      })
+      .then((res) => res.json());
+
+    await apiContext.post(`/api/v1/services/ingestionPipelines`, {
+      data: {
+        airflowConfig: {},
+        name: `${this.entityResponseData['fullyQualifiedName']}_test_suite`,
+        pipelineType: 'TestSuite',
+        service: {
+          id: testSuiteData.id,
+          type: 'testSuite',
+        },
+        sourceConfig: {
+          config: {
+            type: 'TestSuite',
+            entityFullyQualifiedName:
+              this.entityResponseData['fullyQualifiedName'],
+          },
+        },
+      },
+    });
+  }
+
  async delete(apiContext: APIRequestContext) {
    const serviceResponse = await apiContext.delete(
      `/api/v1/services/databaseServices/name/${encodeURIComponent(
--- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionPipelineList/IngestionPipelineList.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionPipelineList/IngestionPipelineList.component.tsx
@ -18,6 +18,7 @@ import cronstrue from 'cronstrue';
 import { isNil, map, startCase } from 'lodash';
 import React, { useEffect, useMemo, useState } from 'react';
 import { useTranslation } from 'react-i18next';
+import { EntityType } from '../../../../../enums/entity.enum';
 import { ServiceCategory } from '../../../../../enums/service.enum';
 import {
  IngestionPipeline,
@ -48,8 +49,10 @@ import { IngestionRecentRuns } from '../IngestionRecentRun/IngestionRecentRuns.c

 export const IngestionPipelineList = ({
  serviceName,
+  className,
 }: {
-  serviceName: ServiceCategory;
+  serviceName: ServiceCategory | 'testSuites';
+  className?: string;
 }) => {
  const { theme } = useApplicationStore();
  const [pipelines, setPipelines] = useState<Array<IngestionPipeline>>();
@ -192,7 +195,10 @@ export const IngestionPipelineList = ({
    try {
      const { data, paging: pagingRes } = await getIngestionPipelines({
        arrQueryFields: ['owner'],
-        serviceType: getEntityTypeFromServiceCategory(serviceName),
+        serviceType:
+          serviceName === 'testSuites'
+            ? EntityType.TEST_SUITE
+            : getEntityTypeFromServiceCategory(serviceName),
        paging,
        pipelineType,
        limit,
@ -250,7 +256,7 @@ export const IngestionPipelineList = ({
  }

  return (
-    <Row gutter={[16, 16]}>
+    <Row className={className} gutter={[16, 16]}>
      <Col className="text-right" span={24}>
        <Button
          disabled={selectedPipelines?.length === 0}
--- a/openmetadata-ui/src/main/resources/ui/src/constants/GlobalSettings.constants.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/constants/GlobalSettings.constants.ts
@ -70,6 +70,7 @@ export enum GlobalSettingOptions {
  PROFILER_CONFIGURATION = 'profiler-configuration',
  APPEARANCE = 'appearance',
  DASHBOARD_DATA_MODEL = 'dashboardDataModels',
+  DATA_OBSERVABILITY = 'dataObservability',
 }

 export const GLOBAL_SETTING_PERMISSION_RESOURCES = [
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json
@ -1653,6 +1653,7 @@
    "page-sub-header-for-column-profile": "Überwache und verstehe die Struktur deiner Spalten mit dem Profiler.",
    "page-sub-header-for-customize-landing-page": "Customize your application landing page based on user personas",
    "page-sub-header-for-dashboards": "Ingestion von Metadaten aus den beliebtesten Dashboard-Diensten.",
+    "page-sub-header-for-data-observability": "Ingest metadata from test suite services right from the UI.",
    "page-sub-header-for-data-quality": "Vertrauen in deine Daten aufbauen mit Qualitätsprüfungen und zuverlässige Datenerzeugnisse erstellen.",
    "page-sub-header-for-databases": "Ingestion von Metadaten aus den beliebtesten Datenbankdiensten.",
    "page-sub-header-for-login-configuration": "Login configuration such as failed attempts or expiry timer.",
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json
@ -1653,6 +1653,7 @@
    "page-sub-header-for-column-profile": "Monitor and understand your columns structure with the profiler",
    "page-sub-header-for-customize-landing-page": "Customize your OpenMetadata landing page to cater to distinct user personas and their user experience.",
    "page-sub-header-for-dashboards": "Ingest metadata from the most popular dashboard services.",
+    "page-sub-header-for-data-observability": "Ingest metadata from test suite services right from the UI.",
    "page-sub-header-for-data-quality": "Build trust in your data with quality tests and create reliable data products.",
    "page-sub-header-for-databases": "Ingest metadata from the most popular database services.",
    "page-sub-header-for-login-configuration": "Define the handling of failed login attempts and JWT token expiry.",
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json
@ -1653,6 +1653,7 @@
    "page-sub-header-for-column-profile": "Monitorea y comprende la estructura de tus columnas con el perfilador",
    "page-sub-header-for-customize-landing-page": "Personaliza la página de inicio de tu aplicación basándote en las personas del usuario",
    "page-sub-header-for-dashboards": "Ingresa metadatos desde los servicios de dashboard más populares.",
+    "page-sub-header-for-data-observability": "Ingest metadata from test suite services right from the UI.",
    "page-sub-header-for-data-quality": "Construye confianza en tus datos con pruebas de calidad y crea productos de datos fiables.",
    "page-sub-header-for-databases": "Ingresa metadatos desde los servicios de base de datos más populares.",
    "page-sub-header-for-login-configuration": "Configuración de inicio de sesión como intentos fallidos o temporizador de vencimiento.",
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json
@ -1653,6 +1653,7 @@
    "page-sub-header-for-column-profile": "Surveillez et comprenez la structure de vos colonnes avec le profilage",
    "page-sub-header-for-customize-landing-page": "Customize your application landing page based on user personas",
    "page-sub-header-for-dashboards": "Ingestion de métadonnées à partir des services de tableau de bord les plus populaires.",
+    "page-sub-header-for-data-observability": "Ingest metadata from test suite services right from the UI.",
    "page-sub-header-for-data-quality": "Gagnez en confiance dans vos données grâce à des tests de qualité et créez des produits de données fiables.",
    "page-sub-header-for-databases": "Ingestion de métadonnées à partir des services de base de données les plus populaires.",
    "page-sub-header-for-login-configuration": "Login configuration such as failed attempts or expiry timer.",
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json
@ -1653,6 +1653,7 @@
    "page-sub-header-for-column-profile": "ניטור והבנה של מבנה העמודות שלך עם הפרופילר.",
    "page-sub-header-for-customize-landing-page": "התאמה אישית של דף הנחיתה באפליקציה שלך על פי דמויות המשתמש.",
    "page-sub-header-for-dashboards": "שלב מטה-דאטה ממוצרי הדשבורדים הפופולריים והנפוצים.",
+    "page-sub-header-for-data-observability": "Ingest metadata from test suite services right from the UI.",
    "page-sub-header-for-data-quality": "בנה אמון בנתונים שלך עם בדיקות איכות וצור מוצרי נתונים אמינים.",
    "page-sub-header-for-databases": "שלב מטה-דאטה מבסיס הנתונים הארגוניים. רוב בסיסי הנתונים הפופולריים נתמכים.",
    "page-sub-header-for-login-configuration": "Login configuration such as failed attempts or expiry timer.",
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json
@ -1653,6 +1653,7 @@
    "page-sub-header-for-column-profile": "Monitor and understand your columns structure with the profiler",
    "page-sub-header-for-customize-landing-page": "Customize your application landing page based on user personas",
    "page-sub-header-for-dashboards": "Ingest metadata from the most popular dashboard services.",
+    "page-sub-header-for-data-observability": "Ingest metadata from test suite services right from the UI.",
    "page-sub-header-for-data-quality": "Build trust in your data with quality tests and create reliable data products.",
    "page-sub-header-for-databases": "Ingest metadata from the most popular database services.",
    "page-sub-header-for-login-configuration": "Login configuration such as failed attempts or expiry timer.",
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json
@ -1653,6 +1653,7 @@
    "page-sub-header-for-column-profile": "Volg en begrijp de structuur van je kolommen met de profiler",
    "page-sub-header-for-customize-landing-page": "Uw OpenMetadata-landingspagina aanpassen aan verschillende gebruikerspersona's en hun gebruikerservaring",
    "page-sub-header-for-dashboards": "Ingest metadata van de meestgebruikte dashboardservices.",
+    "page-sub-header-for-data-observability": "Ingest metadata from test suite services right from the UI.",
    "page-sub-header-for-data-quality": "Bouw vertrouwen op in je data met kwaliteitstests en maak betrouwbare dataproducten.",
    "page-sub-header-for-databases": "Ingest metadata van de meestgebruikte databaseservices.",
    "page-sub-header-for-login-configuration": "Loginconfiguratie zoals mislukte pogingen of verlooptimer.",
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json
@ -1653,6 +1653,7 @@
    "page-sub-header-for-column-profile": "Monitore e compreenda a estrutura de suas colunas com o examinador.",
    "page-sub-header-for-customize-landing-page": "Personalize a página inicial do aplicativo com base nas personas do usuário.",
    "page-sub-header-for-dashboards": "Ingestão de metadados dos serviços de painel mais populares.",
+    "page-sub-header-for-data-observability": "Ingest metadata from test suite services right from the UI.",
    "page-sub-header-for-data-quality": "Construa confiança em seus dados com testes de qualidade e crie produtos de dados confiáveis.",
    "page-sub-header-for-databases": "Ingestão de metadados dos serviços de banco de dados mais populares.",
    "page-sub-header-for-login-configuration": "Login configuration such as failed attempts or expiry timer.",
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json
@ -1653,6 +1653,7 @@
    "page-sub-header-for-column-profile": "Отслеживайте и анализируйте структуру столбцов с помощью профайлера",
    "page-sub-header-for-customize-landing-page": "Customize your application landing page based on user personas",
    "page-sub-header-for-dashboards": "Получайте метаданные из самых популярных сервисов информационных панелей.",
+    "page-sub-header-for-data-observability": "Ingest metadata from test suite services right from the UI.",
    "page-sub-header-for-data-quality": "Укрепляйте доверие к своим данным с помощью тестов качества и создавайте надежные информационные продукты.",
    "page-sub-header-for-databases": "Получение метаданных из самых популярных сервисов баз данных.",
    "page-sub-header-for-login-configuration": "Login configuration such as failed attempts or expiry timer.",
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json
@ -1653,6 +1653,7 @@
    "page-sub-header-for-column-profile": "通过数据分析工具了解和跟踪您的数据表列结构",
    "page-sub-header-for-customize-landing-page": "Customize your application landing page based on user personas",
    "page-sub-header-for-dashboards": "从最流行的仪表板类型服务中提取元数据",
+    "page-sub-header-for-data-observability": "Ingest metadata from test suite services right from the UI.",
    "page-sub-header-for-data-quality": "通过引入数据质控测试提升数据的可信任度，构建稳健的衍生数据产品",
    "page-sub-header-for-databases": "从最流行的数据库类型服务中提取元数据",
    "page-sub-header-for-login-configuration": "Login configuration such as failed attempts or expiry timer.",
--- a/openmetadata-ui/src/main/resources/ui/src/pages/ServicesPage/ServicesPage.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/pages/ServicesPage/ServicesPage.tsx
@ -15,6 +15,7 @@ import { Col, Row, Tabs } from 'antd';
 import { capitalize, isEmpty } from 'lodash';
 import qs from 'qs';
 import React, { useMemo } from 'react';
+import { useTranslation } from 'react-i18next';
 import { useHistory, useLocation, useParams } from 'react-router-dom';
 import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder';
 import TitleBreadcrumb from '../../components/common/TitleBreadcrumb/TitleBreadcrumb.component';
@ -22,7 +23,10 @@ import { TitleBreadcrumbProps } from '../../components/common/TitleBreadcrumb/Ti
 import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1';
 import { IngestionPipelineList } from '../../components/Settings/Services/Ingestion/IngestionPipelineList/IngestionPipelineList.component';
 import Services from '../../components/Settings/Services/Services';
-import { GlobalSettingsMenuCategory } from '../../constants/GlobalSettings.constants';
+import {
+  GlobalSettingOptions,
+  GlobalSettingsMenuCategory,
+} from '../../constants/GlobalSettings.constants';
 import { SERVICE_CATEGORY } from '../../constants/Services.constant';
 import { usePermissionProvider } from '../../context/PermissionProvider/PermissionProvider';
 import { ERROR_PLACEHOLDER_TYPE } from '../../enums/common.enum';
@ -36,16 +40,22 @@ const ServicesPage = () => {
  const { tab } = useParams<{ tab: string }>();
  const location = useLocation();
  const history = useHistory();
+  const { t } = useTranslation();
  const { isAdminUser } = useAuth();
  const search =
    qs.parse(
      location.search.startsWith('?')
        ? location.search.substring(1)
        : location.search
-    ).tab ?? 'services';
+    ).tab ?? tab === GlobalSettingOptions.DATA_OBSERVABILITY
+      ? 'pipelines'
+      : 'services';

  const serviceName = useMemo(
-    () => SERVICE_CATEGORY[tab] ?? ServiceCategory.DATABASE_SERVICES,
+    () =>
+      tab === GlobalSettingOptions.DATA_OBSERVABILITY
+        ? 'dataObservabilityServices'
+        : SERVICE_CATEGORY[tab] ?? ServiceCategory.DATABASE_SERVICES,
    [tab]
  );

@ -65,7 +75,9 @@ const ServicesPage = () => {
    () =>
      getSettingPageEntityBreadCrumb(
        GlobalSettingsMenuCategory.SERVICES,
-        capitalize(tab)
+        tab === GlobalSettingOptions.DATA_OBSERVABILITY
+          ? t('label.data-observability')
+          : capitalize(tab)
      ),
    []
  );
@ -81,17 +93,27 @@ const ServicesPage = () => {
            destroyInactiveTabPane
            activeKey={search as string}
            items={[
-              {
-                key: 'services',
-                children: <Services serviceName={serviceName} />,
-                label: 'Services',
-              },
+              ...(serviceName === 'dataObservabilityServices'
+                ? []
+                : [
+                    {
+                      key: 'services',
+                      children: <Services serviceName={serviceName} />,
+                      label: 'Services',
+                    },
+                  ]),
              ...(isAdminUser
                ? [
                    {
                      key: 'pipelines',
                      children: (
-                        <IngestionPipelineList serviceName={serviceName} />
+                        <IngestionPipelineList
+                          serviceName={
+                            serviceName === 'dataObservabilityServices'
+                              ? 'testSuites'
+                              : serviceName
+                          }
+                        />
                      ),
                      label: 'Pipelines',
                    },
--- a/openmetadata-ui/src/main/resources/ui/src/utils/GlobalSettingsUtils.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/GlobalSettingsUtils.tsx
@ -36,6 +36,7 @@ import { ReactComponent as RolesIcon } from '../assets/svg/role-colored.svg';
 import { ReactComponent as SearchIcon } from '../assets/svg/search-colored.svg';
 import { ReactComponent as AccessControlIcon } from '../assets/svg/setting-access-control.svg';
 import { ReactComponent as CustomProperties } from '../assets/svg/setting-custom-properties.svg';
+import { ReactComponent as DataObservability } from '../assets/svg/setting-data-observability.svg';
 import { ReactComponent as ManagementIcon } from '../assets/svg/setting-management.svg';
 import { ReactComponent as NotificationIcon } from '../assets/svg/setting-notification.svg';
 import { ReactComponent as ServiceIcon } from '../assets/svg/setting-services.svg';
@ -160,6 +161,15 @@ export const getGlobalSettingsMenuWithPermission = (
          key: `${GlobalSettingsMenuCategory.SERVICES}.${GlobalSettingOptions.METADATA}`,
          icon: OpenMetadataIcon,
        },
+        {
+          label: i18next.t('label.data-observability'),
+          description: i18next.t(
+            'message.page-sub-header-for-data-observability'
+          ),
+          isProtected: true,
+          key: `${GlobalSettingsMenuCategory.SERVICES}.${GlobalSettingOptions.DATA_OBSERVABILITY}`,
+          icon: DataObservability,
+        },
      ],
    },
    {