Refactor ServiceBaseClass: Implement exponential backoff for API requests and enhance error handling in pipeline status retrieval (#20628)

* Refactor ServiceBaseClass: Implement exponential backoff for API requests and enhance error handling in pipeline status retrieval

* Enhance ServiceBaseClass: Introduce MAX_CONSECUTIVE_ERRORS constant and refactor API request handling with makeRetryRequest utility for improved error management

openmetadata-ui/src/main/resources/ui/playwright/constant/service.ts

@@ -91,3 +91,5 @@ export const DBT = {
   dataQualityTest1: 'unique_customers_customer_id',
   dataQualityTest2: 'not_null_customers_customer_id',
 };
+
+export const MAX_CONSECUTIVE_ERRORS = 3;

openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/ServiceBaseClass.ts

@@ -18,6 +18,7 @@ import {
   PlaywrightWorkerArgs,
   TestType,
 } from '@playwright/test';
+import { MAX_CONSECUTIVE_ERRORS } from '../../../constant/service';
 import {
   descriptionBox,
   getApiContext,
@@ -30,6 +31,7 @@ import { visitServiceDetailsPage } from '../../../utils/service';
 import {
   deleteService,
   getServiceCategoryFromService,
+  makeRetryRequest,
   Services,
   testConnection,
 } from '../../../utils/serviceIngestion';
@@ -275,19 +277,31 @@ class ServiceBaseClass {
     )[0];
 
     const oneHourBefore = Date.now() - 86400000;
+    let consecutiveErrors = 0;
 
     await expect
       .poll(
         async () => {
-          const response = await apiContext
-            .get(
-              `/api/v1/services/ingestionPipelines/${encodeURIComponent(
+          try {
+            const response = await makeRetryRequest({
+              url: `/api/v1/services/ingestionPipelines/${encodeURIComponent(
                 workflowData.fullyQualifiedName
-              )}/pipelineStatus?startTs=${oneHourBefore}&endTs=${Date.now()}`
-            )
-            .then((res) => res.json());
+              )}/pipelineStatus?startTs=${oneHourBefore}&endTs=${Date.now()}`,
+              page,
+            });
+            consecutiveErrors = 0; // Reset error counter on success
 
-          return response.data[0]?.pipelineState;
+            return response.data[0]?.pipelineState;
+          } catch (error) {
+            consecutiveErrors++;
+            if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
+              throw new Error(
+                `Failed to get pipeline status after ${MAX_CONSECUTIVE_ERRORS} consecutive attempts`
+              );
+            }
+
+            return 'running';
+          }
         },
         {
           // Custom expect message for reporting, optional.
@@ -311,7 +325,6 @@ class ServiceBaseClass {
     await page.waitForSelector('[data-testid="data-assets-header"]');
 
     await pipelinePromise;
-
     await statusPromise;
 
     await page.waitForSelector('[data-testid="ingestions"]');

openmetadata-ui/src/main/resources/ui/playwright/utils/serviceIngestion.ts

@@ -14,7 +14,7 @@
 import { expect, Page } from '@playwright/test';
 import { GlobalSettingOptions } from '../constant/settings';
 import { EntityTypeEndpoint } from '../support/entity/Entity.interface';
-import { toastNotification } from './common';
+import { getApiContext, toastNotification } from './common';
 import { escapeESReservedCharacters } from './entity';
 
 export enum Services {
@@ -160,3 +160,25 @@ export const checkServiceFieldSectionHighlighting = async (
 ) => {
   await page.waitForSelector(`[data-id="${field}"][data-highlighted="true"]`);
 };
+
+export const makeRetryRequest = async (data: {
+  url: string;
+  page: Page;
+  retries?: number;
+}) => {
+  const { url, page, retries = 3 } = data;
+  const { apiContext } = await getApiContext(page);
+
+  for (let i = 0; i < retries; i++) {
+    try {
+      const response = await apiContext.get(url);
+
+      return response.json();
+    } catch (error) {
+      if (i === retries - 1) {
+        throw error;
+      }
+      await page.waitForTimeout(1000 * (i + 1)); // Exponential backoff
+    }
+  }
+};