From d8a3e5f5edbd25d35c357a3ceb041253400ffe01 Mon Sep 17 00:00:00 2001 From: sonika-shah <58761340+sonika-shah@users.noreply.github.com> Date: Fri, 26 Sep 2025 18:17:56 +0530 Subject: [PATCH 1/8] UI: Handle deleted flag in /aggregate API to restore dropdown search results (#23558) * add searchSettings aggregation in searchSettings.json * refactor: update ExploreQuickFilters and ExploreUtils to include showDeleted and deleted parameters * feat: enhance ExploreDiscovery tests to handle deleted assets visibility based on showDeleted toggle * Revert schemaChanges.sql files to match main --------- Co-authored-by: Aniket Katkar --- .../e2e/Flow/ExploreDiscovery.spec.ts | 185 +++++++++++++++++- .../Explore/ExploreQuickFilters.tsx | 6 +- .../src/main/resources/ui/src/rest/miscAPI.ts | 4 +- .../resources/ui/src/utils/ExploreUtils.tsx | 77 ++++---- 4 files changed, 221 insertions(+), 51 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/ExploreDiscovery.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/ExploreDiscovery.spec.ts index f4829e13a08..98dccfb8004 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/ExploreDiscovery.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/ExploreDiscovery.spec.ts @@ -11,33 +11,58 @@ * limitations under the License. */ import test, { expect } from '@playwright/test'; +import { SidebarItem } from '../../constant/sidebar'; +import { Domain } from '../../support/domain/Domain'; import { TableClass } from '../../support/entity/TableClass'; +import { UserClass } from '../../support/user/UserClass'; import { createNewPage, redirectToHomePage } from '../../utils/common'; +import { + getEncodedFqn, + waitForAllLoadersToDisappear, +} from '../../utils/entity'; import { getJsonTreeObject } from '../../utils/exploreDiscovery'; +import { sidebarClick } from '../../utils/sidebar'; // use the admin user to login test.use({ storageState: 'playwright/.auth/admin.json' }); const table = new TableClass(); const table1 = new TableClass(); +const user = new UserClass(); +const domain = new Domain(); test.describe('Explore Assets Discovery', () => { test.beforeAll(async ({ browser }) => { const { apiContext, afterAction } = await createNewPage(browser); + await user.create(apiContext); + await domain.create(apiContext); await table.create(apiContext); await table1.create(apiContext); + await table.patch({ + apiContext, + patchData: [ + { + op: 'add', + value: { + type: 'user', + id: user.responseData.id, + }, + path: '/owners/0', + }, + { + op: 'add', + path: '/domains/0', + value: { + id: domain.responseData.id, + type: 'domain', + name: domain.responseData.name, + displayName: domain.responseData.displayName, + }, + }, + ], + }); await table.delete(apiContext, false); - // await table1.delete(apiContext, false); - - await afterAction(); - }); - - test.afterAll(async ({ browser }) => { - const { apiContext, afterAction } = await createNewPage(browser); - - await table.delete(apiContext); - await table1.delete(apiContext); await afterAction(); }); @@ -229,4 +254,144 @@ test.describe('Explore Assets Discovery', () => { page.locator('.ant-popover-inner-content').textContent() ).not.toContain(table1.entityResponseData.name); }); + + test('Should not display domain and owner of deleted asset in suggestions when showDeleted is off', async ({ + page, + }) => { + await sidebarClick(page, SidebarItem.EXPLORE); + await page.waitForLoadState('networkidle'); + await waitForAllLoadersToDisappear(page); + + // The user should not be visible in the owners filter when the deleted switch is off + await page.click('[data-testid="search-dropdown-Owners"]'); + const searchResOwner = page.waitForResponse( + `/api/v1/search/aggregate?index=dataAsset&field=owners.displayName.keyword*deleted=false*` + ); + + await page.fill( + '[data-testid="search-input"]', + user.responseData.displayName + ); + await searchResOwner; + + await waitForAllLoadersToDisappear(page); + + await expect( + page + .getByTestId('drop-down-menu') + .getByTestId(user.responseData.displayName) + ).not.toBeAttached(); + + await page.getByTestId('close-btn').click(); + + // The domain should not be visible in the domains filter when the deleted switch is off + await page.click('[data-testid="search-dropdown-Domains"]'); + + const searchResDomain = page.waitForResponse( + `/api/v1/search/aggregate?index=dataAsset&field=domains.displayName.keyword*deleted=false*` + ); + + await page.fill( + '[data-testid="search-input"]', + domain.responseData.displayName + ); + await searchResDomain; + + await waitForAllLoadersToDisappear(page); + + await expect( + page + .getByTestId('drop-down-menu') + .getByTestId(domain.responseData.displayName) + ).not.toBeAttached(); + + await page.getByTestId('close-btn').click(); + }); + + test('Should display domain and owner of deleted asset in suggestions when showDeleted is on', async ({ + page, + }) => { + await sidebarClick(page, SidebarItem.EXPLORE); + await page.waitForLoadState('networkidle'); + await waitForAllLoadersToDisappear(page); + + // Click on the show deleted toggle button + await page.getByTestId('show-deleted').click(); + + await page.waitForLoadState('networkidle'); + await waitForAllLoadersToDisappear(page); + + // The user should be visible in the owners filter when the deleted switch is on + const ownerSearchText = user.responseData.displayName.toLowerCase(); + await page.click('[data-testid="search-dropdown-Owners"]'); + + const searchResOwner = page.waitForResponse( + `/api/v1/search/aggregate?index=dataAsset&field=owners.displayName.keyword*deleted=true*` + ); + + await page.fill('[data-testid="search-input"]', ownerSearchText); + await searchResOwner; + + await waitForAllLoadersToDisappear(page); + + await expect( + page.getByTestId('drop-down-menu').getByTestId(ownerSearchText) + ).toBeAttached(); + + await page + .getByTestId('drop-down-menu') + .getByTestId(ownerSearchText) + .click(); + + const fetchWithOwner = page.waitForResponse( + `/api/v1/search/query?*deleted=true*owners.displayName.keyword*${ownerSearchText}*` + ); + await page.getByTestId('update-btn').click(); + await fetchWithOwner; + + await page.waitForLoadState('networkidle'); + await waitForAllLoadersToDisappear(page); + + // The domain should be visible in the domains filter when the deleted switch is on + const domainSearchText = domain.responseData.displayName.toLowerCase(); + await page.click('[data-testid="search-dropdown-Domains"]'); + + const searchResDomain = page.waitForResponse( + `/api/v1/search/aggregate?index=dataAsset&field=domains.displayName.keyword*deleted=true*` + ); + + await page.fill('[data-testid="search-input"]', domainSearchText); + await searchResDomain; + + await waitForAllLoadersToDisappear(page); + + await expect( + page.getByTestId('drop-down-menu').getByTestId(domainSearchText) + ).toBeAttached(); + + await page + .getByTestId('drop-down-menu') + .getByTestId(domainSearchText) + .click(); + + const fetchWithDomain = page.waitForResponse( + `/api/v1/search/query?*deleted=true*domains.displayName.keyword*${getEncodedFqn( + domainSearchText, + true + )}*` + ); + await page.getByTestId('update-btn').click(); + await fetchWithDomain; + + await page.waitForLoadState('networkidle'); + await waitForAllLoadersToDisappear(page); + + // Only the table option should be visible for the data assets filter when the deleted switch is on + // with the owner and domain filter applied + await page.click('[data-testid="search-dropdown-Data Assets"]'); + + await expect( + page.getByTestId('drop-down-menu').getByTestId('table') + ).toBeAttached(); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.tsx index a5140471bdb..5db2b7fc73a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.tsx @@ -94,7 +94,8 @@ const ExploreQuickFilters: FC = ({ key, '', JSON.stringify(combinedQueryFilter), - independent + independent, + showDeleted ), key === TIER_FQN_KEY ? getTags({ parent: 'Tier', limit: 50 }) @@ -159,7 +160,8 @@ const ExploreQuickFilters: FC = ({ key, value, JSON.stringify(combinedQueryFilter), - independent + independent, + showDeleted ); const buckets = res.data.aggregations[`sterms#${key}`].buckets; diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/miscAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/miscAPI.ts index 1cb7c8310d1..fa5b71f9d03 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/miscAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/miscAPI.ts @@ -168,7 +168,8 @@ export const getAggregateFieldOptions = ( field: string, value: string, q: string, - sourceFields?: string + sourceFields?: string, + deleted = false ) => { const withWildCardValue = value ? `.*${escapeESReservedCharacters(value)}.*` @@ -179,6 +180,7 @@ export const getAggregateFieldOptions = ( value: withWildCardValue, q, sourceFields, + deleted, }; return APIClient.get>( diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ExploreUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/ExploreUtils.tsx index d3c3451e729..525bab30813 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ExploreUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ExploreUtils.tsx @@ -200,41 +200,6 @@ export const extractTermKeys = (objects: QueryFieldInterface[]): string[] => { return termKeys; }; -export const getSubLevelHierarchyKey = ( - isDatabaseHierarchy = false, - filterField?: ExploreQuickFilterField[], - key?: EntityFields, - value?: string -) => { - const queryFilter = { - query: { bool: {} }, - }; - - if ((key && value) || filterField) { - (queryFilter.query.bool as EsBoolQuery).must = isUndefined(filterField) - ? { term: { [key ?? '']: value } } - : getExploreQueryFilterMust(filterField); - } - - const bucketMapping = isDatabaseHierarchy - ? { - [EntityFields.SERVICE_TYPE]: EntityFields.SERVICE, - [EntityFields.SERVICE]: EntityFields.DATABASE_DISPLAY_NAME, - [EntityFields.DATABASE_DISPLAY_NAME]: - EntityFields.DATABASE_SCHEMA_DISPLAY_NAME, - [EntityFields.DATABASE_SCHEMA_DISPLAY_NAME]: EntityFields.ENTITY_TYPE, - } - : { - [EntityFields.SERVICE_TYPE]: EntityFields.SERVICE, - [EntityFields.SERVICE]: EntityFields.ENTITY_TYPE, - }; - - return { - bucket: bucketMapping[key as DatabaseFields] ?? EntityFields.SERVICE_TYPE, - queryFilter, - }; -}; - export const getExploreQueryFilterMust = (data: ExploreQuickFilterField[]) => { const must = [] as Array; @@ -281,6 +246,41 @@ export const getExploreQueryFilterMust = (data: ExploreQuickFilterField[]) => { return must; }; +export const getSubLevelHierarchyKey = ( + isDatabaseHierarchy = false, + filterField?: ExploreQuickFilterField[], + key?: EntityFields, + value?: string +) => { + const queryFilter = { + query: { bool: {} }, + }; + + if ((key && value) || filterField) { + (queryFilter.query.bool as EsBoolQuery).must = isUndefined(filterField) + ? { term: { [key ?? '']: value } } + : getExploreQueryFilterMust(filterField); + } + + const bucketMapping = isDatabaseHierarchy + ? { + [EntityFields.SERVICE_TYPE]: EntityFields.SERVICE, + [EntityFields.SERVICE]: EntityFields.DATABASE_DISPLAY_NAME, + [EntityFields.DATABASE_DISPLAY_NAME]: + EntityFields.DATABASE_SCHEMA_DISPLAY_NAME, + [EntityFields.DATABASE_SCHEMA_DISPLAY_NAME]: EntityFields.ENTITY_TYPE, + } + : { + [EntityFields.SERVICE_TYPE]: EntityFields.SERVICE, + [EntityFields.SERVICE]: EntityFields.ENTITY_TYPE, + }; + + return { + bucket: bucketMapping[key as DatabaseFields] ?? EntityFields.SERVICE_TYPE, + queryFilter, + }; +}; + export const updateTreeData = ( list: ExploreTreeNode[], key: React.Key, @@ -360,11 +360,12 @@ export const getAggregationOptions = async ( key: string, value: string, filter: string, - isIndependent: boolean + isIndependent: boolean, + deleted = false ) => { return isIndependent ? postAggregateFieldOptions(index, key, value, filter) - : getAggregateFieldOptions(index, key, value, filter); + : getAggregateFieldOptions(index, key, value, filter, undefined, deleted); }; export const updateTreeDataWithCounts = ( @@ -412,7 +413,7 @@ export const isElasticsearchError = (error: unknown): boolean => { return false; } - const data = axiosError.response.data as Record; + const data = axiosError.response.data as Record; const message = data.message as string; return ( From 94104e080677b1bc506b22150c120de0e9a419b2 Mon Sep 17 00:00:00 2001 From: Keshav Mohta <68001229+keshavmohta09@users.noreply.github.com> Date: Fri, 26 Sep 2025 18:22:01 +0530 Subject: [PATCH 2/8] fix: lineage flow and improved logging for databricks pipeline (#23586) --- .../source/database/databricks/client.py | 14 ++++++++++- .../source/database/databricks/queries.py | 2 ++ .../pipeline/databrickspipeline/metadata.py | 25 ++++++++++++------- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/client.py b/ingestion/src/metadata/ingestion/source/database/databricks/client.py index 2462de40ddd..f0fc17f092c 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/client.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/client.py @@ -72,7 +72,9 @@ class DatabricksClient: "Content-Type": "application/json", } self.api_timeout = self.config.connectionTimeout or 120 + self._job_table_lineage_executed: bool = False self.job_table_lineage: dict[str, list[dict[str, str]]] = {} + self._job_column_lineage_executed: bool = False self.job_column_lineage: dict[ str, dict[Tuple[str, str], list[Tuple[str, str]]] ] = {} @@ -193,7 +195,6 @@ class DatabricksClient: """ Method returns List all the created jobs in a Databricks Workspace """ - self.cache_lineage() try: iteration_count = 1 data = {"limit": PAGE_SIZE, "expand_tasks": True, "offset": 0} @@ -266,6 +267,10 @@ class DatabricksClient: Method returns table lineage for a job by the specified job_id """ try: + if not self._job_table_lineage_executed: + logger.debug("Executing cache_lineage...") + self.cache_lineage() + return self.job_table_lineage.get(str(job_id)) except Exception as exc: logger.debug( @@ -281,6 +286,10 @@ class DatabricksClient: Method returns column lineage for a job by the specified job_id and table key """ try: + if not self._job_column_lineage_executed: + logger.debug("Job column lineage not found. Executing cache_lineage...") + self.cache_lineage() + return self.job_column_lineage.get(str(job_id), {}).get(TableKey) except Exception as exc: logger.debug( @@ -325,6 +334,7 @@ class DatabricksClient: f"Error parsing row: {row} due to {traceback.format_exc()}" ) continue + self._job_table_lineage_executed = True # Not every job has column lineage, so we need to check if the job exists in the column_lineage table # we will cache the column lineage for jobs that have column lineage @@ -355,3 +365,5 @@ class DatabricksClient: f"Error parsing row: {row} due to {traceback.format_exc()}" ) continue + self._job_column_lineage_executed = True + logger.debug("Table and column lineage caching completed.") diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/queries.py b/ingestion/src/metadata/ingestion/source/database/databricks/queries.py index 80b86819f70..ad29712d1b7 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/queries.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/queries.py @@ -96,6 +96,7 @@ DATABRICKS_GET_TABLE_LINEAGE_FOR_JOB = textwrap.dedent( target_table_full_name as target_table_full_name FROM system.access.table_lineage WHERE entity_type ILIKE 'job' + AND event_time >= current_date() - INTERVAL 90 DAYS """ ) @@ -110,5 +111,6 @@ DATABRICKS_GET_COLUMN_LINEAGE_FOR_JOB = textwrap.dedent( target_column_name as target_column_name FROM system.access.column_lineage WHERE entity_type ILIKE 'job' + AND event_time >= current_date() - INTERVAL 90 DAYS """ ) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/databrickspipeline/metadata.py b/ingestion/src/metadata/ingestion/source/pipeline/databrickspipeline/metadata.py index c32c89a5c69..0ff1c8da83b 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/databrickspipeline/metadata.py @@ -130,9 +130,11 @@ class DatabrickspipelineSource(PipelineServiceSource): displayName=pipeline_details.settings.name, description=Markdown(description) if description else None, tasks=self.get_tasks(pipeline_details), - scheduleInterval=str(pipeline_details.settings.schedule.cron) - if pipeline_details.settings.schedule - else None, + scheduleInterval=( + str(pipeline_details.settings.schedule.cron) + if pipeline_details.settings.schedule + else None + ), service=FullyQualifiedEntityName(self.context.get().pipeline_service), ) yield Either(right=pipeline_request) @@ -176,12 +178,14 @@ class DatabrickspipelineSource(PipelineServiceSource): Task( name=str(task.name), taskType=pipeline_details.settings.task_type, - sourceUrl=SourceUrl(run.run_page_url) - if run.run_page_url - else None, - description=Markdown(task.description) - if task.description - else None, + sourceUrl=( + SourceUrl(run.run_page_url) + if run.run_page_url + else None + ), + description=( + Markdown(task.description) if task.description else None + ), downstreamTasks=[ depend_task.name for depend_task in task.depends_on or [] @@ -314,6 +318,9 @@ class DatabrickspipelineSource(PipelineServiceSource): table_lineage_list = self.client.get_table_lineage( job_id=pipeline_details.job_id ) + logger.debug( + f"Processing pipeline lineage for job {pipeline_details.job_id}" + ) if table_lineage_list: for table_lineage in table_lineage_list: source_table_full_name = table_lineage.get("source_table_full_name") From f528616b2fecdd7cf76756d6bc69dd6968310792 Mon Sep 17 00:00:00 2001 From: Eugenio Date: Fri, 26 Sep 2025 15:30:15 +0200 Subject: [PATCH 3/8] Identify when Airflow DAG import errors in CI (#23589) --- docker/run_local_docker.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker/run_local_docker.sh b/docker/run_local_docker.sh index 183b2f491e3..b220df798ca 100755 --- a/docker/run_local_docker.sh +++ b/docker/run_local_docker.sh @@ -10,6 +10,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +RED='\033[0;31m' + cd "$(dirname "${BASH_SOURCE[0]}")" || exit helpFunction() @@ -116,6 +118,13 @@ until curl -s -f "http://localhost:9200/_cat/indices/openmetadata_team_search_in done until curl -s -f --header 'Authorization: Basic YWRtaW46YWRtaW4=' "http://localhost:8080/api/v1/dags/sample_data"; do + IMPORT_ERRORS="$(curl -s --header 'Authorization: Basic YWRtaW46YWRtaW4=' "http://localhost:8080/api/v1/importErrors")" + echo $IMPORT_ERRORS | grep "/airflow_sample_data.py" > /dev/null; + if [[ "$?" == "0" ]]; then + echo -e "${RED}Airflow found an error importing \`sample_data\` DAG" + printf '%s' "$IMPORT_ERRORS" | jq '.["import_errors"] | .[] | select(.filename | endswith("airflow_sample_data.py"))' + exit 1 + fi echo 'Checking if Sample Data DAG is reachable...\n' sleep 5 done From bcae472cc0ff1fdb02b37964f677a4ba473d2a1d Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Fri, 26 Sep 2025 20:17:09 +0530 Subject: [PATCH 4/8] improve playwright test of bulk import/edit (#23538) --- .../e2e/Features/BulkEditEntity.spec.ts | 79 +++++++------- .../e2e/Features/BulkImport.spec.ts | 100 ++++++++++-------- 2 files changed, 95 insertions(+), 84 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/BulkEditEntity.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/BulkEditEntity.spec.ts index b2177e96028..c73a8c59497 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/BulkEditEntity.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/BulkEditEntity.spec.ts @@ -15,10 +15,11 @@ import { expect, test } from '@playwright/test'; import { RDG_ACTIVE_CELL_SELECTOR } from '../../constant/bulkImportExport'; import { SERVICE_TYPE } from '../../constant/service'; import { GlobalSettingOptions } from '../../constant/settings'; -import { EntityDataClass } from '../../support/entity/EntityDataClass'; +import { Domain } from '../../support/domain/Domain'; import { TableClass } from '../../support/entity/TableClass'; import { Glossary } from '../../support/glossary/Glossary'; import { GlossaryTerm } from '../../support/glossary/GlossaryTerm'; +import { UserClass } from '../../support/user/UserClass'; import { createNewPage, descriptionBoxReadOnly, @@ -49,9 +50,16 @@ test.use({ storageState: 'playwright/.auth/admin.json', }); +const user1 = new UserClass(); +const user2 = new UserClass(); +const glossary = new Glossary(); +const glossaryTerm = new GlossaryTerm(glossary); +const domain1 = new Domain(); +const domain2 = new Domain(); + const glossaryDetails = { - name: EntityDataClass.glossaryTerm1.data.name, - parent: EntityDataClass.glossary1.data.name, + name: glossaryTerm.data.name, + parent: glossary.data.name, }; const databaseSchemaDetails1 = { @@ -70,19 +78,16 @@ const columnDetails1 = { }; test.describe('Bulk Edit Entity', () => { - test.beforeAll('setup pre-test', async ({ browser }, testInfo) => { + test.beforeAll('setup pre-test', async ({ browser }) => { const { apiContext, afterAction } = await createNewPage(browser); - testInfo.setTimeout(90000); - await EntityDataClass.preRequisitesForTests(apiContext); - await afterAction(); - }); + await user1.create(apiContext); + await user2.create(apiContext); + await glossary.create(apiContext); + await glossaryTerm.create(apiContext); + await domain1.create(apiContext); + await domain2.create(apiContext); - test.afterAll('Cleanup', async ({ browser }, testInfo) => { - const { apiContext, afterAction } = await createNewPage(browser); - - testInfo.setTimeout(90000); - await EntityDataClass.postRequisitesForTests(apiContext); await afterAction(); }); @@ -109,7 +114,7 @@ test.describe('Bulk Edit Entity', () => { await test.step('Perform bulk edit action', async () => { const databaseDetails = { ...createDatabaseRowDetails(), - domains: EntityDataClass.domain1.responseData, + domains: domain1.responseData, glossary: glossaryDetails, }; @@ -145,8 +150,8 @@ test.describe('Bulk Edit Entity', () => { ...databaseDetails, name: table.database.name, owners: [ - EntityDataClass.user1.responseData?.['displayName'], - EntityDataClass.user2.responseData?.['displayName'], + user1.responseData?.['displayName'], + user2.responseData?.['displayName'], ], retentionPeriod: undefined, sourceUrl: undefined, @@ -191,10 +196,10 @@ test.describe('Bulk Edit Entity', () => { // Verify Owners await expect( - page.getByTestId(EntityDataClass.user1.responseData?.['displayName']) + page.getByTestId(user1.responseData?.['displayName']) ).toBeVisible(); await expect( - page.getByTestId(EntityDataClass.user2.responseData?.['displayName']) + page.getByTestId(user2.responseData?.['displayName']) ).toBeVisible(); // Verify Tags @@ -212,7 +217,7 @@ test.describe('Bulk Edit Entity', () => { await expect( page.getByRole('link', { - name: EntityDataClass.glossaryTerm1.data.displayName, + name: glossaryTerm.data.displayName, }) ).toBeVisible(); }); @@ -280,10 +285,10 @@ test.describe('Bulk Edit Entity', () => { ...databaseSchemaDetails1, name: table.schema.name, owners: [ - EntityDataClass.user1.responseData?.['displayName'], - EntityDataClass.user2.responseData?.['displayName'], + user1.responseData?.['displayName'], + user2.responseData?.['displayName'], ], - domains: EntityDataClass.domain1.responseData, + domains: domain1.responseData, }, page, customPropertyRecord @@ -327,11 +332,11 @@ test.describe('Bulk Edit Entity', () => { // Verify Owners await expect( - page.getByTestId(EntityDataClass.user1.responseData?.['displayName']) + page.getByTestId(user1.responseData?.['displayName']) ).toBeVisible(); await expect( - page.getByTestId(EntityDataClass.user2.responseData?.['displayName']) + page.getByTestId(user2.responseData?.['displayName']) ).toBeVisible(); await page.getByTestId('column-display-name').click(); @@ -354,7 +359,7 @@ test.describe('Bulk Edit Entity', () => { await expect( page.getByRole('link', { - name: EntityDataClass.glossaryTerm1.data.displayName, + name: glossaryTerm.data.displayName, }) ).toBeVisible(); }); @@ -420,10 +425,10 @@ test.describe('Bulk Edit Entity', () => { ...tableDetails1, name: table.entity.name, owners: [ - EntityDataClass.user1.responseData?.['displayName'], - EntityDataClass.user2.responseData?.['displayName'], + user1.responseData?.['displayName'], + user2.responseData?.['displayName'], ], - domains: EntityDataClass.domain1.responseData, + domains: domain1.responseData, }, page, customPropertyRecord @@ -464,16 +469,16 @@ test.describe('Bulk Edit Entity', () => { // Verify Domain await expect(page.getByTestId('domain-link')).toContainText( - EntityDataClass.domain1.responseData.displayName + domain1.responseData.displayName ); // Verify Owners await expect( - page.getByTestId(EntityDataClass.user1.responseData?.['displayName']) + page.getByTestId(user1.responseData?.['displayName']) ).toBeVisible(); await expect( - page.getByTestId(EntityDataClass.user2.responseData?.['displayName']) + page.getByTestId(user2.responseData?.['displayName']) ).toBeVisible(); // Verify Tags @@ -491,7 +496,7 @@ test.describe('Bulk Edit Entity', () => { await expect( page.getByRole('link', { - name: EntityDataClass.glossaryTerm1.data.displayName, + name: glossaryTerm.data.displayName, }) ).toBeVisible(); }); @@ -583,7 +588,7 @@ test.describe('Bulk Edit Entity', () => { await expect( page.getByRole('link', { - name: EntityDataClass.glossaryTerm1.data.displayName, + name: glossaryTerm.data.displayName, }) ).toBeVisible(); }); @@ -626,8 +631,8 @@ test.describe('Bulk Edit Entity', () => { { ...additionalGlossaryTerm, name: glossaryTerm.data.name, - owners: [EntityDataClass.user1.responseData?.['displayName']], - reviewers: [EntityDataClass.user2.responseData?.['displayName']], + owners: [user1.responseData?.['displayName']], + reviewers: [user2.responseData?.['displayName']], relatedTerm: { parent: glossary.data.name, name: glossaryTerm.data.name, @@ -689,12 +694,12 @@ test.describe('Bulk Edit Entity', () => { // Verify Owners await expect( - page.getByTestId(EntityDataClass.user1.responseData?.['displayName']) + page.getByTestId(user1.responseData?.['displayName']) ).toBeVisible(); // Verify Reviewers await expect( - page.getByTestId(EntityDataClass.user2.responseData?.['displayName']) + page.getByTestId(user2.responseData?.['displayName']) ).toBeVisible(); }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/BulkImport.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/BulkImport.spec.ts index e24f240346d..c4b97bd3511 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/BulkImport.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/BulkImport.spec.ts @@ -14,11 +14,14 @@ import { expect, test } from '@playwright/test'; import { RDG_ACTIVE_CELL_SELECTOR } from '../../constant/bulkImportExport'; import { GlobalSettingOptions } from '../../constant/settings'; +import { Domain } from '../../support/domain/Domain'; import { DatabaseClass } from '../../support/entity/DatabaseClass'; import { DatabaseSchemaClass } from '../../support/entity/DatabaseSchemaClass'; -import { EntityDataClass } from '../../support/entity/EntityDataClass'; import { DatabaseServiceClass } from '../../support/entity/service/DatabaseServiceClass'; import { TableClass } from '../../support/entity/TableClass'; +import { Glossary } from '../../support/glossary/Glossary'; +import { GlossaryTerm } from '../../support/glossary/GlossaryTerm'; +import { UserClass } from '../../support/user/UserClass'; import { createNewPage, getApiContext, @@ -53,9 +56,16 @@ test.use({ }, }); +const user1 = new UserClass(); +const user2 = new UserClass(); +const glossary = new Glossary(); +const glossaryTerm = new GlossaryTerm(glossary); +const domain1 = new Domain(); +const domain2 = new Domain(); + const glossaryDetails = { - name: EntityDataClass.glossaryTerm1.data.name, - parent: EntityDataClass.glossary1.data.name, + name: glossaryTerm.data.name, + parent: glossary.data.name, }; const databaseDetails1 = { @@ -104,19 +114,15 @@ const storedProcedureDetails = { }; test.describe('Bulk Import Export', () => { - test.beforeAll('setup pre-test', async ({ browser }, testInfo) => { + test.beforeAll('setup pre-test', async ({ browser }) => { const { apiContext, afterAction } = await createNewPage(browser); - testInfo.setTimeout(90000); - await EntityDataClass.preRequisitesForTests(apiContext); - await afterAction(); - }); - - test.afterAll('Cleanup', async ({ browser }, testInfo) => { - const { apiContext, afterAction } = await createNewPage(browser); - - testInfo.setTimeout(90000); - await EntityDataClass.postRequisitesForTests(apiContext); + await user1.create(apiContext); + await user2.create(apiContext); + await glossary.create(apiContext); + await glossaryTerm.create(apiContext); + await domain1.create(apiContext); + await domain2.create(apiContext); await afterAction(); }); @@ -176,10 +182,10 @@ test.describe('Bulk Import Export', () => { { ...databaseDetails1, owners: [ - EntityDataClass.user1.responseData?.['displayName'], - EntityDataClass.user2.responseData?.['displayName'], + user1.responseData?.['displayName'], + user2.responseData?.['displayName'], ], - domains: EntityDataClass.domain1.responseData, + domains: domain1.responseData, }, page, customPropertyRecord @@ -206,10 +212,10 @@ test.describe('Bulk Import Export', () => { { ...databaseSchemaDetails1, owners: [ - EntityDataClass.user1.responseData?.['displayName'], - EntityDataClass.user2.responseData?.['displayName'], + user1.responseData?.['displayName'], + user2.responseData?.['displayName'], ], - domains: EntityDataClass.domain1.responseData, + domains: domain1.responseData, }, page ); @@ -236,10 +242,10 @@ test.describe('Bulk Import Export', () => { { ...tableDetails1, owners: [ - EntityDataClass.user1.responseData?.['displayName'], - EntityDataClass.user2.responseData?.['displayName'], + user1.responseData?.['displayName'], + user2.responseData?.['displayName'], ], - domains: EntityDataClass.domain1.responseData, + domains: domain1.responseData, }, page ); @@ -284,10 +290,10 @@ test.describe('Bulk Import Export', () => { { ...storedProcedureDetails, owners: [ - EntityDataClass.user1.responseData?.['displayName'], - EntityDataClass.user2.responseData?.['displayName'], + user1.responseData?.['displayName'], + user2.responseData?.['displayName'], ], - domains: EntityDataClass.domain2.responseData, + domains: domain2.responseData, }, page ); @@ -317,10 +323,10 @@ test.describe('Bulk Import Export', () => { { ...databaseDetails2, owners: [ - EntityDataClass.user1.responseData?.['displayName'], - EntityDataClass.user2.responseData?.['displayName'], + user1.responseData?.['displayName'], + user2.responseData?.['displayName'], ], - domains: EntityDataClass.domain2.responseData, + domains: domain2.responseData, }, page ); @@ -426,10 +432,10 @@ test.describe('Bulk Import Export', () => { { ...databaseSchemaDetails1, owners: [ - EntityDataClass.user1.responseData?.['displayName'], - EntityDataClass.user2.responseData?.['displayName'], + user1.responseData?.['displayName'], + user2.responseData?.['displayName'], ], - domains: EntityDataClass.domain1.responseData, + domains: domain1.responseData, }, page, customPropertyRecord @@ -457,10 +463,10 @@ test.describe('Bulk Import Export', () => { { ...tableDetails1, owners: [ - EntityDataClass.user1.responseData?.['displayName'], - EntityDataClass.user2.responseData?.['displayName'], + user1.responseData?.['displayName'], + user2.responseData?.['displayName'], ], - domains: EntityDataClass.domain1.responseData, + domains: domain1.responseData, }, page ); @@ -505,10 +511,10 @@ test.describe('Bulk Import Export', () => { { ...databaseSchemaDetails2, owners: [ - EntityDataClass.user1.responseData?.['displayName'], - EntityDataClass.user2.responseData?.['displayName'], + user1.responseData?.['displayName'], + user2.responseData?.['displayName'], ], - domains: EntityDataClass.domain1.responseData, + domains: domain1.responseData, }, page ); @@ -626,10 +632,10 @@ test.describe('Bulk Import Export', () => { { ...tableDetails1, owners: [ - EntityDataClass.user1.responseData?.['displayName'], - EntityDataClass.user2.responseData?.['displayName'], + user1.responseData?.['displayName'], + user2.responseData?.['displayName'], ], - domains: EntityDataClass.domain1.responseData, + domains: domain1.responseData, }, page, customPropertyRecord @@ -676,10 +682,10 @@ test.describe('Bulk Import Export', () => { { ...tableDetails2, owners: [ - EntityDataClass.user1.responseData?.['displayName'], - EntityDataClass.user2.responseData?.['displayName'], + user1.responseData?.['displayName'], + user2.responseData?.['displayName'], ], - domains: EntityDataClass.domain1.responseData, + domains: domain1.responseData, }, page, customPropertyRecord @@ -881,10 +887,10 @@ test.describe('Bulk Import Export', () => { { ...databaseDetails1, owners: [ - EntityDataClass.user1.responseData?.['displayName'], - EntityDataClass.user2.responseData?.['displayName'], + user1.responseData?.['displayName'], + user2.responseData?.['displayName'], ], - domains: EntityDataClass.domain1.responseData, + domains: domain1.responseData, }, page, undefined, From f934b79d651a45c63f68f9e55065a1e0155f6ace Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Fri, 26 Sep 2025 20:17:19 +0530 Subject: [PATCH 5/8] supported scroll in certification popover list (#23534) --- .../resources/ui/playwright/utils/common.ts | 41 ++++ .../resources/ui/playwright/utils/entity.ts | 20 ++ .../Certification/Certification.component.tsx | 212 +++++++++++------- 3 files changed, 190 insertions(+), 83 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/common.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/common.ts index a0d3514c480..d31596518f8 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/common.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/common.ts @@ -532,3 +532,44 @@ export const executeWithRetry = async ( } } }; + +export const readElementInListWithScroll = async ( + page: Page, + locator: Locator, + hierarchyElementLocator: Locator +) => { + const element = locator; + + // Reset scroll position to top before starting pagination + await hierarchyElementLocator.hover(); + await page.mouse.wheel(0, -99999); + + await page.waitForTimeout(1000); + + // Retry mechanism for pagination + let elementCount = await element.count(); + let retryCount = 0; + const maxRetries = 10; + + while (elementCount === 0 && retryCount < maxRetries) { + await hierarchyElementLocator.hover(); + await page.mouse.wheel(0, 1000); + await page.waitForTimeout(500); + + // Create fresh locator and check if the article is now visible after this retry + const freshArticle = locator; + const count = await freshArticle.count(); + + // Check if the article is now visible after this retry + elementCount = count; + + // If we found the element, validate it and break out of the loop + if (count > 0) { + await expect(freshArticle).toBeVisible(); + + return; // Exit the function early since we found and validated the article + } + + retryCount++; + } +}; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/entity.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/entity.ts index c8d1f19f587..f57f8b7065f 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/entity.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/entity.ts @@ -26,6 +26,7 @@ import { TagClass } from '../support/tag/TagClass'; import { clickOutside, descriptionBox, + readElementInListWithScroll, redirectToHomePage, toastNotification, uuid, @@ -486,8 +487,24 @@ export const assignCertification = async ( certification: TagClass, endpoint: string ) => { + const certificationResponse = page.waitForResponse( + '/api/v1/tags?parent=Certification&limit=50' + ); await page.getByTestId('edit-certification').click(); + await certificationResponse; + await page.waitForSelector('.certification-card-popover', { + state: 'visible', + }); await page.waitForSelector('[data-testid="loader"]', { state: 'detached' }); + + await readElementInListWithScroll( + page, + page.getByTestId( + `radio-btn-${certification.responseData.fullyQualifiedName}` + ), + page.locator('[data-testid="certification-cards"] .ant-radio-group') + ); + await page .getByTestId(`radio-btn-${certification.responseData.fullyQualifiedName}`) .click(); @@ -503,6 +520,9 @@ export const assignCertification = async ( export const removeCertification = async (page: Page, endpoint: string) => { await page.getByTestId('edit-certification').click(); + await page.waitForSelector('.certification-card-popover', { + state: 'visible', + }); await page.waitForSelector('[data-testid="loader"]', { state: 'detached' }); const patchRequest = page.waitForResponse(`/api/v1/${endpoint}/*`); await page.getByTestId('clear-certification').click(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Certification/Certification.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Certification/Certification.component.tsx index 75fcb8e93cc..4694f2d3884 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Certification/Certification.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Certification/Certification.component.tsx @@ -17,6 +17,7 @@ import { useEffect, useMemo, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { ReactComponent as CertificationIcon } from '../../assets/svg/ic-certification.svg'; import { Tag } from '../../generated/entity/classification/tag'; +import { Paging } from '../../generated/type/paging'; import { getTags } from '../../rest/tagAPI'; import { getEntityName } from '../../utils/EntityUtils'; import { stringToHTML } from '../../utils/StringsUtils'; @@ -35,59 +36,84 @@ const Certification = ({ onClose, }: CertificationProps) => { const { t } = useTranslation(); - const popoverRef = useRef(null); + const popoverRef = useRef<{ close: () => void } | null>(null); const [isLoadingCertificationData, setIsLoadingCertificationData] = useState(false); + const [hasContentLoading, setHasContentLoading] = useState(false); const [certifications, setCertifications] = useState>([]); const [selectedCertification, setSelectedCertification] = useState( currentCertificate ?? '' ); - const certificationCardData = useMemo(() => { - return ( - - {certifications.map((certificate) => { - const tagSrc = getTagImageSrc(certificate.style?.iconURL ?? ''); - const title = getEntityName(certificate); - const { id, fullyQualifiedName, description } = certificate; + const [paging, setPaging] = useState({} as Paging); + const [currentPage, setCurrentPage] = useState(1); - return ( -
{ - setSelectedCertification(fullyQualifiedName ?? ''); - }}> - -
- {tagSrc ? ( - {title} - ) : ( -
- -
- )} -
- - {title} - - - {stringToHTML(description)} - -
-
-
- ); - })} - - ); - }, [certifications, selectedCertification]); + const getCertificationData = async (page = 1, append = false) => { + if (page === 1) { + setIsLoadingCertificationData(true); + } else { + setHasContentLoading(true); + } + + try { + const response = await getTags({ + parent: 'Certification', + limit: 50, + after: page > 1 ? paging.after : undefined, + }); + + const { data, paging: newPaging } = response; + + // Sort certifications with Gold, Silver, Bronze first (only for initial load) + const sortedData = + page === 1 + ? [...data].sort((a, b) => { + const order: Record = { + Gold: 0, + Silver: 1, + Bronze: 2, + }; + + const aName = getEntityName(a); + const bName = getEntityName(b); + + const aOrder = order[aName] ?? 3; + const bOrder = order[bName] ?? 3; + + return aOrder - bOrder; + }) + : data; + + if (append) { + setCertifications((prev) => [...prev, ...sortedData]); + } else { + setCertifications(sortedData); + } + + setPaging(newPaging); + setCurrentPage(page); + } catch (err) { + showErrorToast( + err as AxiosError, + t('server.entity-fetch-error', { + entity: t('label.certification-plural-lowercase'), + }) + ); + } finally { + setIsLoadingCertificationData(false); + setHasContentLoading(false); + } + }; + + const handleScroll = async (e: React.UIEvent) => { + const { currentTarget } = e; + const isAtBottom = + currentTarget.scrollTop + currentTarget.offsetHeight >= + currentTarget.scrollHeight - 1; // -1 for precision tolerance + + if (isAtBottom && paging.after && !hasContentLoading) { + await getCertificationData(currentPage + 1, true); + } + }; const updateCertificationData = async (value?: string) => { setIsLoadingCertificationData(true); @@ -98,43 +124,60 @@ const Certification = ({ setIsLoadingCertificationData(false); popoverRef.current?.close(); }; - const getCertificationData = async () => { - setIsLoadingCertificationData(true); - try { - const { data } = await getTags({ - parent: 'Certification', - limit: 50, - }); - // Sort certifications with Gold, Silver, Bronze first - const sortedData = [...data].sort((a, b) => { - const order: Record = { - Gold: 0, - Silver: 1, - Bronze: 2, - }; + const certificationCardData = useMemo(() => { + return ( +
+ + {certifications.map((certificate) => { + const tagSrc = getTagImageSrc(certificate.style?.iconURL ?? ''); + const title = getEntityName(certificate); + const { id, fullyQualifiedName, description } = certificate; - const aName = getEntityName(a); - const bName = getEntityName(b); - - const aOrder = order[aName] ?? 3; - const bOrder = order[bName] ?? 3; - - return aOrder - bOrder; - }); - - setCertifications(sortedData); - } catch (err) { - showErrorToast( - err as AxiosError, - t('server.entity-fetch-error', { - entity: t('label.certification-plural-lowercase'), - }) - ); - } finally { - setIsLoadingCertificationData(false); - } - }; + return ( +
{ + setSelectedCertification(fullyQualifiedName ?? ''); + }}> + +
+ {tagSrc ? ( + {title} + ) : ( +
+ +
+ )} +
+ + {title} + + + {stringToHTML(description)} + +
+
+
+ ); + })} + + {hasContentLoading && ( +
+ +
+ )} +
+ ); + }, [certifications, selectedCertification, hasContentLoading, handleScroll]); const handleCloseCertification = async () => { popoverRef.current?.close(); @@ -143,16 +186,19 @@ const Certification = ({ const onOpenChange = (visible: boolean) => { if (visible) { - getCertificationData(); + getCertificationData(1); setSelectedCertification(currentCertificate); + setCurrentPage(1); + setPaging({} as Paging); } else { setSelectedCertification(''); + setCertifications([]); } }; useEffect(() => { if (popoverProps?.open && certifications.length === 0) { - getCertificationData(); + getCertificationData(1); } }, [popoverProps?.open]); From 063e37a8d8dc977297194512fdc6931d4dc043d6 Mon Sep 17 00:00:00 2001 From: Aniket Katkar Date: Fri, 26 Sep 2025 20:31:30 +0530 Subject: [PATCH 6/8] fix: improve timestamp sorting logic in IngestionRecentRuns component (#23572) --- .../IngestionRecentRun.test.tsx | 181 ++++++++++++++++++ .../IngestionRecentRuns.component.tsx | 11 +- 2 files changed, 190 insertions(+), 2 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionRecentRun/IngestionRecentRun.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionRecentRun/IngestionRecentRun.test.tsx index 05b2093352e..58b059dc522 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionRecentRun/IngestionRecentRun.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionRecentRun/IngestionRecentRun.test.tsx @@ -319,4 +319,185 @@ describe('Test IngestionRecentRun component', () => { expect(getRunHistoryForPipeline).toHaveBeenCalledTimes(2); expect(mockHandlePipelineIdToFetchStatus).toHaveBeenCalled(); }); + + describe('Timestamp Sorting Logic', () => { + it('should sort runs by timestamp in ascending order and show last 5 runs', async () => { + const unorderedRuns = [ + { + runId: 'run-3', + pipelineState: 'success', + startDate: 1667307000, + timestamp: 1667307000, + endDate: 1667307003, + }, + { + runId: 'run-1', + pipelineState: 'failed', + startDate: 1667301000, + timestamp: 1667301000, + endDate: 1667301003, + }, + { + runId: 'run-5', + pipelineState: 'success', + startDate: 1667309000, + timestamp: 1667309000, + endDate: 1667309003, + }, + { + runId: 'run-2', + pipelineState: 'partialSuccess', + startDate: 1667304000, + timestamp: 1667304000, + endDate: 1667304003, + }, + { + runId: 'run-4', + pipelineState: 'running', + startDate: 1667308000, + timestamp: 1667308000, + endDate: 1667308003, + }, + { + runId: 'run-6', + pipelineState: 'queued', + startDate: 1667310000, + timestamp: 1667310000, + endDate: 1667310003, + }, + { + runId: 'run-7', + pipelineState: 'success', + startDate: 1667311000, + timestamp: 1667311000, + endDate: 1667311003, + }, + ]; + + (getRunHistoryForPipeline as jest.Mock).mockResolvedValueOnce({ + data: unorderedRuns, + paging: { total: 7 }, + }); + + await act(async () => { + render(); + }); + + const runs = await screen.findAllByTestId('pipeline-status'); + + expect(runs).toHaveLength(5); + + const latestRun = await screen.findByText(/Success/); + + expect(latestRun).toBeInTheDocument(); + }); + + it('should handle runs with missing timestamps gracefully by treating them as 0', async () => { + const runsWithMissingTimestamps = [ + { + runId: 'run-with-timestamp', + pipelineState: 'success', + startDate: 1667307000, + timestamp: 1667307000, + endDate: 1667307003, + }, + { + runId: 'run-without-timestamp', + pipelineState: 'failed', + startDate: 1667301000, + endDate: 1667301003, + }, + { + runId: 'run-with-null-timestamp', + pipelineState: 'partialSuccess', + startDate: 1667304000, + timestamp: null, + endDate: 1667304003, + }, + ]; + + (getRunHistoryForPipeline as jest.Mock).mockResolvedValueOnce({ + data: runsWithMissingTimestamps, + paging: { total: 3 }, + }); + + await act(async () => { + render(); + }); + + const runs = await screen.findAllByTestId('pipeline-status'); + + expect(runs).toHaveLength(3); + + const latestRun = await screen.findByText(/Success/); + + expect(latestRun).toBeInTheDocument(); + }); + + it('should sort appRuns by timestamp when provided', async () => { + const unorderedAppRuns = [ + { + runId: 'app-run-2', + status: 'Success', + startTime: 1667307000, + timestamp: 1667307000, + endTime: 1667307003, + appId: 'app-2', + }, + { + runId: 'app-run-1', + status: 'Failed', + startTime: 1667301000, + timestamp: 1667301000, + endTime: 1667301003, + appId: 'app-1', + }, + { + runId: 'app-run-3', + status: 'Running', + startTime: 1667309000, + timestamp: 1667309000, + endTime: 1667309003, + appId: 'app-3', + }, + ]; + + await act(async () => { + render( + + ); + }); + + const runs = await screen.findAllByTestId('pipeline-status'); + + expect(runs).toHaveLength(3); + + const latestRun = await screen.findByText(/Running/); + + expect(latestRun).toBeInTheDocument(); + }); + + it('should limit results to maximum 5 runs after sorting', async () => { + const manyRuns = Array.from({ length: 10 }, (_, i) => ({ + runId: `run-${i}`, + pipelineState: 'success', + startDate: 1667300000 + i * 1000, + timestamp: 1667300000 + i * 1000, + endDate: 1667300003 + i * 1000, + })); + + (getRunHistoryForPipeline as jest.Mock).mockResolvedValueOnce({ + data: manyRuns, + paging: { total: 10 }, + }); + + await act(async () => { + render(); + }); + + const runs = await screen.findAllByTestId('pipeline-status'); + + expect(runs).toHaveLength(5); + }); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionRecentRun/IngestionRecentRuns.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionRecentRun/IngestionRecentRuns.component.tsx index 5f06357ebe9..49219d3d856 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionRecentRun/IngestionRecentRuns.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionRecentRun/IngestionRecentRuns.component.tsx @@ -65,7 +65,10 @@ export const IngestionRecentRuns = < queryParams ); - const runs = response.data.splice(0, 5).reverse() ?? []; + const runs = + response.data + .sort((a, b) => (a.timestamp ?? 0) - (b.timestamp ?? 0)) + .slice(-5) ?? []; setRecentRunStatus( (runs.length === 0 && ingestionPipeline?.pipelineStatuses @@ -86,7 +89,11 @@ export const IngestionRecentRuns = < useEffect(() => { if (!isEmpty(appRuns)) { - setRecentRunStatus(appRuns?.splice(0, 5).reverse() ?? []); + setRecentRunStatus( + appRuns + ?.sort((a, b) => (a.timestamp ?? 0) - (b.timestamp ?? 0)) + .slice(-5) ?? [] + ); } }, [appRuns]); From dc474e44f6eb935279e2747765d7a8376ea35c16 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Fri, 26 Sep 2025 21:16:22 +0530 Subject: [PATCH 7/8] supported created_by and created_at info to display in UI of DataContract (#23531) * supported created by and at info of data contract * re-order sla select unit options * added unit test for sla component and fix data visibility --- .../ContractDetailTab/ContractDetail.test.tsx | 45 +- .../ContractDetailTab/ContractDetail.tsx | 51 ++ .../ContractDetailTab/contract-detail.less | 5 + .../ContractSLACard/ContractSLA.component.tsx | 6 +- .../ContractSLACard/ContractSLA.test.tsx | 440 ++++++++++++++++++ .../ContractSLAFormTab/ContractSLAFormTab.tsx | 24 +- .../src/constants/DataContract.constants.ts | 6 + .../ui/src/locale/languages/de-de.json | 2 + .../ui/src/locale/languages/en-us.json | 2 + .../ui/src/locale/languages/es-es.json | 2 + .../ui/src/locale/languages/fr-fr.json | 2 + .../ui/src/locale/languages/gl-es.json | 2 + .../ui/src/locale/languages/he-he.json | 2 + .../ui/src/locale/languages/ja-jp.json | 2 + .../ui/src/locale/languages/ko-kr.json | 2 + .../ui/src/locale/languages/mr-in.json | 2 + .../ui/src/locale/languages/nl-nl.json | 2 + .../ui/src/locale/languages/pr-pr.json | 2 + .../ui/src/locale/languages/pt-br.json | 2 + .../ui/src/locale/languages/pt-pt.json | 2 + .../ui/src/locale/languages/ru-ru.json | 2 + .../ui/src/locale/languages/th-th.json | 2 + .../ui/src/locale/languages/tr-tr.json | 2 + .../ui/src/locale/languages/zh-cn.json | 2 + .../ui/src/locale/languages/zh-tw.json | 2 + .../utils/DataContract/DataContractUtils.ts | 10 +- 26 files changed, 604 insertions(+), 19 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractSLACard/ContractSLA.test.tsx diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.test.tsx index ec43a33e59b..aa4e9ff17c0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.test.tsx @@ -226,7 +226,7 @@ describe('ContractDetail', () => { }); it('should display contract actions', () => { - const { getByTestId, debug } = render( + const { getByTestId } = render( { />, { wrapper: MemoryRouter } ); - debug(); + + expect(getByTestId('manage-contract-actions')).toBeInTheDocument(); + }); + + it('should not display contract created by or created at if data is not present', () => { + const { getByTestId, queryByTestId } = render( + , + { wrapper: MemoryRouter } + ); + + expect( + queryByTestId('contract-created-by-label') + ).not.toBeInTheDocument(); + + expect( + queryByTestId('contract-created-at-label') + ).not.toBeInTheDocument(); + + expect(getByTestId('manage-contract-actions')).toBeInTheDocument(); + }); + + it('should display contract created by or created at if data is present', () => { + const { getByTestId, queryByTestId } = render( + , + { wrapper: MemoryRouter } + ); + + expect(queryByTestId('contract-created-by-label')).toBeInTheDocument(); + + expect(queryByTestId('contract-created-at-label')).toBeInTheDocument(); expect(getByTestId('manage-contract-actions')).toBeInTheDocument(); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.tsx index 8911ff381cd..4f4c715c4ca 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractDetailTab/ContractDetail.tsx @@ -37,6 +37,7 @@ import { ReactComponent as SettingIcon } from '../../../assets/svg/ic-settings-v import { ReactComponent as DeleteIcon } from '../../../assets/svg/ic-trash.svg'; import { + CONTRACT_DATE_TIME_FORMAT, DataContractMode, DATA_CONTRACT_ACTION_DROPDOWN_KEY, } from '../../../constants/DataContract.constants'; @@ -53,6 +54,7 @@ import { downloadContractYamlFile, getConstraintStatus, } from '../../../utils/DataContract/DataContractUtils'; +import { customFormatDateTime } from '../../../utils/date-time/DateTimeUtils'; import { getEntityName } from '../../../utils/EntityUtils'; import { pruneEmptyChildren } from '../../../utils/TableUtils'; import { showErrorToast, showSuccessToast } from '../../../utils/ToastUtils'; @@ -271,6 +273,55 @@ const ContractDetail: React.FC<{ + {contract.createdBy && ( + <> +
+ + {`${t('label.created-by')} : `} + + + +
+ + + + )} + + {contract.createdAt && ( + <> +
+ + {`${t('label.created-at')} : `} + + + + {customFormatDateTime( + contract.createdAt, + CONTRACT_DATE_TIME_FORMAT + )} + +
+ + + + )} +
({ + Transi18next: ({ i18nKey, values }: any) => ( + + {i18nKey} - {values?.label}: {values?.data} + + ), +})); + +jest.mock('../../../assets/svg/ic-check-circle-2.svg', () => ({ + ReactComponent: () => , +})); + +describe('ContractSLA Component', () => { + it('should render null when contract.sla is empty', () => { + const contract: DataContract = { + fullyQualifiedName: 'test.contract', + name: 'Test Contract', + sla: {}, + } as DataContract; + + const { container } = render(); + + expect(container.firstChild).toBeNull(); + }); + + it('should render null when contract.sla is undefined', () => { + const contract: DataContract = { + fullyQualifiedName: 'test.contract', + name: 'Test Contract', + } as DataContract; + + const { container } = render(); + + expect(container.firstChild).toBeNull(); + }); + + it('should render SLA card with refresh frequency', () => { + const contract: DataContract = { + fullyQualifiedName: 'test.contract', + name: 'Test Contract', + id: MOCK_DATA_CONTRACT.id, + entity: MOCK_DATA_CONTRACT.entity, + sla: { + refreshFrequency: { + interval: 24, + unit: RefreshFrequencyUnit.Hour, + }, + }, + } as DataContract; + + render(); + + expect(screen.getByTestId('contract-sla-card')).toBeInTheDocument(); + expect( + screen.getByText('label.service-level-agreement') + ).toBeInTheDocument(); + expect( + screen.getByText( + 'message.freshness-sla-description - label.freshness: 24 hour' + ) + ).toBeInTheDocument(); + expect(screen.getByTestId('check-icon')).toBeInTheDocument(); + }); + + it('should render SLA card with availability time', () => { + const contract: DataContract = { + fullyQualifiedName: 'test.contract', + name: 'Test Contract', + sla: { + availabilityTime: '09:00 UTC', + }, + } as DataContract; + + render(); + + expect(screen.getByTestId('contract-sla-card')).toBeInTheDocument(); + expect( + screen.getByText( + 'message.completeness-sla-description - label.completeness: 09:00 UTC' + ) + ).toBeInTheDocument(); + }); + + it('should render SLA card with max latency', () => { + const contract: DataContract = { + fullyQualifiedName: 'test.contract', + name: 'Test Contract', + id: MOCK_DATA_CONTRACT.id, + entity: MOCK_DATA_CONTRACT.entity, + sla: { + maxLatency: { + value: 5, + unit: MaxLatencyUnit.Minute, + }, + }, + } as DataContract; + + render(); + + expect(screen.getByTestId('contract-sla-card')).toBeInTheDocument(); + expect( + screen.getByText( + 'message.latency-sla-description - label.latency: 5 minute' + ) + ).toBeInTheDocument(); + }); + + it('should render SLA card with retention', () => { + const contract: DataContract = { + fullyQualifiedName: 'test.contract', + name: 'Test Contract', + id: MOCK_DATA_CONTRACT.id, + entity: MOCK_DATA_CONTRACT.entity, + sla: { + retention: { + period: 30, + unit: RetentionUnit.Day, + }, + }, + } as DataContract; + + render(); + + expect(screen.getByTestId('contract-sla-card')).toBeInTheDocument(); + expect( + screen.getByText( + 'message.retention-sla-description - label.retention: 30 day' + ) + ).toBeInTheDocument(); + }); + + it('should render all SLA items when all properties are present', () => { + const contract: DataContract = { + fullyQualifiedName: 'test.contract', + name: 'Test Contract', + id: MOCK_DATA_CONTRACT.id, + entity: MOCK_DATA_CONTRACT.entity, + sla: { + refreshFrequency: { + interval: 24, + unit: RefreshFrequencyUnit.Hour, + }, + availabilityTime: '09:00 UTC', + maxLatency: { + value: 5, + unit: MaxLatencyUnit.Minute, + }, + retention: { + period: 30, + unit: RetentionUnit.Day, + }, + }, + } as DataContract; + + render(); + + expect(screen.getByTestId('contract-sla-card')).toBeInTheDocument(); + + // Check all SLA items are rendered + expect( + screen.getByText( + 'message.freshness-sla-description - label.freshness: 24 hour' + ) + ).toBeInTheDocument(); + expect( + screen.getByText( + 'message.completeness-sla-description - label.completeness: 09:00 UTC' + ) + ).toBeInTheDocument(); + expect( + screen.getByText( + 'message.latency-sla-description - label.latency: 5 minute' + ) + ).toBeInTheDocument(); + expect( + screen.getByText( + 'message.retention-sla-description - label.retention: 30 day' + ) + ).toBeInTheDocument(); + + // Check that all check icons are rendered + const checkIcons = screen.getAllByTestId('check-icon'); + + expect(checkIcons).toHaveLength(4); + }); + + it('should render only provided SLA items', () => { + const contract: DataContract = { + fullyQualifiedName: 'test.contract', + name: 'Test Contract', + id: MOCK_DATA_CONTRACT.id, + entity: MOCK_DATA_CONTRACT.entity, + sla: { + refreshFrequency: { + interval: 12, + unit: RefreshFrequencyUnit.Hour, + }, + retention: { + period: 7, + unit: RetentionUnit.Day, + }, + }, + } as DataContract; + + render(); + + expect(screen.getByTestId('contract-sla-card')).toBeInTheDocument(); + + // Check only provided items are rendered + expect( + screen.getByText( + 'message.freshness-sla-description - label.freshness: 12 hour' + ) + ).toBeInTheDocument(); + expect( + screen.getByText( + 'message.retention-sla-description - label.retention: 7 day' + ) + ).toBeInTheDocument(); + + // Check that availability and latency are not rendered + expect( + screen.queryByText(/message.completeness-sla-description/) + ).not.toBeInTheDocument(); + expect( + screen.queryByText(/message.latency-sla-description/) + ).not.toBeInTheDocument(); + + // Check correct number of check icons + const checkIcons = screen.getAllByTestId('check-icon'); + + expect(checkIcons).toHaveLength(2); + }); + + it('should handle uppercase units and convert them to lowercase', () => { + const contract: DataContract = { + fullyQualifiedName: 'test.contract', + name: 'Test Contract', + id: MOCK_DATA_CONTRACT.id, + entity: MOCK_DATA_CONTRACT.entity, + sla: { + refreshFrequency: { + interval: 1, + unit: RefreshFrequencyUnit.Day, + }, + maxLatency: { + value: 10, + unit: MaxLatencyUnit.Minute, + }, + }, + } as DataContract; + + render(); + + expect( + screen.getByText( + 'message.freshness-sla-description - label.freshness: 1 day' + ) + ).toBeInTheDocument(); + expect( + screen.getByText( + 'message.latency-sla-description - label.latency: 10 minute' + ) + ).toBeInTheDocument(); + }); + + it('should have correct CSS classes applied', () => { + const contract: DataContract = { + fullyQualifiedName: 'test.contract', + name: 'Test Contract', + id: MOCK_DATA_CONTRACT.id, + entity: MOCK_DATA_CONTRACT.entity, + sla: { + refreshFrequency: { + interval: 24, + unit: RefreshFrequencyUnit.Hour, + }, + }, + } as DataContract; + + const { container } = render(); + + expect(container.querySelector('.contract-card-items')).toBeInTheDocument(); + expect( + container.querySelector('.contract-card-header-container') + ).toBeInTheDocument(); + expect( + container.querySelector('.contract-card-header') + ).toBeInTheDocument(); + expect( + container.querySelector('.contract-dash-separator') + ).toBeInTheDocument(); + expect(container.querySelector('.sla-item-container')).toBeInTheDocument(); + expect(container.querySelector('.sla-item')).toBeInTheDocument(); + expect(container.querySelector('.sla-icon')).toBeInTheDocument(); + expect(container.querySelector('.sla-description')).toBeInTheDocument(); + }); + + it('should render SLA items in the correct order', () => { + const contract: DataContract = { + fullyQualifiedName: 'test.contract', + name: 'Test Contract', + id: MOCK_DATA_CONTRACT.id, + entity: MOCK_DATA_CONTRACT.entity, + sla: { + retention: { + period: 30, + unit: RetentionUnit.Day, + }, + maxLatency: { + value: 5, + unit: MaxLatencyUnit.Minute, + }, + availabilityTime: '09:00 UTC', + refreshFrequency: { + interval: 24, + unit: RefreshFrequencyUnit.Hour, + }, + }, + } as DataContract; + + const { container } = render(); + + const slaDescriptions = container.querySelectorAll('.sla-description'); + + // Order should be: refreshFrequency, availabilityTime, maxLatency, retention + expect(slaDescriptions[0]).toHaveTextContent( + 'message.freshness-sla-description' + ); + expect(slaDescriptions[1]).toHaveTextContent( + 'message.completeness-sla-description' + ); + expect(slaDescriptions[2]).toHaveTextContent( + 'message.latency-sla-description' + ); + expect(slaDescriptions[3]).toHaveTextContent( + 'message.retention-sla-description' + ); + }); + + describe('Edge Cases', () => { + it('should handle zero values correctly', () => { + const contract: DataContract = { + fullyQualifiedName: 'test.contract', + name: 'Test Contract', + id: MOCK_DATA_CONTRACT.id, + entity: MOCK_DATA_CONTRACT.entity, + sla: { + refreshFrequency: { + interval: 0, + unit: RefreshFrequencyUnit.Hour, + }, + maxLatency: { + value: 0, + unit: MaxLatencyUnit.Minute, + }, + }, + } as DataContract; + + render(); + + expect( + screen.getByText( + 'message.freshness-sla-description - label.freshness: 0 hour' + ) + ).toBeInTheDocument(); + expect( + screen.getByText( + 'message.latency-sla-description - label.latency: 0 minute' + ) + ).toBeInTheDocument(); + }); + + it('should handle empty string for availabilityTime', () => { + const contract: DataContract = { + fullyQualifiedName: 'test.contract', + name: 'Test Contract', + sla: { + availabilityTime: '', + }, + } as DataContract; + + const { container } = render(); + + // Should not render anything as availabilityTime is empty + expect(container.firstChild).toBeNull(); + }); + + it('should handle partial SLA objects with undefined properties', () => { + const contract: DataContract = { + fullyQualifiedName: 'test.contract', + name: 'Test Contract', + sla: { + refreshFrequency: undefined, + availabilityTime: '09:00 UTC', + maxLatency: undefined, + retention: undefined, + }, + } as DataContract; + + render(); + + // Only availabilityTime should be rendered + expect(screen.getByTestId('contract-sla-card')).toBeInTheDocument(); + expect( + screen.getByText( + 'message.completeness-sla-description - label.completeness: 09:00 UTC' + ) + ).toBeInTheDocument(); + + // Check only one check icon is rendered + const checkIcons = screen.getAllByTestId('check-icon'); + + expect(checkIcons).toHaveLength(1); + }); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractSLAFormTab/ContractSLAFormTab.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractSLAFormTab/ContractSLAFormTab.tsx index f8a7abccc3d..99454b4771f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractSLAFormTab/ContractSLAFormTab.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataContract/ContractSLAFormTab/ContractSLAFormTab.tsx @@ -26,15 +26,17 @@ import moment from 'moment'; import { useEffect, useMemo } from 'react'; import { useTranslation } from 'react-i18next'; import { ReactComponent as LeftOutlined } from '../../../assets/svg/left-arrow.svg'; -import { SLA_AVAILABILITY_TIME_FORMAT } from '../../../constants/DataContract.constants'; +import { + MAX_LATENCY_UNITS, + REFRESH_FREQUENCY_UNITS, + RETENTION_UNITS, + SLA_AVAILABILITY_TIME_FORMAT, +} from '../../../constants/DataContract.constants'; import { ContractSLA, DataContract, - MaxLatencyUnit, - RefreshFrequencyUnit, - RetentionUnit, } from '../../../generated/entity/data/dataContract'; -import { enumToSelectOptions } from '../../../utils/DataContract/DataContractUtils'; +import { generateSelectOptionsFromString } from '../../../utils/DataContract/DataContractUtils'; import './contract-sla-form-tab.less'; export const ContractSLAFormTab: React.FC<{ @@ -52,9 +54,11 @@ export const ContractSLAFormTab: React.FC<{ REFRESH_FREQUENCY_UNIT_OPTIONS, } = useMemo(() => { return { - REFRESH_FREQUENCY_UNIT_OPTIONS: enumToSelectOptions(RefreshFrequencyUnit), - RETENTION_UNIT_OPTIONS: enumToSelectOptions(RetentionUnit), - MAX_LATENCY_OPTIONS: enumToSelectOptions(MaxLatencyUnit), + REFRESH_FREQUENCY_UNIT_OPTIONS: generateSelectOptionsFromString( + REFRESH_FREQUENCY_UNITS + ), + RETENTION_UNIT_OPTIONS: generateSelectOptionsFromString(RETENTION_UNITS), + MAX_LATENCY_OPTIONS: generateSelectOptionsFromString(MAX_LATENCY_UNITS), }; }, []); @@ -166,7 +170,7 @@ export const ContractSLAFormTab: React.FC<{ label={t('label.unit')} name="refresh_frequency_unit">