/* * Copyright 2024 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import { expect, Page } from '@playwright/test'; import { get } from 'lodash'; import { parseCSV } from '../../src/utils/EntityImport/EntityImportUtils'; import { ApiEndpointClass } from '../support/entity/ApiEndpointClass'; import { ContainerClass } from '../support/entity/ContainerClass'; import { DashboardClass } from '../support/entity/DashboardClass'; import { ResponseDataType } from '../support/entity/Entity.interface'; import { EntityClass } from '../support/entity/EntityClass'; import { MetricClass } from '../support/entity/MetricClass'; import { MlModelClass } from '../support/entity/MlModelClass'; import { PipelineClass } from '../support/entity/PipelineClass'; import { SearchIndexClass } from '../support/entity/SearchIndexClass'; import { TableClass } from '../support/entity/TableClass'; import { TopicClass } from '../support/entity/TopicClass'; import { getApiContext, getEntityTypeSearchIndexMapping, toastNotification, } from './common'; type LineageCSVRecord = { fromEntityFQN: string; fromServiceName: string; fromServiceType: string; toEntityFQN: string; toServiceName: string; toServiceType: string; pipelineName: string; }; export const LINEAGE_CSV_HEADERS = [ 'fromEntityFQN', 'fromServiceName', 'fromServiceType', 'fromOwners', 'fromDomain', 'toEntityFQN', 'toServiceName', 'toServiceType', 'toOwners', 'toDomain', 'fromChildEntityFQN', 'toChildEntityFQN', 'pipelineName', 'pipelineType', 'pipelineDescription', 'pipelineOwners', 'pipelineDomain', 'pipelineServiceName', 'pipelineServiceType', ]; export const verifyColumnLayerInactive = async (page: Page) => { await page.click('[data-testid="lineage-layer-btn"]'); // Open Layer popover await page.waitForSelector( '[data-testid="lineage-layer-column-btn"]:not(.active)' ); await page.click('[data-testid="lineage-layer-btn"]'); // Close Layer popover }; export const activateColumnLayer = async (page: Page) => { await page.click('[data-testid="lineage-layer-btn"]'); await page.click('[data-testid="lineage-layer-column-btn"]'); }; export const editLineage = async (page: Page) => { await page.click('[data-testid="edit-lineage"]'); await expect( page.getByTestId('table_search_index-draggable-icon') ).toBeVisible(); }; export const performZoomOut = async (page: Page) => { const zoomOutBtn = page.locator('.react-flow__controls-zoomout'); const enabled = await zoomOutBtn.isEnabled(); if (enabled) { for (const _ of Array.from({ length: 8 })) { await zoomOutBtn.dispatchEvent('click'); } } }; export const deleteEdge = async ( page: Page, fromNode: EntityClass, toNode: EntityClass ) => { const fromNodeFqn = get(fromNode, 'entityResponseData.fullyQualifiedName'); const toNodeFqn = get(toNode, 'entityResponseData.fullyQualifiedName'); await page .locator(`[data-testid="edge-${fromNodeFqn}-${toNodeFqn}"]`) .dispatchEvent('click'); await page.locator('[data-testid="add-pipeline"]').dispatchEvent('click'); await expect(page.locator('[role="dialog"]')).toBeVisible(); await page .locator( '[data-testid="add-edge-modal"] [data-testid="remove-edge-button"]' ) .dispatchEvent('click'); await expect(page.locator('[role="dialog"]')).toBeVisible(); const deleteRes = page.waitForResponse('/api/v1/lineage/**'); await page .locator('[data-testid="delete-edge-confirmation-modal"] .ant-btn-primary') .dispatchEvent('click'); await deleteRes; }; export const dragAndDropNode = async ( page: Page, originSelector: string, destinationSelector: string ) => { const destinationElement = await page.waitForSelector(destinationSelector); await page.hover(originSelector); await page.mouse.down(); const box = (await destinationElement.boundingBox())!; const x = box.x + 250; const y = box.y + box.height / 2; await page.mouse.move(x, y, { steps: 20 }); await page.mouse.up(); }; export const dragConnection = async ( page: Page, sourceId: string, targetId: string, isColumnLineage = false ) => { const selector = !isColumnLineage ? '.lineage-node-handle' : '.lineage-column-node-handle'; const lineageRes = page.waitForResponse('/api/v1/lineage'); await page .locator(`[data-testid="${sourceId}"] ${selector}.react-flow__handle-right`) .dispatchEvent('click'); await page .locator(`[data-testid="${targetId}"] ${selector}.react-flow__handle-left`) .dispatchEvent('click'); await lineageRes; }; export const connectEdgeBetweenNodes = async ( page: Page, fromNode: EntityClass, toNode: EntityClass ) => { const type = getEntityTypeSearchIndexMapping(toNode.type); const fromNodeFqn = get(fromNode, 'entityResponseData.fullyQualifiedName'); const toNodeName = get(toNode, 'entityResponseData.name'); const toNodeFqn = get(toNode, 'entityResponseData.fullyQualifiedName'); const source = `[data-testid="${type}-draggable-icon"]`; const target = '[data-testid="lineage-details"]'; await dragAndDropNode(page, source, target); await page.locator('[data-testid="suggestion-node"]').dispatchEvent('click'); const waitForSearchResponse = page.waitForResponse( `/api/v1/search/query?q=*&from=0&size=10&*` ); await page.locator('[data-testid="suggestion-node"] input').fill(toNodeName); await waitForSearchResponse; await page .locator(`[data-testid="node-suggestion-${toNodeFqn}"]`) .dispatchEvent('click'); await dragConnection( page, `lineage-node-${fromNodeFqn}`, `lineage-node-${toNodeFqn}` ); }; export const performExpand = async ( page: Page, node: EntityClass, upstream: boolean, newNode?: EntityClass ) => { const nodeFqn = get(node, 'entityResponseData.fullyQualifiedName'); const handleDirection = upstream ? 'left' : 'right'; const expandBtn = page .locator(`[data-testid="lineage-node-${nodeFqn}"]`) .locator(`.react-flow__handle-${handleDirection}`) .getByTestId('plus-icon'); if (newNode) { const expandRes = page.waitForResponse('/api/v1/lineage/getLineage?*'); await expandBtn.click(); await expandRes; await verifyNodePresent(page, newNode); } else { await expect(expandBtn).toBeVisible(); } }; export const verifyNodePresent = async (page: Page, node: EntityClass) => { const nodeFqn = get(node, 'entityResponseData.fullyQualifiedName'); const name = get(node, 'entityResponseData.name'); const lineageNode = page.locator(`[data-testid="lineage-node-${nodeFqn}"]`); await expect(lineageNode).toBeVisible(); const entityHeaderName = lineageNode.locator( '[data-testid="entity-header-name"]' ); await expect(entityHeaderName).toHaveText(name); }; export const setupEntitiesForLineage = async ( page: Page, currentEntity: | TableClass | DashboardClass | TopicClass | MlModelClass | ContainerClass | SearchIndexClass | ApiEndpointClass | MetricClass ) => { const entities = [ new TableClass(), new DashboardClass(), new TopicClass(), new MlModelClass(), new ContainerClass(), new SearchIndexClass(), new ApiEndpointClass(), new MetricClass(), ] as const; const { apiContext, afterAction } = await getApiContext(page); for (const entity of entities) { await entity.create(apiContext); } await currentEntity.create(apiContext); const cleanup = async () => { await currentEntity.delete(apiContext); for (const entity of entities) { await entity.delete(apiContext); } await afterAction(); }; return { currentEntity, entities, cleanup }; }; export const editPipelineEdgeDescription = async ( page: Page, fromNode: EntityClass, toNode: EntityClass, pipelineData: ResponseDataType, description: string ) => { const fromNodeFqn = get(fromNode, 'entityResponseData.fullyQualifiedName'); const toNodeFqn = get(toNode, 'entityResponseData.fullyQualifiedName'); await page.click( `[data-testid="pipeline-label-${fromNodeFqn}-${toNodeFqn}"]` ); await page.locator('.edge-info-drawer').isVisible(); await page .locator('.edge-info-drawer [data-testid="Edge"] a') .filter({ hasText: pipelineData.name }); await page.click('.edge-info-drawer [data-testid="edit-description"]'); await page.locator('.ProseMirror').first().click(); await page.locator('.ProseMirror').first().clear(); await page.locator('.ProseMirror').first().fill(description); const descRes = page.waitForResponse('/api/v1/lineage'); await page.getByTestId('save').click(); await descRes; await expect( page.getByTestId('asset-description-container').getByRole('paragraph') ).toContainText(description); }; const verifyPipelineDataInDrawer = async ( page: Page, fromNode: EntityClass, toNode: EntityClass, pipelineItem: PipelineClass, bVisitPipelinePageFromDrawer: boolean ) => { const fromNodeFqn = get(fromNode, 'entityResponseData.fullyQualifiedName'); const toNodeFqn = get(toNode, 'entityResponseData.fullyQualifiedName'); const pipelineName = get(pipelineItem, 'entityResponseData.name'); await page.click( `[data-testid="pipeline-label-${fromNodeFqn}-${toNodeFqn}"]` ); await page.locator('.edge-info-drawer').isVisible(); await page .locator('.edge-info-drawer [data-testid="Edge"] a') .filter({ hasText: pipelineName }); if (bVisitPipelinePageFromDrawer) { await page.locator('.edge-info-drawer [data-testid="Edge"] a').click(); await page.click('[data-testid="lineage"]'); await fromNode.visitEntityPage(page); await page.click('[data-testid="lineage"]'); } else { await page.click('.edge-info-drawer .ant-drawer-header .anticon-close'); } }; export const applyPipelineFromModal = async ( page: Page, fromNode: EntityClass, toNode: EntityClass, pipelineItem?: PipelineClass ) => { const fromNodeFqn = get(fromNode, 'entityResponseData.fullyQualifiedName'); const toNodeFqn = get(toNode, 'entityResponseData.fullyQualifiedName'); const pipelineName = get(pipelineItem, 'entityResponseData.name'); const pipelineFqn = get( pipelineItem, 'entityResponseData.fullyQualifiedName' ); await page .locator(`[data-testid="edge-${fromNodeFqn}-${toNodeFqn}"]`) .click({ force: true }); await page.locator('[data-testid="add-pipeline"]').dispatchEvent('click'); const waitForSearchResponse = page.waitForResponse( `/api/v1/search/query?q=*&from=0&size=10&*` ); await page .locator('[data-testid="add-edge-modal"] [data-testid="field-input"]') .fill(pipelineName); await waitForSearchResponse; await page.click(`[data-testid="pipeline-entry-${pipelineFqn}"]`); const saveRes = page.waitForResponse('/api/v1/lineage'); await page.click('[data-testid="save-button"]'); await saveRes; }; export const deleteNode = async (page: Page, node: EntityClass) => { const nodeFqn = get(node, 'entityResponseData.fullyQualifiedName'); await page .locator(`[data-testid="lineage-node-${nodeFqn}"]`) .dispatchEvent('click'); const lineageRes = page.waitForResponse('/api/v1/lineage/**'); await page .locator('[data-testid="lineage-node-remove-btn"]') .dispatchEvent('click'); await lineageRes; }; export const addColumnLineage = async ( page: Page, fromColumnNode: string, toColumnNode: string, exitEditMode = true ) => { const lineageRes = page.waitForResponse('/api/v1/lineage'); await dragConnection( page, `column-${fromColumnNode}`, `column-${toColumnNode}`, true ); await lineageRes; if (exitEditMode) { await page.click('[data-testid="edit-lineage"]'); } await expect( page.locator( `[data-testid="column-edge-${btoa(fromColumnNode)}-${btoa( toColumnNode )}"]` ) ).toBeVisible(); }; export const removeColumnLineage = async ( page: Page, fromColumnNode: string, toColumnNode: string ) => { await page .locator( `[data-testid="column-edge-${btoa(fromColumnNode)}-${btoa( toColumnNode )}"]` ) .dispatchEvent('click'); await page.locator('[data-testid="delete-button"]').dispatchEvent('click'); const deleteRes = page.waitForResponse('/api/v1/lineage'); await page .locator('[data-testid="delete-edge-confirmation-modal"] .ant-btn-primary') .dispatchEvent('click'); await deleteRes; await page.click('[data-testid="edit-lineage"]'); await expect( page.locator( `[data-testid="column-edge-${btoa(fromColumnNode)}-${btoa( toColumnNode )}"]` ) ).not.toBeVisible(); }; export const addPipelineBetweenNodes = async ( page: Page, sourceEntity: EntityClass, targetEntity: EntityClass, pipelineItem?: PipelineClass, bVerifyPipeline = false ) => { await sourceEntity.visitEntityPage(page); await page.click('[data-testid="lineage"]'); await editLineage(page); await performZoomOut(page); await connectEdgeBetweenNodes(page, sourceEntity, targetEntity); if (pipelineItem) { await applyPipelineFromModal( page, sourceEntity, targetEntity, pipelineItem ); await page.click('[data-testid="edit-lineage"]'); await verifyPipelineDataInDrawer( page, sourceEntity, targetEntity, pipelineItem, bVerifyPipeline ); } }; export const visitLineageTab = async (page: Page) => { const lineageRes = page.waitForResponse('/api/v1/lineage/getLineage?*'); await page.click('[data-testid="lineage"]'); await lineageRes; }; export const fillLineageConfigForm = async ( page: Page, config: { upstreamDepth: number; downstreamDepth: number; layer: string } ) => { await page .getByTestId('field-upstream') .fill(config.upstreamDepth.toString()); await page .getByTestId('field-downstream') .fill(config.downstreamDepth.toString()); await page.getByTestId('field-lineage-layer').click(); await page.locator(`.ant-select-item[title="${config.layer}"]`).click(); const saveRes = page.waitForResponse('/api/v1/system/settings'); await page.getByTestId('save-button').click(); await saveRes; await toastNotification(page, /Lineage Config updated successfully/); }; export const verifyColumnLayerActive = async (page: Page) => { await page.click('[data-testid="lineage-layer-btn"]'); // Open Layer popover await page.waitForSelector('[data-testid="lineage-layer-column-btn"].active'); await page.click('[data-testid="lineage-layer-btn"]'); // Close Layer popover }; export const verifyCSVHeaders = async (headers: string[]) => { LINEAGE_CSV_HEADERS.forEach((expectedHeader) => { expect(headers).toContain(expectedHeader); }); }; export const getLineageCSVData = async (page: Page) => { await page.waitForSelector('[data-testid="lineage-export"]', { state: 'visible', }); await expect(page.getByTestId('lineage-export')).toBeEnabled(); await page.getByTestId('lineage-export').click(); await page.waitForSelector( '[data-testid="export-entity-modal"] #submit-button', { state: 'visible', } ); const [download] = await Promise.all([ page.waitForEvent('download'), page.click( '[data-testid="export-entity-modal"] button#submit-button:visible' ), ]); const filePath = await download.path(); expect(filePath).not.toBeNull(); const fileContent = await download.createReadStream(); let fileData = ''; for await (const item of fileContent) { fileData += item.toString(); } const csvRows = fileData .split('\n') .map((row) => row.split(',').map((cell) => cell.replace(/"/g, '').trim())); const headers = csvRows[0]; await verifyCSVHeaders(headers); return parseCSV(csvRows); }; export const verifyExportLineageCSV = async ( page: Page, currentEntity: EntityClass, entities: readonly [ TableClass, DashboardClass, TopicClass, MlModelClass, ContainerClass, SearchIndexClass, ApiEndpointClass, MetricClass ], pipeline: PipelineClass ) => { const parsedData = await getLineageCSVData(page); const currentEntityFQN = get( currentEntity, 'entityResponseData.fullyQualifiedName' ); const arr = []; for (let i = 0; i < entities.length; i++) { arr.push({ fromEntityFQN: currentEntityFQN, fromServiceName: get( currentEntity, 'entityResponseData.service.name', '' ), fromServiceType: get(currentEntity, 'entityResponseData.serviceType', ''), toEntityFQN: get( entities[i], 'entityResponseData.fullyQualifiedName', '' ), toServiceName: get(entities[i], 'entityResponseData.service.name', ''), toServiceType: get(entities[i], 'entityResponseData.serviceType', ''), pipelineName: get(pipeline, 'entityResponseData.name', ''), }); } arr.forEach((expectedRow: LineageCSVRecord) => { const matchingRow = parsedData.find((row) => Object.keys(expectedRow).every( (key) => row[key] === expectedRow[key as keyof LineageCSVRecord] ) ); expect(matchingRow).toBeDefined(); // Ensure a matching row exists }); }; export const verifyColumnLineageInCSV = async ( page: Page, sourceEntity: EntityClass, targetEntity: EntityClass, sourceColFqn: string, targetColFqn: string ) => { const parsedData = await getLineageCSVData(page); const expectedRow = { fromEntityFQN: get(sourceEntity, 'entityResponseData.fullyQualifiedName'), fromServiceName: get(sourceEntity, 'entityResponseData.service.name', ''), fromServiceType: get(sourceEntity, 'entityResponseData.serviceType', ''), toEntityFQN: get(targetEntity, 'entityResponseData.fullyQualifiedName', ''), toServiceName: get(targetEntity, 'entityResponseData.service.name', ''), toServiceType: get(targetEntity, 'entityResponseData.serviceType', ''), fromChildEntityFQN: sourceColFqn, toChildEntityFQN: targetColFqn, pipelineName: '', }; const matchingRow = parsedData.find((row) => Object.keys(expectedRow).every( (key) => row[key] === expectedRow[key as keyof LineageCSVRecord] ) ); expect(matchingRow).toBeDefined(); // Ensure a matching row exists };