yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-26 09:55:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

#18677 - Added Lineage Field back to SearchLineage (#18687)

Browse Source 
* fix: add lineage field back to searchLineage

* add lineage tests

---------

Co-authored-by: karanh37 <karanh37@gmail.com>
This commit is contained in:
Teddy 2024-11-20 11:13:09 +01:00 committed by GitHub
parent a5bddc3cb8
commit 429e48aa3a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 168 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View File

@ -783,7 +783,6 @@ public class ElasticSearchClient implements SearchClient {
Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
List<String> sourceFieldsToExcludeCopy = new ArrayList<>(SOURCE_FIELDS_TO_EXCLUDE);
sourceFieldsToExcludeCopy.add("lineage");
searchSourceBuilder.fetchSource(null, sourceFieldsToExcludeCopy.toArray(String[]::new));
searchSourceBuilder.query(
QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fullyQualifiedName", fqn)));
@ -1059,7 +1058,6 @@ public class ElasticSearchClient implements SearchClient {
List<Map<String, Object>> lineage =
(List<Map<String, Object>>) hit.getSourceAsMap().get("lineage");
HashMap<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap()));
tempMap.remove("lineage");
nodes.add(tempMap);
for (Map<String, Object> lin : lineage) {
Map<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity");
@ -1249,7 +1247,6 @@ public class ElasticSearchClient implements SearchClient {
List<Map<String, Object>> lineage =
(List<Map<String, Object>>) hit.getSourceAsMap().get("lineage");
HashMap<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap()));
tempMap.remove("lineage");
nodes.add(tempMap);
for (Map<String, Object> lin : lineage) {
HashMap<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity");

3
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
View File

@ -778,7 +778,6 @@ public class OpenSearchClient implements SearchClient {
Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
List<String> sourceFieldsToExcludeCopy = new ArrayList<>(SOURCE_FIELDS_TO_EXCLUDE);
sourceFieldsToExcludeCopy.add("lineage");
searchSourceBuilder.fetchSource(null, sourceFieldsToExcludeCopy.toArray(String[]::new));
searchSourceBuilder.query(
QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fullyQualifiedName", fqn)));
@ -1055,7 +1054,6 @@ public class OpenSearchClient implements SearchClient {
List<Map<String, Object>> lineage =
(List<Map<String, Object>>) hit.getSourceAsMap().get("lineage");
HashMap<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap()));
tempMap.remove("lineage");
nodes.add(tempMap);
for (Map<String, Object> lin : lineage) {
HashMap<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity");
@ -1229,7 +1227,6 @@ public class OpenSearchClient implements SearchClient {
List<Map<String, Object>> lineage =
(List<Map<String, Object>>) hit.getSourceAsMap().get("lineage");
HashMap<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap()));
tempMap.remove("lineage");
nodes.add(tempMap);
for (Map<String, Object> lin : lineage) {
HashMap<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity");

80
openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java
View File

@ -31,11 +31,14 @@ import java.net.URLEncoder;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response.Status;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.BeforeAll;
@ -569,6 +572,42 @@ public class LineageResourceTest extends OpenMetadataApplicationTest {
deleteEdge(TABLES.get(6), TABLES.get(7));
}
@Order(7)
@Test
void get_SearchLineage(TestInfo testInfo) throws HttpResponseException {
// our lineage is
// 0
// +-----+-----+
// v v
// 2 4
// +---+---+ v
// v | 5
// 1 | v
// | 6
// | v
// +-----> 7
addEdge(TABLES.get(4), TABLES.get(5));
addEdge(TABLES.get(5), TABLES.get(6));
addEdge(TABLES.get(0), TABLES.get(4));
addEdge(TABLES.get(0), TABLES.get(2));
addEdge(TABLES.get(2), TABLES.get(1));
addEdge(TABLES.get(2), TABLES.get(7));
addEdge(TABLES.get(6), TABLES.get(7));
Map<String, List<Map<String, Object>>> entity =
searchLineage(TABLES.get(5).getEntityReference(), 1, 1);
assertSearchLineageResponseFields(entity);
deleteEdge(TABLES.get(4), TABLES.get(5));
deleteEdge(TABLES.get(5), TABLES.get(6));
deleteEdge(TABLES.get(0), TABLES.get(4));
deleteEdge(TABLES.get(0), TABLES.get(2));
deleteEdge(TABLES.get(2), TABLES.get(1));
deleteEdge(TABLES.get(2), TABLES.get(7));
deleteEdge(TABLES.get(6), TABLES.get(7));
}
public Edge getEdge(Table from, Table to) {
return getEdge(from.getId(), to.getId(), null);
}
@ -738,6 +777,32 @@ public class LineageResourceTest extends OpenMetadataApplicationTest {
assertEquals(lineageById, lineageByName);
}
private void assertSearchLineageResponseFields(Map<String, List<Map<String, Object>>> entity) {
List<Map<String, Object>> entities = entity.get("nodes");
Set<String> nodesFields = Set.of("id", "name", "displayName", "fullyQualifiedName", "lineage");
Set<String> nodesColumnsFields = Set.of("name", "fullyQualifiedName");
entities.forEach(
e -> {
Set<String> keys = e.keySet();
Set<String> missingKeys = new HashSet<>(nodesFields);
missingKeys.removeAll(keys);
String err = String.format("Nodes keys not found in the response: %s", missingKeys);
assertTrue(keys.containsAll(nodesFields), err);
List<Map<String, Object>> columns = (List<Map<String, Object>>) e.get("columns");
columns.forEach(
c -> {
Set<String> columnsKeys = c.keySet();
Set<String> missingColumnKeys = new HashSet<>(nodesColumnsFields);
missingColumnKeys.removeAll(columnsKeys);
String columnErr =
String.format(
"Column nodes keys not found in the response: %s", missingColumnKeys);
assertTrue(columnsKeys.containsAll(nodesColumnsFields), columnErr);
});
});
}
public EntityLineage getLineage(
String entity,
UUID id,
@ -754,6 +819,21 @@ public class LineageResourceTest extends OpenMetadataApplicationTest {
return lineage;
}
public Map<String, List<Map<String, Object>>> searchLineage(
@NonNull EntityReference entityReference,
@NonNull int upstreamDepth,
@NonNull int downstreamDepth)
throws HttpResponseException {
WebTarget target = getResource("lineage/getLineage");
target = target.queryParam("fqn", entityReference.getFullyQualifiedName());
target = target.queryParam("type", entityReference.getType());
target = target.queryParam("upstreamDepth", upstreamDepth);
target = target.queryParam("downstreamDepth", downstreamDepth);
Map<String, List<Map<String, Object>>> entity =
TestUtils.get(target, Map.class, ADMIN_AUTH_HEADERS);
return entity;
}
public EntityLineage getLineageByName(
String entity,
String fqn,

86
openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Lineage.spec.ts
View File

@ -37,6 +37,7 @@ import {
deleteNode,
editLineage,
fillLineageConfigForm,
performExpand,
performZoomOut,
removeColumnLineage,
setupEntitiesForLineage,
@ -374,57 +375,88 @@ test('Verify global lineage config', async ({ browser }) => {
const topic = new TopicClass();
const dashboard = new DashboardClass();
const mlModel = new MlModelClass();
const searchIndex = new SearchIndexClass();
try {
await table.create(apiContext);
await topic.create(apiContext);
await dashboard.create(apiContext);
await mlModel.create(apiContext);
await searchIndex.create(apiContext);
await addPipelineBetweenNodes(page, table, topic);
await addPipelineBetweenNodes(page, topic, dashboard);
await addPipelineBetweenNodes(page, dashboard, mlModel);
await addPipelineBetweenNodes(page, mlModel, searchIndex);
await settingClick(page, GlobalSettingOptions.LINEAGE_CONFIG);
await fillLineageConfigForm(page, {
upstreamDepth: 1,
downstreamDepth: 1,
layer: 'Column Level Lineage',
});
await test.step(
'Update global lineage config and verify lineage',
async () => {
await settingClick(page, GlobalSettingOptions.LINEAGE_CONFIG);
await fillLineageConfigForm(page, {
upstreamDepth: 1,
downstreamDepth: 1,
layer: 'Column Level Lineage',
});
await topic.visitEntityPage(page);
await visitLineageTab(page);
await topic.visitEntityPage(page);
await visitLineageTab(page);
await verifyNodePresent(page, table);
await verifyNodePresent(page, dashboard);
const mlModelFqn = get(
mlModel,
'entityResponseData.fullyQualifiedName'
);
const mlModelNode = page.locator(
`[data-testid="lineage-node-${mlModelFqn}"]`
);
await verifyNodePresent(page, table);
await verifyNodePresent(page, dashboard);
await expect(mlModelNode).not.toBeVisible();
const mlModelFqn = get(mlModel, 'entityResponseData.fullyQualifiedName');
const mlModelNode = page.locator(
`[data-testid="lineage-node-${mlModelFqn}"]`
await verifyColumnLayerActive(page);
}
);
await expect(mlModelNode).not.toBeVisible();
await test.step(
'Verify Upstream and Downstream expand collapse buttons',
async () => {
await dashboard.visitEntityPage(page);
await visitLineageTab(page);
await page.getByTestId('entity-panel-close-icon').click();
await performZoomOut(page);
await verifyNodePresent(page, topic);
await verifyNodePresent(page, mlModel);
await performExpand(page, mlModel, false, searchIndex);
await performExpand(page, topic, true);
}
);
await verifyColumnLayerActive(page);
await test.step(
'Reset global lineage config and verify lineage',
async () => {
await settingClick(page, GlobalSettingOptions.LINEAGE_CONFIG);
await fillLineageConfigForm(page, {
upstreamDepth: 2,
downstreamDepth: 2,
layer: 'Entity Lineage',
});
await settingClick(page, GlobalSettingOptions.LINEAGE_CONFIG);
await fillLineageConfigForm(page, {
upstreamDepth: 2,
downstreamDepth: 2,
layer: 'Entity Lineage',
});
await dashboard.visitEntityPage(page);
await visitLineageTab(page);
await topic.visitEntityPage(page);
await visitLineageTab(page);
await verifyNodePresent(page, table);
await verifyNodePresent(page, dashboard);
await verifyNodePresent(page, mlModel);
await verifyNodePresent(page, table);
await verifyNodePresent(page, dashboard);
await verifyNodePresent(page, mlModel);
await verifyNodePresent(page, searchIndex);
await verifyNodePresent(page, topic);
}
);
} finally {
await table.delete(apiContext);
await topic.delete(apiContext);
await dashboard.delete(apiContext);
await mlModel.delete(apiContext);
await searchIndex.delete(apiContext);
await afterAction();
}

23
openmetadata-ui/src/main/resources/ui/playwright/utils/lineage.ts
View File

@ -161,6 +161,29 @@ export const connectEdgeBetweenNodes = async (
);
};
export const performExpand = async (
page: Page,
node: EntityClass,
upstream: boolean,
newNode?: EntityClass
) => {
const nodeFqn = get(node, 'entityResponseData.fullyQualifiedName');
const handleDirection = upstream ? 'left' : 'right';
const expandBtn = page
.locator(`[data-testid="lineage-node-${nodeFqn}"]`)
.locator(`.react-flow__handle-${handleDirection}`)
.getByTestId('plus-icon');
if (newNode) {
const expandRes = page.waitForResponse('/api/v1/lineage/getLineage?*');
await expandBtn.click();
await expandRes;
await verifyNodePresent(page, newNode);
} else {
await expect(expandBtn).toBeVisible();
}
};
export const verifyNodePresent = async (page: Page, node: EntityClass) => {
const nodeFqn = get(node, 'entityResponseData.fullyQualifiedName');
const name = get(node, 'entityResponseData.name');

8
openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx
View File

@ -78,7 +78,9 @@ export const getExpandHandle = (
? 'react-flow__handle-right'
: 'react-flow__handle-left'
)}
icon={<PlusIcon className="lineage-expand-icon" />}
icon={
<PlusIcon className="lineage-expand-icon" data-testid="plus-icon" />
}
shape="circle"
size="small"
onClick={(e) => {
@ -106,7 +108,9 @@ export const getCollapseHandle = (
? 'downstream-collapse-handle'
: 'upstream-collapse-handle'
}
icon={<MinusIcon className="lineage-expand-icon" />}
icon={
<MinusIcon className="lineage-expand-icon " data-testid="minus-icon" />
}
shape="circle"
size="small"
onClick={(e) => {