yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2026-01-06 12:36:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Create Change Events on import and update in es (#18953)

Browse Source 
* Create Change Events on import and update in es

* add glossary tests

* minor test cleanup

* Make Change Event Update Async

* Fix Test case

---------

Co-authored-by: karanh37 <karanh37@gmail.com>
Co-authored-by: Sriharsha Chintalapani <harshach@users.noreply.github.com>
This commit is contained in:
Mohit Yadav 2024-12-09 21:15:33 +05:30 committed by GitHub
parent de0889d590
commit 5263858067
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 293 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java
View File

@ -23,6 +23,7 @@ import static org.openmetadata.csv.CsvUtil.fieldToEntities;
import static org.openmetadata.csv.CsvUtil.fieldToExtensionStrings;
import static org.openmetadata.csv.CsvUtil.fieldToInternalArray;
import static org.openmetadata.csv.CsvUtil.recordToString;
import static org.openmetadata.service.events.ChangeEventHandler.copyChangeEvent;
import com.fasterxml.jackson.databind.JsonNode;
import com.networknt.schema.JsonSchema;
@ -60,7 +61,9 @@ import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.type.ApiStatus;
import org.openmetadata.schema.type.ChangeEvent;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.EventType;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.schema.type.TagLabel.TagSource;
@ -72,7 +75,9 @@ import org.openmetadata.schema.type.csv.CsvImportResult;
import org.openmetadata.schema.type.customProperties.TableConfig;
import org.openmetadata.service.Entity;
import org.openmetadata.service.TypeRegistry;
import org.openmetadata.service.formatter.util.FormatterUtil;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.util.AsyncService;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.RestUtil.PutResponse;
@ -721,6 +726,9 @@ public abstract class EntityCsv<T extends EntityInterface> {
repository.prepareInternal(entity, false);
PutResponse<T> response = repository.createOrUpdate(null, entity);
responseStatus = response.getStatus();
AsyncService.getInstance()
.getExecutorService()
.submit(() -> createChangeEventAndUpdateInES(response, importedBy));
} catch (Exception ex) {
importFailure(resultsPrinter, ex.getMessage(), csvRecord);
importResult.setStatus(ApiStatus.FAILURE);
@ -744,6 +752,20 @@ public abstract class EntityCsv<T extends EntityInterface> {
}
}
private void createChangeEventAndUpdateInES(PutResponse<T> response, String importedBy) {
if (!response.getChangeType().equals(EventType.ENTITY_NO_CHANGE)) {
ChangeEvent changeEvent =
FormatterUtil.createChangeEventForEntity(
importedBy, response.getChangeType(), response.getEntity());
Object entity = changeEvent.getEntity();
changeEvent = copyChangeEvent(changeEvent);
changeEvent.setEntity(JsonUtils.pojoToMaskedJson(entity));
// Change Event and Update in Es
Entity.getCollectionDAO().changeEventDAO().insert(JsonUtils.pojoToJson(changeEvent));
Entity.getSearchRepository().updateEntity(response.getEntity().getEntityReference());
}
}
@Transaction
protected void createUserEntity(CSVPrinter resultsPrinter, CSVRecord csvRecord, T entity)
throws IOException {

2
openmetadata-service/src/main/java/org/openmetadata/service/events/ChangeEventHandler.java
View File

@ -86,7 +86,7 @@ public class ChangeEventHandler implements EventHandler {
return null;
}
private static ChangeEvent copyChangeEvent(ChangeEvent changeEvent) {
public static ChangeEvent copyChangeEvent(ChangeEvent changeEvent) {
return new ChangeEvent()
.withId(changeEvent.getId())
.withEventType(changeEvent.getEventType())

2
openmetadata-service/src/main/java/org/openmetadata/service/formatter/util/FormatterUtil.java
View File

@ -253,7 +253,7 @@ public class FormatterUtil {
return null;
}
private static ChangeEvent createChangeEventForEntity(
public static ChangeEvent createChangeEventForEntity(
String updateBy, EventType eventType, EntityInterface entityInterface) {
return getChangeEvent(
updateBy, eventType, entityInterface.getEntityReference().getType(), entityInterface)

2
openmetadata-service/src/main/java/org/openmetadata/service/util/RestUtil.java
View File

@ -122,7 +122,7 @@ public final class RestUtil {
@Getter private T entity;
private ChangeEvent changeEvent;
@Getter private final Response.Status status;
private final EventType changeType;
@Getter private final EventType changeType;
/**
* Response.Status.CREATED when PUT operation creates a new entity or Response.Status.OK when PUT operation updates

21
openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java
View File

@ -934,8 +934,25 @@ public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlo
List<String> newRecords =
listOf(
",g3,dsp0,dsc0,h1;h2;h3,,term0;http://term0,PII.Sensitive,,,Approved,\"\"\"glossaryTermTableCol1Cp:row_1_col1_Value,,\"\";\"\"glossaryTermTableCol3Cp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value|row_2_col1_Value,row_2_col2_Value,row_2_col3_Value\"\"\"");
testImportExport(
glossary.getName(), GlossaryCsv.HEADERS, createRecords, updateRecords, newRecords);
Awaitility.await()
.atMost(Duration.ofMillis(120 * 1000L))
.pollInterval(Duration.ofMillis(2000L))
.ignoreExceptions()
.until(
() -> {
try {
testImportExport(
glossary.getName(),
GlossaryCsv.HEADERS,
createRecords,
updateRecords,
newRecords);
return true;
} catch (Exception e) {
// Return false to retry
return false;
}
});
}
@Test

167
openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Glossary.spec.ts
View File

@ -50,11 +50,11 @@ import {
createGlossaryTerms,
createTagTaskForGlossary,
deleteGlossaryOrGlossaryTerm,
descriptionBox,
deselectColumns,
dragAndDropColumn,
dragAndDropTerm,
filterStatus,
getEscapedTermFqn,
goToAssetsTab,
openColumnDropdown,
renameGlossaryTerm,
@ -63,6 +63,8 @@ import {
selectColumns,
toggleAllColumnsSelection,
updateGlossaryTermDataFromTree,
updateGlossaryTermOwners,
updateGlossaryTermReviewers,
validateGlossaryTerm,
verifyAllColumns,
verifyColumnsVisibility,
@ -126,7 +128,7 @@ test.describe('Glossary tests', () => {
await verifyTaskCreated(
page1,
glossary1.data.fullyQualifiedName,
glossary1.data.terms[0].data
glossary1.data.terms[0].data.name
);
await approveGlossaryTermTask(page1, glossary1.data.terms[0].data);
@ -183,7 +185,7 @@ test.describe('Glossary tests', () => {
await verifyTaskCreated(
page1,
glossary2.data.fullyQualifiedName,
glossary2.data.terms[0].data
glossary2.data.terms[0].data.name
);
await approveGlossaryTermTask(page1, glossary2.data.terms[0].data);
@ -304,126 +306,63 @@ test.describe('Glossary tests', () => {
const { page, afterAction, apiContext } = await performAdminLogin(browser);
const glossary1 = new Glossary();
const glossaryTerm1 = new GlossaryTerm(glossary1);
await glossary1.create(apiContext);
await glossaryTerm1.create(apiContext);
const owner1 = new UserClass();
const reviewer1 = new UserClass();
const testtag = `Test tag ${uuid()}`;
try {
await glossary1.create(apiContext);
await glossaryTerm1.create(apiContext);
await owner1.create(apiContext);
await reviewer1.create(apiContext);
await await redirectToHomePage(page);
await sidebarClick(page, SidebarItem.GLOSSARY);
await selectActiveGlossary(page, glossary1.data.displayName);
await redirectToHomePage(page);
await sidebarClick(page, SidebarItem.GLOSSARY);
await updateGlossaryTermOwners(page, glossaryTerm1.data, [
{
name: `${owner1.data.firstName}${owner1.data.lastName}`,
type: 'user',
},
]);
await page.click('[data-testid="add-classification"]');
await updateGlossaryTermReviewers(page, glossaryTerm1.data, [
{
name: `${reviewer1.data.firstName}${reviewer1.data.lastName}`,
type: 'user',
},
]);
await expect(page.getByRole('dialog')).toBeVisible();
await openColumnDropdown(page);
const checkboxLabels = ['Reviewer'];
await selectColumns(page, checkboxLabels);
await clickSaveButton(page);
await verifyColumnsVisibility(page, checkboxLabels, true);
await page.getByTestId('name').fill(testtag);
await page.locator(descriptionBox).fill('Test Description');
const escapedFqn = getEscapedTermFqn(glossaryTerm1.data);
const termRow = page.locator(`[data-row-key="${escapedFqn}"]`);
const glossaryTermResponse = page.waitForResponse('/api/v1/glossaryTerms');
await page.click('[data-testid="save-glossary-term"]');
await glossaryTermResponse;
// Verify the Reviewer
const reviewerSelector = `td:nth-child(3) a[data-testid="owner-link"]`;
const reviewerText = await termRow
.locator(reviewerSelector)
.textContent();
await page.click('[data-testid="expand-icon"]');
expect(reviewerText).toBe(
`${reviewer1.data.firstName}${reviewer1.data.lastName}`
);
await expect(page.getByText(testtag)).toBeVisible();
// Verify the Owner
const ownerSelector = `td:nth-child(4) a[data-testid="owner-link"]`;
const ownerText = await termRow.locator(ownerSelector).textContent();
await page.getByTestId('edit-button').last().click();
await expect(page.getByRole('dialog')).toBeVisible();
await page
.getByTestId('edit-glossary-modal')
.getByTestId('add-owner')
.click();
await expect(page.getByTestId('select-owner-tabs')).toBeVisible();
await page.waitForSelector('[data-testid="loader"]', { state: 'detached' });
await expect(page.getByRole('listitem', { name: 'admin' })).toHaveClass(
/active/
);
const searchUser = page.waitForResponse(
`/api/v1/search/query?q=*${encodeURIComponent(
user1.responseData.displayName
)}*`
);
await page
.getByTestId(`owner-select-users-search-bar`)
.fill(user1.responseData.displayName);
await searchUser;
await page
.getByRole('listitem', {
name: user1.responseData.displayName,
exact: true,
})
.click();
await page.getByTestId('selectable-list-update-btn').click();
await expect(
page.getByRole('link', { name: user1.responseData.displayName })
).toBeVisible();
await page
.getByTestId('edit-glossary-modal')
.getByTestId('add-reviewers')
.click();
await expect(
page
.getByRole('tooltip', { name: 'Selected Reviewers Teams' })
.getByTestId('select-owner-tabs')
).toBeVisible();
const userListResponse = page.waitForResponse(
'/api/v1/users?limit=*&isBot=false*'
);
await page.getByRole('tab', { name: 'Users' }).click();
await userListResponse;
await page.waitForSelector('[data-testid="loader"]', { state: 'detached' });
const searchUserReviewer = page.waitForResponse(
`/api/v1/search/query?q=*${encodeURIComponent(
user2.responseData.displayName
)}*`
);
await page
.getByTestId(`owner-select-users-search-bar`)
.fill(user2.responseData.displayName);
await searchUserReviewer;
await page
.getByRole('listitem', {
name: user2.responseData.displayName,
exact: true,
})
.click();
await page.getByTestId('selectable-list-update-btn').click();
await page.click('[data-testid="save-glossary-term"]');
await glossaryTermResponse;
await expect(
page.getByRole('link', { name: user1.responseData.displayName })
).toBeVisible();
await openColumnDropdown(page);
const checkboxLabels = ['Reviewer'];
await selectColumns(page, checkboxLabels);
await clickSaveButton(page);
await verifyColumnsVisibility(page, checkboxLabels, true);
await expect(
page.getByRole('link', { name: user2.responseData.displayName })
).toBeVisible();
await glossaryTerm1.delete(apiContext);
await glossary1.delete(apiContext);
await afterAction();
expect(ownerText).toBe(`${owner1.data.firstName}${owner1.data.lastName}`);
} finally {
await glossaryTerm1.delete(apiContext);
await glossary1.delete(apiContext);
await owner1.delete(apiContext);
await reviewer1.delete(apiContext);
await afterAction();
}
});
test('Add and Remove Assets', async ({ browser }) => {

40
openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts
View File

@ -18,6 +18,7 @@ import {
} from '../../constant/glossaryImportExport';
import { GlobalSettingOptions } from '../../constant/settings';
import { SidebarItem } from '../../constant/sidebar';
import { EntityTypeEndpoint } from '../../support/entity/Entity.interface';
import { Glossary } from '../../support/glossary/Glossary';
import { GlossaryTerm } from '../../support/glossary/GlossaryTerm';
import { UserClass } from '../../support/user/UserClass';
@ -32,7 +33,12 @@ import {
addCustomPropertiesForEntity,
deleteCreatedProperty,
} from '../../utils/customProperty';
import { selectActiveGlossary } from '../../utils/glossary';
import { addMultiOwner } from '../../utils/entity';
import {
selectActiveGlossary,
selectActiveGlossaryTerm,
verifyTaskCreated,
} from '../../utils/glossary';
import {
createGlossaryTermRowDetails,
fillGlossaryRowDetails,
@ -47,6 +53,7 @@ test.use({
const user1 = new UserClass();
const user2 = new UserClass();
const user3 = new UserClass();
const glossary1 = new Glossary();
const glossary2 = new Glossary();
const glossaryTerm1 = new GlossaryTerm(glossary1);
@ -55,6 +62,8 @@ const propertiesList = Object.values(CUSTOM_PROPERTIES_TYPES);
const propertyListName: Record<string, string> = {};
const additionalGlossaryTerm = createGlossaryTermRowDetails();
test.describe('Glossary Bulk Import Export', () => {
test.slow(true);
@ -63,6 +72,7 @@ test.describe('Glossary Bulk Import Export', () => {
await user1.create(apiContext);
await user2.create(apiContext);
await user3.create(apiContext);
await glossary1.create(apiContext);
await glossary2.create(apiContext);
await glossaryTerm1.create(apiContext);
@ -76,6 +86,7 @@ test.describe('Glossary Bulk Import Export', () => {
await user1.delete(apiContext);
await user2.delete(apiContext);
await user3.delete(apiContext);
await glossary1.delete(apiContext);
await glossary2.delete(apiContext);
@ -129,6 +140,16 @@ test.describe('Glossary Bulk Import Export', () => {
await sidebarClick(page, SidebarItem.GLOSSARY);
await selectActiveGlossary(page, glossary1.data.displayName);
// Update Reviewer
await addMultiOwner({
page,
ownerNames: [user3.getUserName()],
activatorBtnDataTestId: 'Add',
resultTestId: 'glossary-reviewer-name',
endpoint: EntityTypeEndpoint.Glossary,
type: 'Users',
});
// Safety check to close potential glossary not found alert
// Arrived due to parallel testing
await closeFirstPopupAlert(page);
@ -162,7 +183,7 @@ test.describe('Glossary Bulk Import Export', () => {
// Click on first cell and edit
await fillGlossaryRowDetails(
{
...createGlossaryTermRowDetails(),
...additionalGlossaryTerm,
owners: [user1.responseData?.['displayName']],
reviewers: [user2.responseData?.['displayName']],
relatedTerm: {
@ -209,6 +230,21 @@ test.describe('Glossary Bulk Import Export', () => {
}
);
await test.step('should have term in review state', async () => {
await sidebarClick(page, SidebarItem.GLOSSARY);
await selectActiveGlossary(page, glossary1.data.displayName);
await verifyTaskCreated(
page,
glossary1.data.fullyQualifiedName,
glossaryTerm1.data.name
);
await selectActiveGlossaryTerm(page, glossaryTerm1.data.displayName);
const statusBadge = page.locator('.status-badge');
await expect(statusBadge).toHaveText('In Review');
});
await test.step('delete custom properties', async () => {
for (const propertyName of Object.values(propertyListName)) {
await settingClick(page, GlobalSettingOptions.GLOSSARY_TERM, true);

160
openmetadata-ui/src/main/resources/ui/playwright/utils/glossary.ts
View File

@ -22,6 +22,7 @@ import { Glossary } from '../support/glossary/Glossary';
import {
GlossaryData,
GlossaryTermData,
UserTeamRef,
} from '../support/glossary/Glossary.interface';
import { GlossaryTerm } from '../support/glossary/GlossaryTerm';
import {
@ -482,7 +483,7 @@ export const fillGlossaryTermDetails = async (
export const verifyTaskCreated = async (
page: Page,
glossaryFqn: string,
glossaryTermData: GlossaryTermData
glossaryTermData: string
) => {
const { apiContext } = await getApiContext(page);
const entityLink = encodeURIComponent(`<#E::glossary::${glossaryFqn}>`);
@ -509,7 +510,7 @@ export const verifyTaskCreated = async (
intervals: [40_000, 30_000],
}
)
.toContain(glossaryTermData.name);
.toContain(glossaryTermData);
};
export const validateGlossaryTermTask = async (
@ -1233,6 +1234,92 @@ export const filterStatus = async (
}
};
export const addMultiOwnerInDialog = async (data: {
page: Page;
ownerNames: string | string[];
activatorBtnLocator: string;
endpoint: EntityTypeEndpoint;
resultTestId?: string;
isSelectableInsideForm?: boolean;
type: 'Teams' | 'Users';
clearAll?: boolean;
}) => {
const {
page,
ownerNames,
activatorBtnLocator,
resultTestId = 'owner-link',
isSelectableInsideForm = false,
endpoint,
type,
clearAll = true,
} = data;
const isMultipleOwners = Array.isArray(ownerNames);
const owners = isMultipleOwners ? ownerNames : [ownerNames];
await page.click(activatorBtnLocator);
await expect(page.locator("[data-testid='select-owner-tabs']")).toBeVisible();
await page.waitForSelector('[data-testid="loader"]', { state: 'detached' });
await page
.locator("[data-testid='select-owner-tabs']")
.getByRole('tab', { name: 'Users' })
.click();
await page.waitForSelector('[data-testid="loader"]', { state: 'detached' });
if (clearAll && isMultipleOwners) {
await page.click('[data-testid="clear-all-button"]');
}
for (const ownerName of owners) {
const searchOwner = page.waitForResponse(
'api/v1/search/query?q=*&index=user_search_index*'
);
await page.locator('[data-testid="owner-select-users-search-bar"]').clear();
await page.fill('[data-testid="owner-select-users-search-bar"]', ownerName);
await searchOwner;
await page.waitForSelector('[data-testid="loader"]', { state: 'detached' });
const ownerItem = page.getByRole('listitem', {
name: ownerName,
exact: true,
});
if (type === 'Teams') {
if (isSelectableInsideForm) {
await ownerItem.click();
} else {
const patchRequest = page.waitForResponse(`/api/v1/${endpoint}/*`);
await ownerItem.click();
await patchRequest;
}
} else {
await ownerItem.click();
}
}
if (isMultipleOwners) {
const updateButton = page.getByTestId('selectable-list-update-btn');
if (isSelectableInsideForm) {
await updateButton.click();
} else {
const patchRequest = page.waitForResponse(`/api/v1/${endpoint}/*`);
await updateButton.click();
await patchRequest;
}
}
for (const name of owners) {
await expect(page.locator(`[data-testid="${resultTestId}"]`)).toContainText(
name
);
}
};
export const dragAndDropColumn = async (
page: Page,
dragColumn: string,
@ -1255,3 +1342,72 @@ export const dragAndDropColumn = async (
},
});
};
export const getEscapedTermFqn = (term: GlossaryTermData) => {
// eslint-disable-next-line no-useless-escape
return term.fullyQualifiedName.replace(/\"/g, '\\"');
};
export const openEditGlossaryTermModal = async (
page: Page,
term: GlossaryTermData
) => {
const escapedFqn = getEscapedTermFqn(term);
const termRow = page.locator(`[data-row-key="${escapedFqn}"]`);
const glossaryTermRes = page.waitForResponse('/api/v1/glossaryTerms/name/*');
await termRow.getByTestId('edit-button').click();
await glossaryTermRes;
await page.waitForSelector('[role="dialog"].edit-glossary-modal');
await expect(
page.locator('[role="dialog"].edit-glossary-modal')
).toBeVisible();
await expect(page.locator('.ant-modal-title')).toContainText(
'Edit Glossary Term'
);
};
export const updateGlossaryTermOwners = async (
page: Page,
term: GlossaryTermData,
owners: UserTeamRef[]
) => {
await openEditGlossaryTermModal(page, term);
const ownerLocator = '.edit-glossary-modal [data-testid="add-owner"]';
await addMultiOwnerInDialog({
page,
ownerNames: owners.map((owner) => owner.name),
activatorBtnLocator: ownerLocator,
resultTestId: 'owner-container',
endpoint: EntityTypeEndpoint.GlossaryTerm,
isSelectableInsideForm: true,
type: 'Users',
});
const glossaryTermResponse = page.waitForResponse('/api/v1/glossaryTerms/*');
await page.getByTestId('save-glossary-term').click();
await glossaryTermResponse;
};
export const updateGlossaryTermReviewers = async (
page: Page,
term: GlossaryTermData,
reviewers: UserTeamRef[]
) => {
await openEditGlossaryTermModal(page, term);
const reviewerLocator = '.edit-glossary-modal [data-testid="add-reviewers"]';
await addMultiOwnerInDialog({
page,
ownerNames: reviewers.map((reviewer) => reviewer.name),
activatorBtnLocator: reviewerLocator,
resultTestId: 'reviewers-container',
endpoint: EntityTypeEndpoint.Glossary,
isSelectableInsideForm: true,
type: 'Users',
});
const glossaryTermResponse = page.waitForResponse('/api/v1/glossaryTerms/*');
await page.getByTestId('save-glossary-term').click();
await glossaryTermResponse;
};