yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-27 09:58:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(api): Timeline API supports Glossary Terms now (#7229)

Browse Source 
Co-authored-by: RyanHolstien <RyanHolstien@users.noreply.github.com>
Co-authored-by: Shirshanka Das <shirshanka@apache.org>
This commit is contained in:
vojtechneradatos 2023-02-08 21:52:29 +01:00 committed by GitHub
parent 2f2eda0951
commit c72d0105c8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 144 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/dev-guides/timeline.md
View File

@ -3,7 +3,7 @@ title: "Timeline API"
---
The Timeline API supports viewing version history of schemas, documentation, tags, glossary terms, and other updates
to entities. At present, the API only supports Datasets.
to entities. At present, the API only supports Datasets and Glossary Terms.
## Compatibility

30
metadata-io/src/main/java/com/linkedin/metadata/timeline/TimelineServiceImpl.java
View File

@ -21,6 +21,7 @@ import com.linkedin.metadata.timeline.eventgenerator.EditableSchemaMetadataChang
import com.linkedin.metadata.timeline.eventgenerator.EntityChangeEventGenerator;
import com.linkedin.metadata.timeline.eventgenerator.EntityChangeEventGeneratorFactory;
import com.linkedin.metadata.timeline.eventgenerator.GlobalTagsChangeEventGenerator;
import com.linkedin.metadata.timeline.eventgenerator.GlossaryTermInfoChangeEventGenerator;
import com.linkedin.metadata.timeline.eventgenerator.GlossaryTermsChangeEventGenerator;
import com.linkedin.metadata.timeline.eventgenerator.InstitutionalMemoryChangeEventGenerator;
import com.linkedin.metadata.timeline.eventgenerator.OwnershipChangeEventGenerator;
@ -126,7 +127,32 @@ public class TimelineServiceImpl implements TimelineService {
}
datasetElementAspectRegistry.put(elementName, aspects);
}
// GlossaryTerm registry
HashMap<ChangeCategory, Set<String>> glossaryTermElementAspectRegistry = new HashMap<>();
String entityTypeGlossaryTerm = GLOSSARY_TERM_ENTITY_NAME;
for (ChangeCategory elementName : ChangeCategory.values()) {
Set<String> aspects = new HashSet<>();
switch (elementName) {
case OWNER: {
aspects.add(OWNERSHIP_ASPECT_NAME);
_entityChangeEventGeneratorFactory.addGenerator(entityTypeGlossaryTerm, elementName, OWNERSHIP_ASPECT_NAME,
new OwnershipChangeEventGenerator());
}
break;
case DOCUMENTATION: {
aspects.add(GLOSSARY_TERM_INFO_ASPECT_NAME);
_entityChangeEventGeneratorFactory.addGenerator(entityTypeGlossaryTerm, elementName, GLOSSARY_TERM_INFO_ASPECT_NAME,
new GlossaryTermInfoChangeEventGenerator());
}
break;
default:
break;
}
glossaryTermElementAspectRegistry.put(elementName, aspects);
}
entityTypeElementAspectRegistry.put(DATASET_ENTITY_NAME, datasetElementAspectRegistry);
entityTypeElementAspectRegistry.put(GLOSSARY_TERM_ENTITY_NAME, glossaryTermElementAspectRegistry);
}
Set<String> getAspectsFromElements(String entityType, Set<ChangeCategory> elementNames) {
@ -334,8 +360,8 @@ public class TimelineServiceImpl implements TimelineService {
List<ChangeTransaction> semanticChangeTransactions = new ArrayList<>();
JsonPatch rawDiff = getRawDiff(previousValue, currentValue);
for (ChangeCategory element : elementNames) {
EntityChangeEventGenerator entityChangeEventGenerator =
_entityChangeEventGeneratorFactory.getGenerator(entityType, element, aspectName);
EntityChangeEventGenerator entityChangeEventGenerator;
entityChangeEventGenerator = _entityChangeEventGeneratorFactory.getGenerator(entityType, element, aspectName);
if (entityChangeEventGenerator != null) {
try {
ChangeTransaction changeTransaction =

113
metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/GlossaryTermInfoChangeEventGenerator.java Normal file
View File

@ -0,0 +1,113 @@
package com.linkedin.metadata.timeline.eventgenerator;
import com.datahub.util.RecordUtils;
import com.github.fge.jsonpatch.JsonPatch;
import com.linkedin.common.AuditStamp;
import com.linkedin.common.urn.Urn;
import com.linkedin.glossary.GlossaryTermInfo;
import com.linkedin.metadata.entity.EntityAspect;
import com.linkedin.metadata.timeline.data.ChangeCategory;
import com.linkedin.metadata.timeline.data.ChangeEvent;
import com.linkedin.metadata.timeline.data.ChangeOperation;
import com.linkedin.metadata.timeline.data.ChangeTransaction;
import com.linkedin.metadata.timeline.data.SemanticChangeType;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static com.linkedin.metadata.Constants.*;
import static com.linkedin.metadata.timeline.eventgenerator.EditableDatasetPropertiesChangeEventGenerator.*;
public class GlossaryTermInfoChangeEventGenerator extends EntityChangeEventGenerator<GlossaryTermInfo> {
private static List<ChangeEvent> computeDiffs(GlossaryTermInfo baseDatasetProperties,
@Nonnull GlossaryTermInfo targetDatasetProperties, @Nonnull String entityUrn, AuditStamp auditStamp) {
List<ChangeEvent> changeEvents = new ArrayList<>();
String baseDescription = (baseDatasetProperties != null) ? baseDatasetProperties.getDefinition() : null;
String targetDescription = (targetDatasetProperties != null) ? targetDatasetProperties.getDefinition() : null;
if (baseDescription == null && targetDescription != null) {
// Description added
changeEvents.add(ChangeEvent.builder().entityUrn(entityUrn)
.category(ChangeCategory.DOCUMENTATION)
.operation(ChangeOperation.ADD)
.semVerChange(SemanticChangeType.MINOR)
.description(String.format(DESCRIPTION_ADDED, entityUrn, targetDescription))
.auditStamp(auditStamp)
.build());
} else if (baseDescription != null && targetDescription == null) {
// Description removed.
changeEvents.add(ChangeEvent.builder()
.entityUrn(entityUrn)
.category(ChangeCategory.DOCUMENTATION)
.operation(ChangeOperation.REMOVE)
.semVerChange(SemanticChangeType.MINOR)
.description(String.format(DESCRIPTION_REMOVED, entityUrn, baseDescription))
.auditStamp(auditStamp)
.build());
} else if (baseDescription != null && targetDescription != null && !baseDescription.equals(targetDescription)) {
// Description has been modified.
changeEvents.add(ChangeEvent.builder()
.entityUrn(entityUrn)
.category(ChangeCategory.DOCUMENTATION)
.operation(ChangeOperation.MODIFY)
.semVerChange(SemanticChangeType.MINOR)
.description(String.format(DESCRIPTION_CHANGED, entityUrn, baseDescription, targetDescription))
.auditStamp(auditStamp)
.build());
}
return changeEvents;
}
@Nullable
private static GlossaryTermInfo getGlossaryTermInfoFromAspect(EntityAspect entityAspect) {
if (entityAspect != null && entityAspect.getMetadata() != null) {
return RecordUtils.toRecordTemplate(GlossaryTermInfo.class, entityAspect.getMetadata());
}
return null;
}
@Override
public ChangeTransaction getSemanticDiff(EntityAspect previousValue, EntityAspect currentValue,
ChangeCategory element, JsonPatch rawDiff, boolean rawDiffsRequested) {
if (!previousValue.getAspect().equals(GLOSSARY_TERM_INFO_ASPECT_NAME) || !currentValue.getAspect()
.equals(GLOSSARY_TERM_INFO_ASPECT_NAME)) {
throw new IllegalArgumentException("Aspect is not " + GLOSSARY_TERM_INFO_ASPECT_NAME);
}
List<ChangeEvent> changeEvents = new ArrayList<>();
if (element == ChangeCategory.DOCUMENTATION) {
GlossaryTermInfo baseGlossaryTermInfo = getGlossaryTermInfoFromAspect(previousValue);
GlossaryTermInfo targetGlossaryTermInfo = getGlossaryTermInfoFromAspect(currentValue);
changeEvents.addAll(computeDiffs(baseGlossaryTermInfo, targetGlossaryTermInfo, currentValue.getUrn(), null));
}
// Assess the highest change at the transaction(schema) level.
SemanticChangeType highestSemanticChange = SemanticChangeType.NONE;
ChangeEvent highestChangeEvent =
changeEvents.stream().max(Comparator.comparing(ChangeEvent::getSemVerChange)).orElse(null);
if (highestChangeEvent != null) {
highestSemanticChange = highestChangeEvent.getSemVerChange();
}
return ChangeTransaction.builder()
.semVerChange(highestSemanticChange)
.changeEvents(changeEvents)
.timestamp(currentValue.getCreatedOn().getTime())
.rawDiff(rawDiffsRequested ? rawDiff : null)
.actor(currentValue.getCreatedBy())
.build();
}
@Override
public List<ChangeEvent> getChangeEvents(
@Nonnull Urn urn,
@Nonnull String entity,
@Nonnull String aspect,
@Nonnull Aspect<GlossaryTermInfo> from,
@Nonnull Aspect<GlossaryTermInfo> to,
@Nonnull AuditStamp auditStamp) {
return computeDiffs(from.getValue(), to.getValue(), urn.toString(), auditStamp);
}
}

2
metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/EntityChangeEventGeneratorRegistryFactory.java
View File

@ -5,6 +5,7 @@ import com.linkedin.entity.client.RestliEntityClient;
import com.linkedin.metadata.timeline.eventgenerator.AssertionRunEventChangeEventGenerator;
import com.linkedin.metadata.timeline.eventgenerator.DataProcessInstanceRunEventChangeEventGenerator;
import com.linkedin.metadata.timeline.eventgenerator.DatasetPropertiesChangeEventGenerator;
import com.linkedin.metadata.timeline.eventgenerator.GlossaryTermInfoChangeEventGenerator;
import com.linkedin.metadata.timeline.eventgenerator.DeprecationChangeEventGenerator;
import com.linkedin.metadata.timeline.eventgenerator.EditableDatasetPropertiesChangeEventGenerator;
import com.linkedin.metadata.timeline.eventgenerator.EditableSchemaMetadataChangeEventGenerator;
@ -49,6 +50,7 @@ public class EntityChangeEventGeneratorRegistryFactory {
registry.register(OWNERSHIP_ASPECT_NAME, new OwnershipChangeEventGenerator());
registry.register(INSTITUTIONAL_MEMORY_ASPECT_NAME, new InstitutionalMemoryChangeEventGenerator());
registry.register(DATASET_PROPERTIES_ASPECT_NAME, new DatasetPropertiesChangeEventGenerator());
registry.register(GLOSSARY_TERM_INFO_ASPECT_NAME, new GlossaryTermInfoChangeEventGenerator());
registry.register(DOMAINS_ASPECT_NAME, new SingleDomainChangeEventGenerator());
registry.register(DATASET_PROPERTIES_ASPECT_NAME, new DatasetPropertiesChangeEventGenerator());
registry.register(EDITABLE_DATASET_PROPERTIES_ASPECT_NAME, new EditableDatasetPropertiesChangeEventGenerator());