yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-08 00:28:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "refactor search index builder to store urn parts efficiently (#1937)" (#1970)

Browse Source 
This broke MAE processor because not all documents have urns now.

This reverts commit 5fca512a07cea75480633188e634ba07f49cdb28.
This commit is contained in:
John Plaisted 2020-10-28 11:32:18 -07:00 committed by GitHub
parent 38e928d235
commit 5fd2110b7f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 29 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
metadata-builders/src/main/java/com/linkedin/metadata/builders/search/DataProcessIndexBuilder.java
View File

@ -1,6 +1,7 @@
package com.linkedin.metadata.builders.search;
import com.linkedin.common.Ownership;
import com.linkedin.common.Status;
import com.linkedin.common.urn.DataProcessUrn;
import com.linkedin.data.template.RecordTemplate;
import com.linkedin.data.template.StringArray;
@ -41,7 +42,7 @@ public class DataProcessIndexBuilder extends BaseIndexBuilder<DataProcessDocumen
@Nonnull
private DataProcessDocument getDocumentToUpdateFromAspect(@Nonnull DataProcessUrn urn, @Nonnull Ownership ownership) {
final StringArray owners = BuilderUtils.getCorpUserOwners(ownership);
return new DataProcessDocument()
return setUrnDerivedFields(urn)
.setHasOwners(!owners.isEmpty())
.setOwners(owners);
}
@ -49,7 +50,7 @@ public class DataProcessIndexBuilder extends BaseIndexBuilder<DataProcessDocumen
@Nonnull
private DataProcessDocument getDocumentToUpdateFromAspect(@Nonnull DataProcessUrn urn,
@Nonnull DataProcessInfo dataProcessInfo) {
final DataProcessDocument dataProcessDocument = new DataProcessDocument();
DataProcessDocument dataProcessDocument = setUrnDerivedFields(urn);
if (dataProcessInfo.getInputs() != null) {
dataProcessDocument.setInputs(dataProcessInfo.getInputs())
.setNumInputDatasets(dataProcessInfo.getInputs().size());
@ -61,10 +62,16 @@ public class DataProcessIndexBuilder extends BaseIndexBuilder<DataProcessDocumen
return dataProcessDocument;
}
@Nonnull
private DataProcessDocument getDocumentToUpdateFromAspect(@Nonnull DataProcessUrn urn, @Nonnull Status status) {
return setUrnDerivedFields(urn)
.setRemoved(status.isRemoved());
}
@Nonnull
private List<DataProcessDocument> getDocumentsToUpdateFromSnapshotType(@Nonnull DataProcessSnapshot dataProcessSnapshot) {
final DataProcessUrn urn = dataProcessSnapshot.getUrn();
final List<DataProcessDocument> documents = dataProcessSnapshot.getAspects().stream().map(aspect -> {
DataProcessUrn urn = dataProcessSnapshot.getUrn();
return dataProcessSnapshot.getAspects().stream().map(aspect -> {
if (aspect.isDataProcessInfo()) {
return getDocumentToUpdateFromAspect(urn, aspect.getDataProcessInfo());
} else if (aspect.isOwnership()) {
@ -72,8 +79,6 @@ public class DataProcessIndexBuilder extends BaseIndexBuilder<DataProcessDocumen
}
return null;
}).filter(Objects::nonNull).collect(Collectors.toList());
documents.add(setUrnDerivedFields(urn));
return documents;
}
@Nullable

21
metadata-builders/src/main/java/com/linkedin/metadata/builders/search/DatasetIndexBuilder.java
View File

@ -51,40 +51,42 @@ public class DatasetIndexBuilder extends BaseIndexBuilder<DatasetDocument> {
@Nonnull
private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull Ownership ownership) {
final StringArray owners = BuilderUtils.getCorpUserOwners(ownership);
return new DatasetDocument()
return setUrnDerivedFields(urn)
.setHasOwners(!owners.isEmpty())
.setOwners(owners);
}
@Nonnull
private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull Status status) {
return new DatasetDocument()
return setUrnDerivedFields(urn)
.setRemoved(status.isRemoved());
}
@Nonnull
private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull DatasetDeprecation deprecation) {
return new DatasetDocument().setDeprecated(deprecation.isDeprecated());
return setUrnDerivedFields(urn).setDeprecated(deprecation.isDeprecated());
}
@Nonnull
private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull DatasetProperties datasetProperties) {
final DatasetDocument doc = new DatasetDocument();
if (datasetProperties.getDescription() != null) {
final DatasetDocument doc = setUrnDerivedFields(urn);
if (datasetProperties.hasDescription()) {
doc.setDescription(datasetProperties.getDescription());
} else {
doc.setDescription("");
}
return doc;
}
@Nonnull
private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull SchemaMetadata schemaMetadata) {
return new DatasetDocument()
return setUrnDerivedFields(urn)
.setHasSchema(true);
}
@Nonnull
private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull UpstreamLineage upstreamLineage) {
return new DatasetDocument()
return setUrnDerivedFields(urn)
.setUpstreams(new DatasetUrnArray(
upstreamLineage.getUpstreams().stream().map(upstream -> upstream.getDataset()).collect(Collectors.toList())
));
@ -93,7 +95,7 @@ public class DatasetIndexBuilder extends BaseIndexBuilder<DatasetDocument> {
@Nonnull
private List<DatasetDocument> getDocumentsToUpdateFromSnapshotType(@Nonnull DatasetSnapshot datasetSnapshot) {
final DatasetUrn urn = datasetSnapshot.getUrn();
final List<DatasetDocument> documents = datasetSnapshot.getAspects().stream().map(aspect -> {
return datasetSnapshot.getAspects().stream().map(aspect -> {
if (aspect.isDatasetDeprecation()) {
return getDocumentToUpdateFromAspect(urn, aspect.getDatasetDeprecation());
} else if (aspect.isDatasetProperties()) {
@ -109,8 +111,6 @@ public class DatasetIndexBuilder extends BaseIndexBuilder<DatasetDocument> {
}
return null;
}).filter(Objects::nonNull).collect(Collectors.toList());
documents.add(setUrnDerivedFields(urn));
return documents;
}
@Override
@ -123,7 +123,6 @@ public class DatasetIndexBuilder extends BaseIndexBuilder<DatasetDocument> {
}
@Override
@Nonnull
public Class<DatasetDocument> getDocumentType() {
return DatasetDocument.class;
}

5
metadata-builders/src/test/java/com/linkedin/metadata/builders/search/DataProcessIndexBuilderTest.java
View File

@ -39,9 +39,10 @@ public class DataProcessIndexBuilderTest {
new DataProcessSnapshot().setUrn(dataProcessUrn).setAspects(dataProcessAspectArray);
List<DataProcessDocument> actualDocs = new DataProcessIndexBuilder().getDocumentsToUpdate(dataProcessSnapshot);
assertEquals(actualDocs.size(), 2);
assertEquals(actualDocs.size(), 1);
assertEquals(actualDocs.get(0).getUrn(), dataProcessUrn);
assertEquals(actualDocs.get(0).getInputs().get(0), inputDatasetUrn);
assertEquals(actualDocs.get(0).getOutputs().get(0), outputDatasetUrn);
assertEquals(actualDocs.get(1).getUrn(), dataProcessUrn);
}
}

10
metadata-builders/src/test/java/com/linkedin/metadata/builders/search/DatasetIndexBuilderTest.java
View File

@ -25,16 +25,16 @@ public class DatasetIndexBuilderTest {
DatasetSnapshot datasetSnapshot = ModelUtils.newSnapshot(DatasetSnapshot.class, datasetUrn,
Collections.singletonList(ModelUtils.newAspectUnion(DatasetAspect.class, datasetProperties)));
List<DatasetDocument> actualDocs = new DatasetIndexBuilder().getDocumentsToUpdate(datasetSnapshot);
assertEquals(actualDocs.size(), 2);
assertEquals(actualDocs.size(), 1);
assertEquals(actualDocs.get(0).getUrn(), datasetUrn);
assertEquals(actualDocs.get(0).getDescription(), "baz");
assertEquals(actualDocs.get(1).getUrn(), datasetUrn);
datasetProperties = new DatasetProperties();
datasetSnapshot = ModelUtils.newSnapshot(DatasetSnapshot.class, datasetUrn,
Collections.singletonList(ModelUtils.newAspectUnion(DatasetAspect.class, datasetProperties)));
actualDocs = new DatasetIndexBuilder().getDocumentsToUpdate(datasetSnapshot);
assertEquals(actualDocs.size(), 2);
assertNull(actualDocs.get(0).getDescription());
assertEquals(actualDocs.get(1).getUrn(), datasetUrn);
assertEquals(actualDocs.size(), 1);
assertEquals(actualDocs.get(0).getUrn(), datasetUrn);
assertEquals(actualDocs.get(0).getDescription(), "");
}
}