yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-10 17:50:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "refactor search index builder to store urn parts efficiently (#1937)" (#1970)

Browse Source 
This broke MAE processor because not all documents have urns now.

This reverts commit 5fca512a07cea75480633188e634ba07f49cdb28.
This commit is contained in:
John Plaisted 2020-10-28 11:32:18 -07:00 committed by GitHub
parent 38e928d235
commit 5fd2110b7f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 29 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
metadata-builders/src/main/java/com/linkedin/metadata/builders/search/DataProcessIndexBuilder.java
View File

@ -1,6 +1,7 @@
package com.linkedin.metadata.builders.search; package com.linkedin.metadata.builders.search;
import com.linkedin.common.Ownership; import com.linkedin.common.Ownership;
import com.linkedin.common.Status;
import com.linkedin.common.urn.DataProcessUrn; import com.linkedin.common.urn.DataProcessUrn;
import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.RecordTemplate;
import com.linkedin.data.template.StringArray; import com.linkedin.data.template.StringArray;
@ -41,7 +42,7 @@ public class DataProcessIndexBuilder extends BaseIndexBuilder<DataProcessDocumen
@Nonnull @Nonnull
private DataProcessDocument getDocumentToUpdateFromAspect(@Nonnull DataProcessUrn urn, @Nonnull Ownership ownership) { private DataProcessDocument getDocumentToUpdateFromAspect(@Nonnull DataProcessUrn urn, @Nonnull Ownership ownership) {
final StringArray owners = BuilderUtils.getCorpUserOwners(ownership); final StringArray owners = BuilderUtils.getCorpUserOwners(ownership);
return new DataProcessDocument() return setUrnDerivedFields(urn)
.setHasOwners(!owners.isEmpty()) .setHasOwners(!owners.isEmpty())
.setOwners(owners); .setOwners(owners);
} }
@ -49,7 +50,7 @@ public class DataProcessIndexBuilder extends BaseIndexBuilder<DataProcessDocumen
@Nonnull @Nonnull
private DataProcessDocument getDocumentToUpdateFromAspect(@Nonnull DataProcessUrn urn, private DataProcessDocument getDocumentToUpdateFromAspect(@Nonnull DataProcessUrn urn,
@Nonnull DataProcessInfo dataProcessInfo) { @Nonnull DataProcessInfo dataProcessInfo) {
final DataProcessDocument dataProcessDocument = new DataProcessDocument(); DataProcessDocument dataProcessDocument = setUrnDerivedFields(urn);
if (dataProcessInfo.getInputs() != null) { if (dataProcessInfo.getInputs() != null) {
dataProcessDocument.setInputs(dataProcessInfo.getInputs()) dataProcessDocument.setInputs(dataProcessInfo.getInputs())
.setNumInputDatasets(dataProcessInfo.getInputs().size()); .setNumInputDatasets(dataProcessInfo.getInputs().size());
@ -61,10 +62,16 @@ public class DataProcessIndexBuilder extends BaseIndexBuilder<DataProcessDocumen
return dataProcessDocument; return dataProcessDocument;
} }
@Nonnull
private DataProcessDocument getDocumentToUpdateFromAspect(@Nonnull DataProcessUrn urn, @Nonnull Status status) {
return setUrnDerivedFields(urn)
.setRemoved(status.isRemoved());
}
@Nonnull @Nonnull
private List<DataProcessDocument> getDocumentsToUpdateFromSnapshotType(@Nonnull DataProcessSnapshot dataProcessSnapshot) { private List<DataProcessDocument> getDocumentsToUpdateFromSnapshotType(@Nonnull DataProcessSnapshot dataProcessSnapshot) {
final DataProcessUrn urn = dataProcessSnapshot.getUrn(); DataProcessUrn urn = dataProcessSnapshot.getUrn();
final List<DataProcessDocument> documents = dataProcessSnapshot.getAspects().stream().map(aspect -> { return dataProcessSnapshot.getAspects().stream().map(aspect -> {
if (aspect.isDataProcessInfo()) { if (aspect.isDataProcessInfo()) {
return getDocumentToUpdateFromAspect(urn, aspect.getDataProcessInfo()); return getDocumentToUpdateFromAspect(urn, aspect.getDataProcessInfo());
} else if (aspect.isOwnership()) { } else if (aspect.isOwnership()) {
@ -72,8 +79,6 @@ public class DataProcessIndexBuilder extends BaseIndexBuilder<DataProcessDocumen
} }
return null; return null;
}).filter(Objects::nonNull).collect(Collectors.toList()); }).filter(Objects::nonNull).collect(Collectors.toList());
documents.add(setUrnDerivedFields(urn));
return documents;
} }
@Nullable @Nullable

21
metadata-builders/src/main/java/com/linkedin/metadata/builders/search/DatasetIndexBuilder.java
View File

@ -51,40 +51,42 @@ public class DatasetIndexBuilder extends BaseIndexBuilder<DatasetDocument> {
@Nonnull @Nonnull
private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull Ownership ownership) { private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull Ownership ownership) {
final StringArray owners = BuilderUtils.getCorpUserOwners(ownership); final StringArray owners = BuilderUtils.getCorpUserOwners(ownership);
return new DatasetDocument() return setUrnDerivedFields(urn)
.setHasOwners(!owners.isEmpty()) .setHasOwners(!owners.isEmpty())
.setOwners(owners); .setOwners(owners);
} }
@Nonnull @Nonnull
private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull Status status) { private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull Status status) {
return new DatasetDocument() return setUrnDerivedFields(urn)
.setRemoved(status.isRemoved()); .setRemoved(status.isRemoved());
} }
@Nonnull @Nonnull
private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull DatasetDeprecation deprecation) { private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull DatasetDeprecation deprecation) {
return new DatasetDocument().setDeprecated(deprecation.isDeprecated()); return setUrnDerivedFields(urn).setDeprecated(deprecation.isDeprecated());
} }
@Nonnull @Nonnull
private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull DatasetProperties datasetProperties) { private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull DatasetProperties datasetProperties) {
final DatasetDocument doc = new DatasetDocument(); final DatasetDocument doc = setUrnDerivedFields(urn);
if (datasetProperties.getDescription() != null) { if (datasetProperties.hasDescription()) {
doc.setDescription(datasetProperties.getDescription()); doc.setDescription(datasetProperties.getDescription());
} else {
doc.setDescription("");
} }
return doc; return doc;
} }
@Nonnull @Nonnull
private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull SchemaMetadata schemaMetadata) { private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull SchemaMetadata schemaMetadata) {
return new DatasetDocument() return setUrnDerivedFields(urn)
.setHasSchema(true); .setHasSchema(true);
} }
@Nonnull @Nonnull
private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull UpstreamLineage upstreamLineage) { private DatasetDocument getDocumentToUpdateFromAspect(@Nonnull DatasetUrn urn, @Nonnull UpstreamLineage upstreamLineage) {
return new DatasetDocument() return setUrnDerivedFields(urn)
.setUpstreams(new DatasetUrnArray( .setUpstreams(new DatasetUrnArray(
upstreamLineage.getUpstreams().stream().map(upstream -> upstream.getDataset()).collect(Collectors.toList()) upstreamLineage.getUpstreams().stream().map(upstream -> upstream.getDataset()).collect(Collectors.toList())
)); ));
@ -93,7 +95,7 @@ public class DatasetIndexBuilder extends BaseIndexBuilder<DatasetDocument> {
@Nonnull @Nonnull
private List<DatasetDocument> getDocumentsToUpdateFromSnapshotType(@Nonnull DatasetSnapshot datasetSnapshot) { private List<DatasetDocument> getDocumentsToUpdateFromSnapshotType(@Nonnull DatasetSnapshot datasetSnapshot) {
final DatasetUrn urn = datasetSnapshot.getUrn(); final DatasetUrn urn = datasetSnapshot.getUrn();
final List<DatasetDocument> documents = datasetSnapshot.getAspects().stream().map(aspect -> { return datasetSnapshot.getAspects().stream().map(aspect -> {
if (aspect.isDatasetDeprecation()) { if (aspect.isDatasetDeprecation()) {
return getDocumentToUpdateFromAspect(urn, aspect.getDatasetDeprecation()); return getDocumentToUpdateFromAspect(urn, aspect.getDatasetDeprecation());
} else if (aspect.isDatasetProperties()) { } else if (aspect.isDatasetProperties()) {
@ -109,8 +111,6 @@ public class DatasetIndexBuilder extends BaseIndexBuilder<DatasetDocument> {
} }
return null; return null;
}).filter(Objects::nonNull).collect(Collectors.toList()); }).filter(Objects::nonNull).collect(Collectors.toList());
documents.add(setUrnDerivedFields(urn));
return documents;
} }
@Override @Override
@ -123,7 +123,6 @@ public class DatasetIndexBuilder extends BaseIndexBuilder<DatasetDocument> {
} }
@Override @Override
@Nonnull
public Class<DatasetDocument> getDocumentType() { public Class<DatasetDocument> getDocumentType() {
return DatasetDocument.class; return DatasetDocument.class;
} }

5
metadata-builders/src/test/java/com/linkedin/metadata/builders/search/DataProcessIndexBuilderTest.java
View File

@ -39,9 +39,10 @@ public class DataProcessIndexBuilderTest {
new DataProcessSnapshot().setUrn(dataProcessUrn).setAspects(dataProcessAspectArray); new DataProcessSnapshot().setUrn(dataProcessUrn).setAspects(dataProcessAspectArray);
List<DataProcessDocument> actualDocs = new DataProcessIndexBuilder().getDocumentsToUpdate(dataProcessSnapshot); List<DataProcessDocument> actualDocs = new DataProcessIndexBuilder().getDocumentsToUpdate(dataProcessSnapshot);
assertEquals(actualDocs.size(), 2); assertEquals(actualDocs.size(), 1);
assertEquals(actualDocs.get(0).getUrn(), dataProcessUrn);
assertEquals(actualDocs.get(0).getInputs().get(0), inputDatasetUrn); assertEquals(actualDocs.get(0).getInputs().get(0), inputDatasetUrn);
assertEquals(actualDocs.get(0).getOutputs().get(0), outputDatasetUrn); assertEquals(actualDocs.get(0).getOutputs().get(0), outputDatasetUrn);
assertEquals(actualDocs.get(1).getUrn(), dataProcessUrn);
} }
} }

10
metadata-builders/src/test/java/com/linkedin/metadata/builders/search/DatasetIndexBuilderTest.java
View File

@ -25,16 +25,16 @@ public class DatasetIndexBuilderTest {
DatasetSnapshot datasetSnapshot = ModelUtils.newSnapshot(DatasetSnapshot.class, datasetUrn, DatasetSnapshot datasetSnapshot = ModelUtils.newSnapshot(DatasetSnapshot.class, datasetUrn,
Collections.singletonList(ModelUtils.newAspectUnion(DatasetAspect.class, datasetProperties))); Collections.singletonList(ModelUtils.newAspectUnion(DatasetAspect.class, datasetProperties)));
List<DatasetDocument> actualDocs = new DatasetIndexBuilder().getDocumentsToUpdate(datasetSnapshot); List<DatasetDocument> actualDocs = new DatasetIndexBuilder().getDocumentsToUpdate(datasetSnapshot);
assertEquals(actualDocs.size(), 2); assertEquals(actualDocs.size(), 1);
assertEquals(actualDocs.get(0).getUrn(), datasetUrn);
assertEquals(actualDocs.get(0).getDescription(), "baz"); assertEquals(actualDocs.get(0).getDescription(), "baz");
assertEquals(actualDocs.get(1).getUrn(), datasetUrn);
datasetProperties = new DatasetProperties(); datasetProperties = new DatasetProperties();
datasetSnapshot = ModelUtils.newSnapshot(DatasetSnapshot.class, datasetUrn, datasetSnapshot = ModelUtils.newSnapshot(DatasetSnapshot.class, datasetUrn,
Collections.singletonList(ModelUtils.newAspectUnion(DatasetAspect.class, datasetProperties))); Collections.singletonList(ModelUtils.newAspectUnion(DatasetAspect.class, datasetProperties)));
actualDocs = new DatasetIndexBuilder().getDocumentsToUpdate(datasetSnapshot); actualDocs = new DatasetIndexBuilder().getDocumentsToUpdate(datasetSnapshot);
assertEquals(actualDocs.size(), 2); assertEquals(actualDocs.size(), 1);
assertNull(actualDocs.get(0).getDescription()); assertEquals(actualDocs.get(0).getUrn(), datasetUrn);
assertEquals(actualDocs.get(1).getUrn(), datasetUrn); assertEquals(actualDocs.get(0).getDescription(), "");
} }
} }