mirror of
https://github.com/datahub-project/datahub.git
synced 2025-11-03 04:10:43 +00:00
parent
d788cd753c
commit
d795a5357d
@ -22,13 +22,13 @@ public class ProtobufUtils {
|
||||
|
||||
public static String collapseLocationComments(DescriptorProtos.SourceCodeInfo.Location location) {
|
||||
return Stream.concat(
|
||||
location.getLeadingDetachedCommentsList().stream(),
|
||||
Stream.of(location.getLeadingComments(), location.getTrailingComments()))
|
||||
.filter(Objects::nonNull)
|
||||
.flatMap(line -> Arrays.stream(line.split("\n")))
|
||||
.map(line -> line.replaceFirst("^[*/ ]+", ""))
|
||||
.collect(Collectors.joining("\n"))
|
||||
.trim();
|
||||
location.getLeadingDetachedCommentsList().stream(),
|
||||
Stream.of(location.getLeadingComments(), location.getTrailingComments()))
|
||||
.filter(Objects::nonNull)
|
||||
.flatMap(line -> Arrays.stream(line.split("\n")))
|
||||
.map(line -> line.replaceFirst("^[*/ ]+", ""))
|
||||
.collect(Collectors.joining("\n"))
|
||||
.trim();
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@ -19,10 +19,6 @@ import com.linkedin.schema.StringType;
|
||||
import datahub.protobuf.ProtobufUtils;
|
||||
import datahub.protobuf.visitors.ProtobufModelVisitor;
|
||||
import datahub.protobuf.visitors.VisitContext;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
@ -31,7 +27,9 @@ import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
|
||||
@Builder(toBuilder = true)
|
||||
@Getter
|
||||
@ -87,72 +85,78 @@ public class ProtobufField implements ProtobufElement {
|
||||
|
||||
@Override
|
||||
public String nativeType() {
|
||||
return Optional.ofNullable(nativeType).orElseGet(() -> {
|
||||
if (fieldProto.getTypeName().isEmpty()) {
|
||||
return fieldProto.getType().name().split("_")[1].toLowerCase();
|
||||
} else {
|
||||
return fieldProto.getTypeName().replaceFirst("^[.]", "");
|
||||
}
|
||||
});
|
||||
return Optional.ofNullable(nativeType)
|
||||
.orElseGet(
|
||||
() -> {
|
||||
if (fieldProto.getTypeName().isEmpty()) {
|
||||
return fieldProto.getType().name().split("_")[1].toLowerCase();
|
||||
} else {
|
||||
return fieldProto.getTypeName().replaceFirst("^[.]", "");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public String fieldPathType() {
|
||||
return Optional.ofNullable(fieldPathType).orElseGet(() -> {
|
||||
final String pathType;
|
||||
return Optional.ofNullable(fieldPathType)
|
||||
.orElseGet(
|
||||
() -> {
|
||||
final String pathType;
|
||||
|
||||
switch (fieldProto.getType()) {
|
||||
case TYPE_DOUBLE:
|
||||
pathType = "double";
|
||||
break;
|
||||
case TYPE_FLOAT:
|
||||
pathType = "float";
|
||||
break;
|
||||
case TYPE_SFIXED64:
|
||||
case TYPE_FIXED64:
|
||||
case TYPE_UINT64:
|
||||
case TYPE_INT64:
|
||||
case TYPE_SINT64:
|
||||
pathType = "long";
|
||||
break;
|
||||
case TYPE_FIXED32:
|
||||
case TYPE_SFIXED32:
|
||||
case TYPE_INT32:
|
||||
case TYPE_UINT32:
|
||||
case TYPE_SINT32:
|
||||
pathType = "int";
|
||||
break;
|
||||
case TYPE_BYTES:
|
||||
pathType = "bytes";
|
||||
break;
|
||||
case TYPE_ENUM:
|
||||
pathType = "enum";
|
||||
break;
|
||||
case TYPE_BOOL:
|
||||
pathType = "boolean";
|
||||
break;
|
||||
case TYPE_STRING:
|
||||
pathType = "string";
|
||||
break;
|
||||
case TYPE_GROUP:
|
||||
case TYPE_MESSAGE:
|
||||
pathType = nativeType().replace(".", "_");
|
||||
break;
|
||||
default:
|
||||
throw new IllegalStateException(
|
||||
String.format("Unexpected FieldDescriptorProto => FieldPathType %s", fieldProto.getType()));
|
||||
}
|
||||
switch (fieldProto.getType()) {
|
||||
case TYPE_DOUBLE:
|
||||
pathType = "double";
|
||||
break;
|
||||
case TYPE_FLOAT:
|
||||
pathType = "float";
|
||||
break;
|
||||
case TYPE_SFIXED64:
|
||||
case TYPE_FIXED64:
|
||||
case TYPE_UINT64:
|
||||
case TYPE_INT64:
|
||||
case TYPE_SINT64:
|
||||
pathType = "long";
|
||||
break;
|
||||
case TYPE_FIXED32:
|
||||
case TYPE_SFIXED32:
|
||||
case TYPE_INT32:
|
||||
case TYPE_UINT32:
|
||||
case TYPE_SINT32:
|
||||
pathType = "int";
|
||||
break;
|
||||
case TYPE_BYTES:
|
||||
pathType = "bytes";
|
||||
break;
|
||||
case TYPE_ENUM:
|
||||
pathType = "enum";
|
||||
break;
|
||||
case TYPE_BOOL:
|
||||
pathType = "boolean";
|
||||
break;
|
||||
case TYPE_STRING:
|
||||
pathType = "string";
|
||||
break;
|
||||
case TYPE_GROUP:
|
||||
case TYPE_MESSAGE:
|
||||
pathType = nativeType().replace(".", "_");
|
||||
break;
|
||||
default:
|
||||
throw new IllegalStateException(
|
||||
String.format(
|
||||
"Unexpected FieldDescriptorProto => FieldPathType %s",
|
||||
fieldProto.getType()));
|
||||
}
|
||||
|
||||
StringArray fieldPath = new StringArray();
|
||||
StringArray fieldPath = new StringArray();
|
||||
|
||||
if (schemaFieldDataType().getType().isArrayType()) {
|
||||
fieldPath.add("[type=array]");
|
||||
}
|
||||
if (schemaFieldDataType().getType().isArrayType()) {
|
||||
fieldPath.add("[type=array]");
|
||||
}
|
||||
|
||||
fieldPath.add(String.format("[type=%s]", pathType));
|
||||
fieldPath.add(String.format("[type=%s]", pathType));
|
||||
|
||||
return String.join(".", fieldPath);
|
||||
});
|
||||
return String.join(".", fieldPath);
|
||||
});
|
||||
}
|
||||
|
||||
public boolean isMessage() {
|
||||
@ -165,92 +169,110 @@ public class ProtobufField implements ProtobufElement {
|
||||
}
|
||||
|
||||
public SchemaFieldDataType schemaFieldDataType() throws IllegalStateException {
|
||||
return Optional.ofNullable(schemaFieldDataType).orElseGet(() -> {
|
||||
final SchemaFieldDataType.Type fieldType;
|
||||
return Optional.ofNullable(schemaFieldDataType)
|
||||
.orElseGet(
|
||||
() -> {
|
||||
final SchemaFieldDataType.Type fieldType;
|
||||
|
||||
switch (fieldProto.getType()) {
|
||||
case TYPE_DOUBLE:
|
||||
case TYPE_FLOAT:
|
||||
case TYPE_INT64:
|
||||
case TYPE_UINT64:
|
||||
case TYPE_INT32:
|
||||
case TYPE_UINT32:
|
||||
case TYPE_SINT32:
|
||||
case TYPE_SINT64:
|
||||
fieldType = SchemaFieldDataType.Type.create(new NumberType());
|
||||
break;
|
||||
case TYPE_GROUP:
|
||||
case TYPE_MESSAGE:
|
||||
fieldType = SchemaFieldDataType.Type.create(new RecordType());
|
||||
break;
|
||||
case TYPE_BYTES:
|
||||
fieldType = SchemaFieldDataType.Type.create(new BytesType());
|
||||
break;
|
||||
case TYPE_ENUM:
|
||||
fieldType = SchemaFieldDataType.Type.create(new EnumType());
|
||||
break;
|
||||
case TYPE_BOOL:
|
||||
fieldType = SchemaFieldDataType.Type.create(new BooleanType());
|
||||
break;
|
||||
case TYPE_STRING:
|
||||
fieldType = SchemaFieldDataType.Type.create(new StringType());
|
||||
break;
|
||||
case TYPE_FIXED64:
|
||||
case TYPE_FIXED32:
|
||||
case TYPE_SFIXED32:
|
||||
case TYPE_SFIXED64:
|
||||
fieldType = SchemaFieldDataType.Type.create(new FixedType());
|
||||
break;
|
||||
default:
|
||||
throw new IllegalStateException(
|
||||
String.format("Unexpected FieldDescriptorProto => SchemaFieldDataType: %s", fieldProto.getType()));
|
||||
}
|
||||
switch (fieldProto.getType()) {
|
||||
case TYPE_DOUBLE:
|
||||
case TYPE_FLOAT:
|
||||
case TYPE_INT64:
|
||||
case TYPE_UINT64:
|
||||
case TYPE_INT32:
|
||||
case TYPE_UINT32:
|
||||
case TYPE_SINT32:
|
||||
case TYPE_SINT64:
|
||||
fieldType = SchemaFieldDataType.Type.create(new NumberType());
|
||||
break;
|
||||
case TYPE_GROUP:
|
||||
case TYPE_MESSAGE:
|
||||
fieldType = SchemaFieldDataType.Type.create(new RecordType());
|
||||
break;
|
||||
case TYPE_BYTES:
|
||||
fieldType = SchemaFieldDataType.Type.create(new BytesType());
|
||||
break;
|
||||
case TYPE_ENUM:
|
||||
fieldType = SchemaFieldDataType.Type.create(new EnumType());
|
||||
break;
|
||||
case TYPE_BOOL:
|
||||
fieldType = SchemaFieldDataType.Type.create(new BooleanType());
|
||||
break;
|
||||
case TYPE_STRING:
|
||||
fieldType = SchemaFieldDataType.Type.create(new StringType());
|
||||
break;
|
||||
case TYPE_FIXED64:
|
||||
case TYPE_FIXED32:
|
||||
case TYPE_SFIXED32:
|
||||
case TYPE_SFIXED64:
|
||||
fieldType = SchemaFieldDataType.Type.create(new FixedType());
|
||||
break;
|
||||
default:
|
||||
throw new IllegalStateException(
|
||||
String.format(
|
||||
"Unexpected FieldDescriptorProto => SchemaFieldDataType: %s",
|
||||
fieldProto.getType()));
|
||||
}
|
||||
|
||||
if (fieldProto.getLabel().equals(FieldDescriptorProto.Label.LABEL_REPEATED)) {
|
||||
return new SchemaFieldDataType().setType(
|
||||
SchemaFieldDataType.Type.create(new ArrayType().setNestedType(new StringArray())));
|
||||
}
|
||||
if (fieldProto.getLabel().equals(FieldDescriptorProto.Label.LABEL_REPEATED)) {
|
||||
return new SchemaFieldDataType()
|
||||
.setType(
|
||||
SchemaFieldDataType.Type.create(
|
||||
new ArrayType().setNestedType(new StringArray())));
|
||||
}
|
||||
|
||||
return new SchemaFieldDataType().setType(fieldType);
|
||||
});
|
||||
return new SchemaFieldDataType().setType(fieldType);
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public Stream<SourceCodeInfo.Location> messageLocations() {
|
||||
List<SourceCodeInfo.Location> fileLocations = fileProto().getSourceCodeInfo().getLocationList();
|
||||
return fileLocations.stream()
|
||||
.filter(loc -> loc.getPathCount() > 1 && loc.getPath(0) == FileDescriptorProto.MESSAGE_TYPE_FIELD_NUMBER);
|
||||
.filter(
|
||||
loc ->
|
||||
loc.getPathCount() > 1
|
||||
&& loc.getPath(0) == FileDescriptorProto.MESSAGE_TYPE_FIELD_NUMBER);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String comment() {
|
||||
return messageLocations().filter(location -> location.getPathCount() > 3)
|
||||
.filter(location -> !ProtobufUtils.collapseLocationComments(location).isEmpty() && !isEnumType(
|
||||
location.getPathList()))
|
||||
.filter(location -> {
|
||||
List<Integer> pathList = location.getPathList();
|
||||
DescriptorProto messageType = fileProto().getMessageType(pathList.get(1));
|
||||
return messageLocations()
|
||||
.filter(location -> location.getPathCount() > 3)
|
||||
.filter(
|
||||
location ->
|
||||
!ProtobufUtils.collapseLocationComments(location).isEmpty()
|
||||
&& !isEnumType(location.getPathList()))
|
||||
.filter(
|
||||
location -> {
|
||||
List<Integer> pathList = location.getPathList();
|
||||
DescriptorProto messageType = fileProto().getMessageType(pathList.get(1));
|
||||
|
||||
if (!isNestedType && location.getPath(2) == DescriptorProto.FIELD_FIELD_NUMBER
|
||||
&& fieldProto == messageType.getField(location.getPath(3))) {
|
||||
return true;
|
||||
} else if (isNestedType && location.getPath(2) == DescriptorProto.NESTED_TYPE_FIELD_NUMBER
|
||||
&& fieldProto == getNestedTypeFields(pathList, messageType)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
})
|
||||
if (!isNestedType
|
||||
&& location.getPath(2) == DescriptorProto.FIELD_FIELD_NUMBER
|
||||
&& fieldProto == messageType.getField(location.getPath(3))) {
|
||||
return true;
|
||||
} else if (isNestedType
|
||||
&& location.getPath(2) == DescriptorProto.NESTED_TYPE_FIELD_NUMBER
|
||||
&& fieldProto == getNestedTypeFields(pathList, messageType)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
})
|
||||
.map(ProtobufUtils::collapseLocationComments)
|
||||
.collect(Collectors.joining("\n"))
|
||||
.trim();
|
||||
}
|
||||
|
||||
private FieldDescriptorProto getNestedTypeFields(List<Integer> pathList, DescriptorProto messageType) {
|
||||
private FieldDescriptorProto getNestedTypeFields(
|
||||
List<Integer> pathList, DescriptorProto messageType) {
|
||||
int pathSize = pathList.size();
|
||||
List<Integer> nestedValues = new ArrayList<>(pathSize);
|
||||
|
||||
for (int index = 0; index < pathSize; index++) {
|
||||
if (index > 1 && index % 2 == 0 && pathList.get(index) == DescriptorProto.NESTED_TYPE_FIELD_NUMBER) {
|
||||
if (index > 1
|
||||
&& index % 2 == 0
|
||||
&& pathList.get(index) == DescriptorProto.NESTED_TYPE_FIELD_NUMBER) {
|
||||
nestedValues.add(pathList.get(index + 1));
|
||||
}
|
||||
}
|
||||
@ -260,7 +282,9 @@ public class ProtobufField implements ProtobufElement {
|
||||
}
|
||||
|
||||
int fieldIndex = pathList.get(pathList.size() - 1);
|
||||
if (isFieldPath(pathList) && pathSize % 2 == 0 && fieldIndex < messageType.getFieldList().size()) {
|
||||
if (isFieldPath(pathList)
|
||||
&& pathSize % 2 == 0
|
||||
&& fieldIndex < messageType.getFieldList().size()) {
|
||||
return messageType.getField(fieldIndex);
|
||||
}
|
||||
|
||||
@ -273,7 +297,9 @@ public class ProtobufField implements ProtobufElement {
|
||||
|
||||
private boolean isEnumType(List<Integer> pathList) {
|
||||
for (int index = 0; index < pathList.size(); index++) {
|
||||
if (index > 1 && index % 2 == 0 && pathList.get(index) == DescriptorProto.ENUM_TYPE_FIELD_NUMBER) {
|
||||
if (index > 1
|
||||
&& index % 2 == 0
|
||||
&& pathList.get(index) == DescriptorProto.ENUM_TYPE_FIELD_NUMBER) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -327,7 +353,9 @@ public class ProtobufField implements ProtobufElement {
|
||||
}
|
||||
|
||||
public List<DescriptorProtos.EnumValueDescriptorProto> getEnumValues() {
|
||||
return getEnumDescriptor().map(DescriptorProtos.EnumDescriptorProto::getValueList).orElse(Collections.emptyList());
|
||||
return getEnumDescriptor()
|
||||
.map(DescriptorProtos.EnumDescriptorProto::getValueList)
|
||||
.orElse(Collections.emptyList());
|
||||
}
|
||||
|
||||
public Map<String, String> getEnumValuesWithComments() {
|
||||
@ -347,11 +375,12 @@ public class ProtobufField implements ProtobufElement {
|
||||
for (int i = 0; i < values.size(); i++) {
|
||||
DescriptorProtos.EnumValueDescriptorProto value = values.get(i);
|
||||
int finalI = i;
|
||||
String comment = locations.stream()
|
||||
.filter(loc -> isEnumValueLocation(loc, enumIndex, finalI))
|
||||
.findFirst()
|
||||
.map(ProtobufUtils::collapseLocationComments)
|
||||
.orElse("");
|
||||
String comment =
|
||||
locations.stream()
|
||||
.filter(loc -> isEnumValueLocation(loc, enumIndex, finalI))
|
||||
.findFirst()
|
||||
.map(ProtobufUtils::collapseLocationComments)
|
||||
.orElse("");
|
||||
|
||||
valueComments.put(value.getName(), comment);
|
||||
}
|
||||
@ -359,8 +388,8 @@ public class ProtobufField implements ProtobufElement {
|
||||
return valueComments;
|
||||
}
|
||||
|
||||
private boolean isEnumValueLocation(DescriptorProtos.SourceCodeInfo.Location location, int enumIndex,
|
||||
int valueIndex) {
|
||||
private boolean isEnumValueLocation(
|
||||
DescriptorProtos.SourceCodeInfo.Location location, int enumIndex, int valueIndex) {
|
||||
return location.getPathCount() > 3
|
||||
&& location.getPath(0) == DescriptorProtos.FileDescriptorProto.ENUM_TYPE_FIELD_NUMBER
|
||||
&& location.getPath(1) == enumIndex
|
||||
|
||||
@ -1,5 +1,8 @@
|
||||
package datahub.protobuf.visitors.field;
|
||||
|
||||
import static datahub.protobuf.ProtobufUtils.getFieldOptions;
|
||||
import static datahub.protobuf.ProtobufUtils.getMessageOptions;
|
||||
|
||||
import com.linkedin.common.GlobalTags;
|
||||
import com.linkedin.common.GlossaryTermAssociation;
|
||||
import com.linkedin.common.GlossaryTermAssociationArray;
|
||||
@ -15,48 +18,55 @@ import datahub.protobuf.model.ProtobufElement;
|
||||
import datahub.protobuf.model.ProtobufField;
|
||||
import datahub.protobuf.visitors.ProtobufExtensionUtil;
|
||||
import datahub.protobuf.visitors.VisitContext;
|
||||
import org.jgrapht.GraphPath;
|
||||
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static datahub.protobuf.ProtobufUtils.getFieldOptions;
|
||||
import static datahub.protobuf.ProtobufUtils.getMessageOptions;
|
||||
|
||||
import org.jgrapht.GraphPath;
|
||||
|
||||
public class ProtobufExtensionFieldVisitor extends SchemaFieldVisitor {
|
||||
|
||||
@Override
|
||||
public Stream<Pair<SchemaField, Double>> visitField(ProtobufField field, VisitContext context) {
|
||||
boolean isPrimaryKey = getFieldOptions(field.getFieldProto()).stream()
|
||||
.map(Pair::getKey)
|
||||
.anyMatch(fieldDesc -> fieldDesc.getName().matches("(?i).*primary_?key"));
|
||||
boolean isPrimaryKey =
|
||||
getFieldOptions(field.getFieldProto()).stream()
|
||||
.map(Pair::getKey)
|
||||
.anyMatch(fieldDesc -> fieldDesc.getName().matches("(?i).*primary_?key"));
|
||||
|
||||
List<TagAssociation> tags = getTagAssociations(field, context);
|
||||
List<GlossaryTermAssociation> terms = getGlossaryTermAssociations(field, context);
|
||||
|
||||
return context.streamAllPaths(field)
|
||||
.map(path -> Pair.of(createSchemaField(field, context, path, isPrimaryKey, tags, terms),
|
||||
context.calculateSortOrder(path, field)));
|
||||
return context
|
||||
.streamAllPaths(field)
|
||||
.map(
|
||||
path ->
|
||||
Pair.of(
|
||||
createSchemaField(field, context, path, isPrimaryKey, tags, terms),
|
||||
context.calculateSortOrder(path, field)));
|
||||
}
|
||||
|
||||
private SchemaField createSchemaField(ProtobufField field, VisitContext context,
|
||||
GraphPath<ProtobufElement, FieldTypeEdge> path, boolean isPrimaryKey, List<TagAssociation> tags,
|
||||
private SchemaField createSchemaField(
|
||||
ProtobufField field,
|
||||
VisitContext context,
|
||||
GraphPath<ProtobufElement, FieldTypeEdge> path,
|
||||
boolean isPrimaryKey,
|
||||
List<TagAssociation> tags,
|
||||
List<GlossaryTermAssociation> terms) {
|
||||
String description = createFieldDescription(field);
|
||||
|
||||
return new SchemaField().setFieldPath(context.getFieldPath(path))
|
||||
return new SchemaField()
|
||||
.setFieldPath(context.getFieldPath(path))
|
||||
.setNullable(!isPrimaryKey)
|
||||
.setIsPartOfKey(isPrimaryKey)
|
||||
.setDescription(description)
|
||||
.setNativeDataType(field.nativeType())
|
||||
.setType(field.schemaFieldDataType())
|
||||
.setGlobalTags(new GlobalTags().setTags(new TagAssociationArray(tags)))
|
||||
.setGlossaryTerms(new GlossaryTerms().setTerms(new GlossaryTermAssociationArray(terms))
|
||||
.setAuditStamp(context.getAuditStamp()));
|
||||
.setGlossaryTerms(
|
||||
new GlossaryTerms()
|
||||
.setTerms(new GlossaryTermAssociationArray(terms))
|
||||
.setAuditStamp(context.getAuditStamp()));
|
||||
}
|
||||
|
||||
private String createFieldDescription(ProtobufField field) {
|
||||
@ -73,26 +83,33 @@ public class ProtobufExtensionFieldVisitor extends SchemaFieldVisitor {
|
||||
return description.toString();
|
||||
}
|
||||
|
||||
private void appendEnumValues(StringBuilder description, ProtobufField field,
|
||||
Map<String, String> enumValuesWithComments) {
|
||||
enumValuesWithComments.forEach((name, comment) -> {
|
||||
field.getEnumValues().stream().filter(v -> v.getName().equals(name)).findFirst().ifPresent(value -> {
|
||||
description.append(String.format("%d: %s", value.getNumber(), name));
|
||||
if (!comment.isEmpty()) {
|
||||
description.append(" - ").append(comment);
|
||||
}
|
||||
description.append("\n");
|
||||
});
|
||||
});
|
||||
private void appendEnumValues(
|
||||
StringBuilder description, ProtobufField field, Map<String, String> enumValuesWithComments) {
|
||||
enumValuesWithComments.forEach(
|
||||
(name, comment) -> {
|
||||
field.getEnumValues().stream()
|
||||
.filter(v -> v.getName().equals(name))
|
||||
.findFirst()
|
||||
.ifPresent(
|
||||
value -> {
|
||||
description.append(String.format("%d: %s", value.getNumber(), name));
|
||||
if (!comment.isEmpty()) {
|
||||
description.append(" - ").append(comment);
|
||||
}
|
||||
description.append("\n");
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private List<TagAssociation> getTagAssociations(ProtobufField field, VisitContext context) {
|
||||
Stream<TagAssociation> fieldTags =
|
||||
ProtobufExtensionUtil.extractTagPropertiesFromOptions(getFieldOptions(field.getFieldProto()),
|
||||
context.getGraph().getRegistry()).map(tag -> new TagAssociation().setTag(new TagUrn(tag.getName())));
|
||||
ProtobufExtensionUtil.extractTagPropertiesFromOptions(
|
||||
getFieldOptions(field.getFieldProto()), context.getGraph().getRegistry())
|
||||
.map(tag -> new TagAssociation().setTag(new TagUrn(tag.getName())));
|
||||
|
||||
Stream<TagAssociation> promotedTags =
|
||||
promotedTags(field, context).map(tag -> new TagAssociation().setTag(new TagUrn(tag.getName())));
|
||||
promotedTags(field, context)
|
||||
.map(tag -> new TagAssociation().setTag(new TagUrn(tag.getName())));
|
||||
|
||||
return Stream.concat(fieldTags, promotedTags)
|
||||
.distinct()
|
||||
@ -100,10 +117,11 @@ public class ProtobufExtensionFieldVisitor extends SchemaFieldVisitor {
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private List<GlossaryTermAssociation> getGlossaryTermAssociations(ProtobufField field, VisitContext context) {
|
||||
private List<GlossaryTermAssociation> getGlossaryTermAssociations(
|
||||
ProtobufField field, VisitContext context) {
|
||||
Stream<GlossaryTermAssociation> fieldTerms =
|
||||
ProtobufExtensionUtil.extractTermAssociationsFromOptions(getFieldOptions(field.getFieldProto()),
|
||||
context.getGraph().getRegistry());
|
||||
ProtobufExtensionUtil.extractTermAssociationsFromOptions(
|
||||
getFieldOptions(field.getFieldProto()), context.getGraph().getRegistry());
|
||||
|
||||
Stream<GlossaryTermAssociation> promotedTerms = promotedTerms(field, context);
|
||||
|
||||
@ -120,11 +138,12 @@ public class ProtobufExtensionFieldVisitor extends SchemaFieldVisitor {
|
||||
*/
|
||||
private Stream<TagProperties> promotedTags(ProtobufField field, VisitContext context) {
|
||||
if (field.isMessage()) {
|
||||
return context.getGraph()
|
||||
.outgoingEdgesOf(field)
|
||||
.stream()
|
||||
.flatMap(e -> ProtobufExtensionUtil.extractTagPropertiesFromOptions(
|
||||
getMessageOptions(e.getEdgeTarget().messageProto()), context.getGraph().getRegistry()))
|
||||
return context.getGraph().outgoingEdgesOf(field).stream()
|
||||
.flatMap(
|
||||
e ->
|
||||
ProtobufExtensionUtil.extractTagPropertiesFromOptions(
|
||||
getMessageOptions(e.getEdgeTarget().messageProto()),
|
||||
context.getGraph().getRegistry()))
|
||||
.distinct();
|
||||
} else {
|
||||
return Stream.of();
|
||||
@ -138,11 +157,12 @@ public class ProtobufExtensionFieldVisitor extends SchemaFieldVisitor {
|
||||
*/
|
||||
private Stream<GlossaryTermAssociation> promotedTerms(ProtobufField field, VisitContext context) {
|
||||
if (field.isMessage()) {
|
||||
return context.getGraph()
|
||||
.outgoingEdgesOf(field)
|
||||
.stream()
|
||||
.flatMap(e -> ProtobufExtensionUtil.extractTermAssociationsFromOptions(
|
||||
getMessageOptions(e.getEdgeTarget().messageProto()), context.getGraph().getRegistry()))
|
||||
return context.getGraph().outgoingEdgesOf(field).stream()
|
||||
.flatMap(
|
||||
e ->
|
||||
ProtobufExtensionUtil.extractTermAssociationsFromOptions(
|
||||
getMessageOptions(e.getEdgeTarget().messageProto()),
|
||||
context.getGraph().getRegistry()))
|
||||
.distinct();
|
||||
} else {
|
||||
return Stream.of();
|
||||
|
||||
@ -48,15 +48,16 @@ public class ProtobufUtilsTest {
|
||||
|
||||
@Test
|
||||
public void testCollapseLocationCommentsWithUTF8() {
|
||||
DescriptorProtos.SourceCodeInfo.Location location = DescriptorProtos.SourceCodeInfo.Location.newBuilder()
|
||||
.addAllLeadingDetachedComments(Arrays.asList("/* Emoji 😊 */", "/* Accented é */"))
|
||||
.setLeadingComments("/* Chinese 你好 */\n// Russian Привет")
|
||||
.setTrailingComments("// Korean 안녕")
|
||||
.build();
|
||||
DescriptorProtos.SourceCodeInfo.Location location =
|
||||
DescriptorProtos.SourceCodeInfo.Location.newBuilder()
|
||||
.addAllLeadingDetachedComments(Arrays.asList("/* Emoji 😊 */", "/* Accented é */"))
|
||||
.setLeadingComments("/* Chinese 你好 */\n// Russian Привет")
|
||||
.setTrailingComments("// Korean 안녕")
|
||||
.build();
|
||||
|
||||
String actual = ProtobufUtils.collapseLocationComments(location);
|
||||
String expected = "Emoji 😊 */\nAccented é */\nChinese 你好 */\nRussian Привет\nKorean 안녕";
|
||||
|
||||
assertEquals(expected, actual);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -361,14 +361,22 @@ public class ProtobufFieldTest {
|
||||
ProtobufDataset test = getTestProtobufDataset("extended_protobuf", "messageE");
|
||||
SchemaMetadata testMetadata = test.getSchemaMetadata();
|
||||
|
||||
SchemaField timestampField = testMetadata.getFields()
|
||||
.stream()
|
||||
.filter(v -> v.getFieldPath()
|
||||
.equals("[version=2.0].[type=extended_protobuf_TimestampUnitMessage].[type=enum].timestamp_unit_type"))
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
SchemaField timestampField =
|
||||
testMetadata.getFields().stream()
|
||||
.filter(
|
||||
v ->
|
||||
v.getFieldPath()
|
||||
.equals(
|
||||
"[version=2.0].[type=extended_protobuf_TimestampUnitMessage].[type=enum].timestamp_unit_type"))
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
|
||||
assertEquals("timestamp unit\n" + "\n" + "0: MILLISECOND - 10^-3 seconds\n" + "1: MICROSECOND - 10^-6 seconds\n"
|
||||
+ "2: NANOSECOND - 10^-9 seconds\n", timestampField.getDescription());
|
||||
assertEquals(
|
||||
"timestamp unit\n"
|
||||
+ "\n"
|
||||
+ "0: MILLISECOND - 10^-3 seconds\n"
|
||||
+ "1: MICROSECOND - 10^-6 seconds\n"
|
||||
+ "2: NANOSECOND - 10^-9 seconds\n",
|
||||
timestampField.getDescription());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -52,7 +52,7 @@ import org.opensearch.index.query.functionscore.ScoreFunctionBuilders;
|
||||
|
||||
@Slf4j
|
||||
public class SearchQueryBuilder {
|
||||
public static final String STRUCTURED_QUERY_PREFIX = "\\\\/q ";
|
||||
public static final String STRUCTURED_QUERY_PREFIX = "\\/q ";
|
||||
private final ExactMatchConfiguration exactMatchConfiguration;
|
||||
private final PartialConfiguration partialConfiguration;
|
||||
private final WordGramConfiguration wordGramConfiguration;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user