mirror of
				https://github.com/datahub-project/datahub.git
				synced 2025-11-04 04:39:10 +00:00 
			
		
		
		
	
							parent
							
								
									d788cd753c
								
							
						
					
					
						commit
						d795a5357d
					
				@ -22,13 +22,13 @@ public class ProtobufUtils {
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
  public static String collapseLocationComments(DescriptorProtos.SourceCodeInfo.Location location) {
 | 
					  public static String collapseLocationComments(DescriptorProtos.SourceCodeInfo.Location location) {
 | 
				
			||||||
    return Stream.concat(
 | 
					    return Stream.concat(
 | 
				
			||||||
                location.getLeadingDetachedCommentsList().stream(),
 | 
					            location.getLeadingDetachedCommentsList().stream(),
 | 
				
			||||||
                Stream.of(location.getLeadingComments(), location.getTrailingComments()))
 | 
					            Stream.of(location.getLeadingComments(), location.getTrailingComments()))
 | 
				
			||||||
            .filter(Objects::nonNull)
 | 
					        .filter(Objects::nonNull)
 | 
				
			||||||
            .flatMap(line -> Arrays.stream(line.split("\n")))
 | 
					        .flatMap(line -> Arrays.stream(line.split("\n")))
 | 
				
			||||||
            .map(line -> line.replaceFirst("^[*/ ]+", ""))
 | 
					        .map(line -> line.replaceFirst("^[*/ ]+", ""))
 | 
				
			||||||
            .collect(Collectors.joining("\n"))
 | 
					        .collect(Collectors.joining("\n"))
 | 
				
			||||||
            .trim();
 | 
					        .trim();
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  /*
 | 
					  /*
 | 
				
			||||||
 | 
				
			|||||||
@ -19,10 +19,6 @@ import com.linkedin.schema.StringType;
 | 
				
			|||||||
import datahub.protobuf.ProtobufUtils;
 | 
					import datahub.protobuf.ProtobufUtils;
 | 
				
			||||||
import datahub.protobuf.visitors.ProtobufModelVisitor;
 | 
					import datahub.protobuf.visitors.ProtobufModelVisitor;
 | 
				
			||||||
import datahub.protobuf.visitors.VisitContext;
 | 
					import datahub.protobuf.visitors.VisitContext;
 | 
				
			||||||
import lombok.AllArgsConstructor;
 | 
					 | 
				
			||||||
import lombok.Builder;
 | 
					 | 
				
			||||||
import lombok.Getter;
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
import java.util.ArrayList;
 | 
					import java.util.ArrayList;
 | 
				
			||||||
import java.util.Collections;
 | 
					import java.util.Collections;
 | 
				
			||||||
import java.util.LinkedHashMap;
 | 
					import java.util.LinkedHashMap;
 | 
				
			||||||
@ -31,7 +27,9 @@ import java.util.Map;
 | 
				
			|||||||
import java.util.Optional;
 | 
					import java.util.Optional;
 | 
				
			||||||
import java.util.stream.Collectors;
 | 
					import java.util.stream.Collectors;
 | 
				
			||||||
import java.util.stream.Stream;
 | 
					import java.util.stream.Stream;
 | 
				
			||||||
 | 
					import lombok.AllArgsConstructor;
 | 
				
			||||||
 | 
					import lombok.Builder;
 | 
				
			||||||
 | 
					import lombok.Getter;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@Builder(toBuilder = true)
 | 
					@Builder(toBuilder = true)
 | 
				
			||||||
@Getter
 | 
					@Getter
 | 
				
			||||||
@ -87,72 +85,78 @@ public class ProtobufField implements ProtobufElement {
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
  @Override
 | 
					  @Override
 | 
				
			||||||
  public String nativeType() {
 | 
					  public String nativeType() {
 | 
				
			||||||
    return Optional.ofNullable(nativeType).orElseGet(() -> {
 | 
					    return Optional.ofNullable(nativeType)
 | 
				
			||||||
      if (fieldProto.getTypeName().isEmpty()) {
 | 
					        .orElseGet(
 | 
				
			||||||
        return fieldProto.getType().name().split("_")[1].toLowerCase();
 | 
					            () -> {
 | 
				
			||||||
      } else {
 | 
					              if (fieldProto.getTypeName().isEmpty()) {
 | 
				
			||||||
        return fieldProto.getTypeName().replaceFirst("^[.]", "");
 | 
					                return fieldProto.getType().name().split("_")[1].toLowerCase();
 | 
				
			||||||
      }
 | 
					              } else {
 | 
				
			||||||
    });
 | 
					                return fieldProto.getTypeName().replaceFirst("^[.]", "");
 | 
				
			||||||
 | 
					              }
 | 
				
			||||||
 | 
					            });
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  @Override
 | 
					  @Override
 | 
				
			||||||
  public String fieldPathType() {
 | 
					  public String fieldPathType() {
 | 
				
			||||||
    return Optional.ofNullable(fieldPathType).orElseGet(() -> {
 | 
					    return Optional.ofNullable(fieldPathType)
 | 
				
			||||||
      final String pathType;
 | 
					        .orElseGet(
 | 
				
			||||||
 | 
					            () -> {
 | 
				
			||||||
 | 
					              final String pathType;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      switch (fieldProto.getType()) {
 | 
					              switch (fieldProto.getType()) {
 | 
				
			||||||
        case TYPE_DOUBLE:
 | 
					                case TYPE_DOUBLE:
 | 
				
			||||||
          pathType = "double";
 | 
					                  pathType = "double";
 | 
				
			||||||
          break;
 | 
					                  break;
 | 
				
			||||||
        case TYPE_FLOAT:
 | 
					                case TYPE_FLOAT:
 | 
				
			||||||
          pathType = "float";
 | 
					                  pathType = "float";
 | 
				
			||||||
          break;
 | 
					                  break;
 | 
				
			||||||
        case TYPE_SFIXED64:
 | 
					                case TYPE_SFIXED64:
 | 
				
			||||||
        case TYPE_FIXED64:
 | 
					                case TYPE_FIXED64:
 | 
				
			||||||
        case TYPE_UINT64:
 | 
					                case TYPE_UINT64:
 | 
				
			||||||
        case TYPE_INT64:
 | 
					                case TYPE_INT64:
 | 
				
			||||||
        case TYPE_SINT64:
 | 
					                case TYPE_SINT64:
 | 
				
			||||||
          pathType = "long";
 | 
					                  pathType = "long";
 | 
				
			||||||
          break;
 | 
					                  break;
 | 
				
			||||||
        case TYPE_FIXED32:
 | 
					                case TYPE_FIXED32:
 | 
				
			||||||
        case TYPE_SFIXED32:
 | 
					                case TYPE_SFIXED32:
 | 
				
			||||||
        case TYPE_INT32:
 | 
					                case TYPE_INT32:
 | 
				
			||||||
        case TYPE_UINT32:
 | 
					                case TYPE_UINT32:
 | 
				
			||||||
        case TYPE_SINT32:
 | 
					                case TYPE_SINT32:
 | 
				
			||||||
          pathType = "int";
 | 
					                  pathType = "int";
 | 
				
			||||||
          break;
 | 
					                  break;
 | 
				
			||||||
        case TYPE_BYTES:
 | 
					                case TYPE_BYTES:
 | 
				
			||||||
          pathType = "bytes";
 | 
					                  pathType = "bytes";
 | 
				
			||||||
          break;
 | 
					                  break;
 | 
				
			||||||
        case TYPE_ENUM:
 | 
					                case TYPE_ENUM:
 | 
				
			||||||
          pathType = "enum";
 | 
					                  pathType = "enum";
 | 
				
			||||||
          break;
 | 
					                  break;
 | 
				
			||||||
        case TYPE_BOOL:
 | 
					                case TYPE_BOOL:
 | 
				
			||||||
          pathType = "boolean";
 | 
					                  pathType = "boolean";
 | 
				
			||||||
          break;
 | 
					                  break;
 | 
				
			||||||
        case TYPE_STRING:
 | 
					                case TYPE_STRING:
 | 
				
			||||||
          pathType = "string";
 | 
					                  pathType = "string";
 | 
				
			||||||
          break;
 | 
					                  break;
 | 
				
			||||||
        case TYPE_GROUP:
 | 
					                case TYPE_GROUP:
 | 
				
			||||||
        case TYPE_MESSAGE:
 | 
					                case TYPE_MESSAGE:
 | 
				
			||||||
          pathType = nativeType().replace(".", "_");
 | 
					                  pathType = nativeType().replace(".", "_");
 | 
				
			||||||
          break;
 | 
					                  break;
 | 
				
			||||||
        default:
 | 
					                default:
 | 
				
			||||||
          throw new IllegalStateException(
 | 
					                  throw new IllegalStateException(
 | 
				
			||||||
              String.format("Unexpected FieldDescriptorProto => FieldPathType %s", fieldProto.getType()));
 | 
					                      String.format(
 | 
				
			||||||
      }
 | 
					                          "Unexpected FieldDescriptorProto => FieldPathType %s",
 | 
				
			||||||
 | 
					                          fieldProto.getType()));
 | 
				
			||||||
 | 
					              }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      StringArray fieldPath = new StringArray();
 | 
					              StringArray fieldPath = new StringArray();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      if (schemaFieldDataType().getType().isArrayType()) {
 | 
					              if (schemaFieldDataType().getType().isArrayType()) {
 | 
				
			||||||
        fieldPath.add("[type=array]");
 | 
					                fieldPath.add("[type=array]");
 | 
				
			||||||
      }
 | 
					              }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      fieldPath.add(String.format("[type=%s]", pathType));
 | 
					              fieldPath.add(String.format("[type=%s]", pathType));
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      return String.join(".", fieldPath);
 | 
					              return String.join(".", fieldPath);
 | 
				
			||||||
    });
 | 
					            });
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  public boolean isMessage() {
 | 
					  public boolean isMessage() {
 | 
				
			||||||
@ -165,92 +169,110 @@ public class ProtobufField implements ProtobufElement {
 | 
				
			|||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  public SchemaFieldDataType schemaFieldDataType() throws IllegalStateException {
 | 
					  public SchemaFieldDataType schemaFieldDataType() throws IllegalStateException {
 | 
				
			||||||
    return Optional.ofNullable(schemaFieldDataType).orElseGet(() -> {
 | 
					    return Optional.ofNullable(schemaFieldDataType)
 | 
				
			||||||
      final SchemaFieldDataType.Type fieldType;
 | 
					        .orElseGet(
 | 
				
			||||||
 | 
					            () -> {
 | 
				
			||||||
 | 
					              final SchemaFieldDataType.Type fieldType;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      switch (fieldProto.getType()) {
 | 
					              switch (fieldProto.getType()) {
 | 
				
			||||||
        case TYPE_DOUBLE:
 | 
					                case TYPE_DOUBLE:
 | 
				
			||||||
        case TYPE_FLOAT:
 | 
					                case TYPE_FLOAT:
 | 
				
			||||||
        case TYPE_INT64:
 | 
					                case TYPE_INT64:
 | 
				
			||||||
        case TYPE_UINT64:
 | 
					                case TYPE_UINT64:
 | 
				
			||||||
        case TYPE_INT32:
 | 
					                case TYPE_INT32:
 | 
				
			||||||
        case TYPE_UINT32:
 | 
					                case TYPE_UINT32:
 | 
				
			||||||
        case TYPE_SINT32:
 | 
					                case TYPE_SINT32:
 | 
				
			||||||
        case TYPE_SINT64:
 | 
					                case TYPE_SINT64:
 | 
				
			||||||
          fieldType = SchemaFieldDataType.Type.create(new NumberType());
 | 
					                  fieldType = SchemaFieldDataType.Type.create(new NumberType());
 | 
				
			||||||
          break;
 | 
					                  break;
 | 
				
			||||||
        case TYPE_GROUP:
 | 
					                case TYPE_GROUP:
 | 
				
			||||||
        case TYPE_MESSAGE:
 | 
					                case TYPE_MESSAGE:
 | 
				
			||||||
          fieldType = SchemaFieldDataType.Type.create(new RecordType());
 | 
					                  fieldType = SchemaFieldDataType.Type.create(new RecordType());
 | 
				
			||||||
          break;
 | 
					                  break;
 | 
				
			||||||
        case TYPE_BYTES:
 | 
					                case TYPE_BYTES:
 | 
				
			||||||
          fieldType = SchemaFieldDataType.Type.create(new BytesType());
 | 
					                  fieldType = SchemaFieldDataType.Type.create(new BytesType());
 | 
				
			||||||
          break;
 | 
					                  break;
 | 
				
			||||||
        case TYPE_ENUM:
 | 
					                case TYPE_ENUM:
 | 
				
			||||||
          fieldType = SchemaFieldDataType.Type.create(new EnumType());
 | 
					                  fieldType = SchemaFieldDataType.Type.create(new EnumType());
 | 
				
			||||||
          break;
 | 
					                  break;
 | 
				
			||||||
        case TYPE_BOOL:
 | 
					                case TYPE_BOOL:
 | 
				
			||||||
          fieldType = SchemaFieldDataType.Type.create(new BooleanType());
 | 
					                  fieldType = SchemaFieldDataType.Type.create(new BooleanType());
 | 
				
			||||||
          break;
 | 
					                  break;
 | 
				
			||||||
        case TYPE_STRING:
 | 
					                case TYPE_STRING:
 | 
				
			||||||
          fieldType = SchemaFieldDataType.Type.create(new StringType());
 | 
					                  fieldType = SchemaFieldDataType.Type.create(new StringType());
 | 
				
			||||||
          break;
 | 
					                  break;
 | 
				
			||||||
        case TYPE_FIXED64:
 | 
					                case TYPE_FIXED64:
 | 
				
			||||||
        case TYPE_FIXED32:
 | 
					                case TYPE_FIXED32:
 | 
				
			||||||
        case TYPE_SFIXED32:
 | 
					                case TYPE_SFIXED32:
 | 
				
			||||||
        case TYPE_SFIXED64:
 | 
					                case TYPE_SFIXED64:
 | 
				
			||||||
          fieldType = SchemaFieldDataType.Type.create(new FixedType());
 | 
					                  fieldType = SchemaFieldDataType.Type.create(new FixedType());
 | 
				
			||||||
          break;
 | 
					                  break;
 | 
				
			||||||
        default:
 | 
					                default:
 | 
				
			||||||
          throw new IllegalStateException(
 | 
					                  throw new IllegalStateException(
 | 
				
			||||||
              String.format("Unexpected FieldDescriptorProto => SchemaFieldDataType: %s", fieldProto.getType()));
 | 
					                      String.format(
 | 
				
			||||||
      }
 | 
					                          "Unexpected FieldDescriptorProto => SchemaFieldDataType: %s",
 | 
				
			||||||
 | 
					                          fieldProto.getType()));
 | 
				
			||||||
 | 
					              }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      if (fieldProto.getLabel().equals(FieldDescriptorProto.Label.LABEL_REPEATED)) {
 | 
					              if (fieldProto.getLabel().equals(FieldDescriptorProto.Label.LABEL_REPEATED)) {
 | 
				
			||||||
        return new SchemaFieldDataType().setType(
 | 
					                return new SchemaFieldDataType()
 | 
				
			||||||
            SchemaFieldDataType.Type.create(new ArrayType().setNestedType(new StringArray())));
 | 
					                    .setType(
 | 
				
			||||||
      }
 | 
					                        SchemaFieldDataType.Type.create(
 | 
				
			||||||
 | 
					                            new ArrayType().setNestedType(new StringArray())));
 | 
				
			||||||
 | 
					              }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      return new SchemaFieldDataType().setType(fieldType);
 | 
					              return new SchemaFieldDataType().setType(fieldType);
 | 
				
			||||||
    });
 | 
					            });
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  @Override
 | 
					  @Override
 | 
				
			||||||
  public Stream<SourceCodeInfo.Location> messageLocations() {
 | 
					  public Stream<SourceCodeInfo.Location> messageLocations() {
 | 
				
			||||||
    List<SourceCodeInfo.Location> fileLocations = fileProto().getSourceCodeInfo().getLocationList();
 | 
					    List<SourceCodeInfo.Location> fileLocations = fileProto().getSourceCodeInfo().getLocationList();
 | 
				
			||||||
    return fileLocations.stream()
 | 
					    return fileLocations.stream()
 | 
				
			||||||
        .filter(loc -> loc.getPathCount() > 1 && loc.getPath(0) == FileDescriptorProto.MESSAGE_TYPE_FIELD_NUMBER);
 | 
					        .filter(
 | 
				
			||||||
 | 
					            loc ->
 | 
				
			||||||
 | 
					                loc.getPathCount() > 1
 | 
				
			||||||
 | 
					                    && loc.getPath(0) == FileDescriptorProto.MESSAGE_TYPE_FIELD_NUMBER);
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  @Override
 | 
					  @Override
 | 
				
			||||||
  public String comment() {
 | 
					  public String comment() {
 | 
				
			||||||
    return messageLocations().filter(location -> location.getPathCount() > 3)
 | 
					    return messageLocations()
 | 
				
			||||||
        .filter(location -> !ProtobufUtils.collapseLocationComments(location).isEmpty() && !isEnumType(
 | 
					        .filter(location -> location.getPathCount() > 3)
 | 
				
			||||||
            location.getPathList()))
 | 
					        .filter(
 | 
				
			||||||
        .filter(location -> {
 | 
					            location ->
 | 
				
			||||||
          List<Integer> pathList = location.getPathList();
 | 
					                !ProtobufUtils.collapseLocationComments(location).isEmpty()
 | 
				
			||||||
          DescriptorProto messageType = fileProto().getMessageType(pathList.get(1));
 | 
					                    && !isEnumType(location.getPathList()))
 | 
				
			||||||
 | 
					        .filter(
 | 
				
			||||||
 | 
					            location -> {
 | 
				
			||||||
 | 
					              List<Integer> pathList = location.getPathList();
 | 
				
			||||||
 | 
					              DescriptorProto messageType = fileProto().getMessageType(pathList.get(1));
 | 
				
			||||||
 | 
					
 | 
				
			||||||
          if (!isNestedType && location.getPath(2) == DescriptorProto.FIELD_FIELD_NUMBER
 | 
					              if (!isNestedType
 | 
				
			||||||
              && fieldProto == messageType.getField(location.getPath(3))) {
 | 
					                  && location.getPath(2) == DescriptorProto.FIELD_FIELD_NUMBER
 | 
				
			||||||
            return true;
 | 
					                  && fieldProto == messageType.getField(location.getPath(3))) {
 | 
				
			||||||
          } else if (isNestedType && location.getPath(2) == DescriptorProto.NESTED_TYPE_FIELD_NUMBER
 | 
					                return true;
 | 
				
			||||||
              && fieldProto == getNestedTypeFields(pathList, messageType)) {
 | 
					              } else if (isNestedType
 | 
				
			||||||
            return true;
 | 
					                  && location.getPath(2) == DescriptorProto.NESTED_TYPE_FIELD_NUMBER
 | 
				
			||||||
          }
 | 
					                  && fieldProto == getNestedTypeFields(pathList, messageType)) {
 | 
				
			||||||
          return false;
 | 
					                return true;
 | 
				
			||||||
        })
 | 
					              }
 | 
				
			||||||
 | 
					              return false;
 | 
				
			||||||
 | 
					            })
 | 
				
			||||||
        .map(ProtobufUtils::collapseLocationComments)
 | 
					        .map(ProtobufUtils::collapseLocationComments)
 | 
				
			||||||
        .collect(Collectors.joining("\n"))
 | 
					        .collect(Collectors.joining("\n"))
 | 
				
			||||||
        .trim();
 | 
					        .trim();
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  private FieldDescriptorProto getNestedTypeFields(List<Integer> pathList, DescriptorProto messageType) {
 | 
					  private FieldDescriptorProto getNestedTypeFields(
 | 
				
			||||||
 | 
					      List<Integer> pathList, DescriptorProto messageType) {
 | 
				
			||||||
    int pathSize = pathList.size();
 | 
					    int pathSize = pathList.size();
 | 
				
			||||||
    List<Integer> nestedValues = new ArrayList<>(pathSize);
 | 
					    List<Integer> nestedValues = new ArrayList<>(pathSize);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    for (int index = 0; index < pathSize; index++) {
 | 
					    for (int index = 0; index < pathSize; index++) {
 | 
				
			||||||
      if (index > 1 && index % 2 == 0 && pathList.get(index) == DescriptorProto.NESTED_TYPE_FIELD_NUMBER) {
 | 
					      if (index > 1
 | 
				
			||||||
 | 
					          && index % 2 == 0
 | 
				
			||||||
 | 
					          && pathList.get(index) == DescriptorProto.NESTED_TYPE_FIELD_NUMBER) {
 | 
				
			||||||
        nestedValues.add(pathList.get(index + 1));
 | 
					        nestedValues.add(pathList.get(index + 1));
 | 
				
			||||||
      }
 | 
					      }
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
@ -260,7 +282,9 @@ public class ProtobufField implements ProtobufElement {
 | 
				
			|||||||
    }
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    int fieldIndex = pathList.get(pathList.size() - 1);
 | 
					    int fieldIndex = pathList.get(pathList.size() - 1);
 | 
				
			||||||
    if (isFieldPath(pathList) && pathSize % 2 == 0 && fieldIndex < messageType.getFieldList().size()) {
 | 
					    if (isFieldPath(pathList)
 | 
				
			||||||
 | 
					        && pathSize % 2 == 0
 | 
				
			||||||
 | 
					        && fieldIndex < messageType.getFieldList().size()) {
 | 
				
			||||||
      return messageType.getField(fieldIndex);
 | 
					      return messageType.getField(fieldIndex);
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@ -273,7 +297,9 @@ public class ProtobufField implements ProtobufElement {
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
  private boolean isEnumType(List<Integer> pathList) {
 | 
					  private boolean isEnumType(List<Integer> pathList) {
 | 
				
			||||||
    for (int index = 0; index < pathList.size(); index++) {
 | 
					    for (int index = 0; index < pathList.size(); index++) {
 | 
				
			||||||
      if (index > 1 && index % 2 == 0 && pathList.get(index) == DescriptorProto.ENUM_TYPE_FIELD_NUMBER) {
 | 
					      if (index > 1
 | 
				
			||||||
 | 
					          && index % 2 == 0
 | 
				
			||||||
 | 
					          && pathList.get(index) == DescriptorProto.ENUM_TYPE_FIELD_NUMBER) {
 | 
				
			||||||
        return true;
 | 
					        return true;
 | 
				
			||||||
      }
 | 
					      }
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
@ -327,7 +353,9 @@ public class ProtobufField implements ProtobufElement {
 | 
				
			|||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  public List<DescriptorProtos.EnumValueDescriptorProto> getEnumValues() {
 | 
					  public List<DescriptorProtos.EnumValueDescriptorProto> getEnumValues() {
 | 
				
			||||||
    return getEnumDescriptor().map(DescriptorProtos.EnumDescriptorProto::getValueList).orElse(Collections.emptyList());
 | 
					    return getEnumDescriptor()
 | 
				
			||||||
 | 
					        .map(DescriptorProtos.EnumDescriptorProto::getValueList)
 | 
				
			||||||
 | 
					        .orElse(Collections.emptyList());
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  public Map<String, String> getEnumValuesWithComments() {
 | 
					  public Map<String, String> getEnumValuesWithComments() {
 | 
				
			||||||
@ -347,11 +375,12 @@ public class ProtobufField implements ProtobufElement {
 | 
				
			|||||||
    for (int i = 0; i < values.size(); i++) {
 | 
					    for (int i = 0; i < values.size(); i++) {
 | 
				
			||||||
      DescriptorProtos.EnumValueDescriptorProto value = values.get(i);
 | 
					      DescriptorProtos.EnumValueDescriptorProto value = values.get(i);
 | 
				
			||||||
      int finalI = i;
 | 
					      int finalI = i;
 | 
				
			||||||
      String comment = locations.stream()
 | 
					      String comment =
 | 
				
			||||||
          .filter(loc -> isEnumValueLocation(loc, enumIndex, finalI))
 | 
					          locations.stream()
 | 
				
			||||||
          .findFirst()
 | 
					              .filter(loc -> isEnumValueLocation(loc, enumIndex, finalI))
 | 
				
			||||||
          .map(ProtobufUtils::collapseLocationComments)
 | 
					              .findFirst()
 | 
				
			||||||
          .orElse("");
 | 
					              .map(ProtobufUtils::collapseLocationComments)
 | 
				
			||||||
 | 
					              .orElse("");
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      valueComments.put(value.getName(), comment);
 | 
					      valueComments.put(value.getName(), comment);
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
@ -359,8 +388,8 @@ public class ProtobufField implements ProtobufElement {
 | 
				
			|||||||
    return valueComments;
 | 
					    return valueComments;
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  private boolean isEnumValueLocation(DescriptorProtos.SourceCodeInfo.Location location, int enumIndex,
 | 
					  private boolean isEnumValueLocation(
 | 
				
			||||||
      int valueIndex) {
 | 
					      DescriptorProtos.SourceCodeInfo.Location location, int enumIndex, int valueIndex) {
 | 
				
			||||||
    return location.getPathCount() > 3
 | 
					    return location.getPathCount() > 3
 | 
				
			||||||
        && location.getPath(0) == DescriptorProtos.FileDescriptorProto.ENUM_TYPE_FIELD_NUMBER
 | 
					        && location.getPath(0) == DescriptorProtos.FileDescriptorProto.ENUM_TYPE_FIELD_NUMBER
 | 
				
			||||||
        && location.getPath(1) == enumIndex
 | 
					        && location.getPath(1) == enumIndex
 | 
				
			||||||
 | 
				
			|||||||
@ -1,5 +1,8 @@
 | 
				
			|||||||
package datahub.protobuf.visitors.field;
 | 
					package datahub.protobuf.visitors.field;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import static datahub.protobuf.ProtobufUtils.getFieldOptions;
 | 
				
			||||||
 | 
					import static datahub.protobuf.ProtobufUtils.getMessageOptions;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
import com.linkedin.common.GlobalTags;
 | 
					import com.linkedin.common.GlobalTags;
 | 
				
			||||||
import com.linkedin.common.GlossaryTermAssociation;
 | 
					import com.linkedin.common.GlossaryTermAssociation;
 | 
				
			||||||
import com.linkedin.common.GlossaryTermAssociationArray;
 | 
					import com.linkedin.common.GlossaryTermAssociationArray;
 | 
				
			||||||
@ -15,48 +18,55 @@ import datahub.protobuf.model.ProtobufElement;
 | 
				
			|||||||
import datahub.protobuf.model.ProtobufField;
 | 
					import datahub.protobuf.model.ProtobufField;
 | 
				
			||||||
import datahub.protobuf.visitors.ProtobufExtensionUtil;
 | 
					import datahub.protobuf.visitors.ProtobufExtensionUtil;
 | 
				
			||||||
import datahub.protobuf.visitors.VisitContext;
 | 
					import datahub.protobuf.visitors.VisitContext;
 | 
				
			||||||
import org.jgrapht.GraphPath;
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
import java.util.Comparator;
 | 
					import java.util.Comparator;
 | 
				
			||||||
import java.util.List;
 | 
					import java.util.List;
 | 
				
			||||||
import java.util.Map;
 | 
					import java.util.Map;
 | 
				
			||||||
import java.util.stream.Collectors;
 | 
					import java.util.stream.Collectors;
 | 
				
			||||||
import java.util.stream.Stream;
 | 
					import java.util.stream.Stream;
 | 
				
			||||||
 | 
					import org.jgrapht.GraphPath;
 | 
				
			||||||
import static datahub.protobuf.ProtobufUtils.getFieldOptions;
 | 
					 | 
				
			||||||
import static datahub.protobuf.ProtobufUtils.getMessageOptions;
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
 | 
					
 | 
				
			||||||
public class ProtobufExtensionFieldVisitor extends SchemaFieldVisitor {
 | 
					public class ProtobufExtensionFieldVisitor extends SchemaFieldVisitor {
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  @Override
 | 
					  @Override
 | 
				
			||||||
  public Stream<Pair<SchemaField, Double>> visitField(ProtobufField field, VisitContext context) {
 | 
					  public Stream<Pair<SchemaField, Double>> visitField(ProtobufField field, VisitContext context) {
 | 
				
			||||||
    boolean isPrimaryKey = getFieldOptions(field.getFieldProto()).stream()
 | 
					    boolean isPrimaryKey =
 | 
				
			||||||
        .map(Pair::getKey)
 | 
					        getFieldOptions(field.getFieldProto()).stream()
 | 
				
			||||||
        .anyMatch(fieldDesc -> fieldDesc.getName().matches("(?i).*primary_?key"));
 | 
					            .map(Pair::getKey)
 | 
				
			||||||
 | 
					            .anyMatch(fieldDesc -> fieldDesc.getName().matches("(?i).*primary_?key"));
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    List<TagAssociation> tags = getTagAssociations(field, context);
 | 
					    List<TagAssociation> tags = getTagAssociations(field, context);
 | 
				
			||||||
    List<GlossaryTermAssociation> terms = getGlossaryTermAssociations(field, context);
 | 
					    List<GlossaryTermAssociation> terms = getGlossaryTermAssociations(field, context);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    return context.streamAllPaths(field)
 | 
					    return context
 | 
				
			||||||
        .map(path -> Pair.of(createSchemaField(field, context, path, isPrimaryKey, tags, terms),
 | 
					        .streamAllPaths(field)
 | 
				
			||||||
            context.calculateSortOrder(path, field)));
 | 
					        .map(
 | 
				
			||||||
 | 
					            path ->
 | 
				
			||||||
 | 
					                Pair.of(
 | 
				
			||||||
 | 
					                    createSchemaField(field, context, path, isPrimaryKey, tags, terms),
 | 
				
			||||||
 | 
					                    context.calculateSortOrder(path, field)));
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  private SchemaField createSchemaField(ProtobufField field, VisitContext context,
 | 
					  private SchemaField createSchemaField(
 | 
				
			||||||
      GraphPath<ProtobufElement, FieldTypeEdge> path, boolean isPrimaryKey, List<TagAssociation> tags,
 | 
					      ProtobufField field,
 | 
				
			||||||
 | 
					      VisitContext context,
 | 
				
			||||||
 | 
					      GraphPath<ProtobufElement, FieldTypeEdge> path,
 | 
				
			||||||
 | 
					      boolean isPrimaryKey,
 | 
				
			||||||
 | 
					      List<TagAssociation> tags,
 | 
				
			||||||
      List<GlossaryTermAssociation> terms) {
 | 
					      List<GlossaryTermAssociation> terms) {
 | 
				
			||||||
    String description = createFieldDescription(field);
 | 
					    String description = createFieldDescription(field);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    return new SchemaField().setFieldPath(context.getFieldPath(path))
 | 
					    return new SchemaField()
 | 
				
			||||||
 | 
					        .setFieldPath(context.getFieldPath(path))
 | 
				
			||||||
        .setNullable(!isPrimaryKey)
 | 
					        .setNullable(!isPrimaryKey)
 | 
				
			||||||
        .setIsPartOfKey(isPrimaryKey)
 | 
					        .setIsPartOfKey(isPrimaryKey)
 | 
				
			||||||
        .setDescription(description)
 | 
					        .setDescription(description)
 | 
				
			||||||
        .setNativeDataType(field.nativeType())
 | 
					        .setNativeDataType(field.nativeType())
 | 
				
			||||||
        .setType(field.schemaFieldDataType())
 | 
					        .setType(field.schemaFieldDataType())
 | 
				
			||||||
        .setGlobalTags(new GlobalTags().setTags(new TagAssociationArray(tags)))
 | 
					        .setGlobalTags(new GlobalTags().setTags(new TagAssociationArray(tags)))
 | 
				
			||||||
        .setGlossaryTerms(new GlossaryTerms().setTerms(new GlossaryTermAssociationArray(terms))
 | 
					        .setGlossaryTerms(
 | 
				
			||||||
            .setAuditStamp(context.getAuditStamp()));
 | 
					            new GlossaryTerms()
 | 
				
			||||||
 | 
					                .setTerms(new GlossaryTermAssociationArray(terms))
 | 
				
			||||||
 | 
					                .setAuditStamp(context.getAuditStamp()));
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  private String createFieldDescription(ProtobufField field) {
 | 
					  private String createFieldDescription(ProtobufField field) {
 | 
				
			||||||
@ -73,26 +83,33 @@ public class ProtobufExtensionFieldVisitor extends SchemaFieldVisitor {
 | 
				
			|||||||
    return description.toString();
 | 
					    return description.toString();
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  private void appendEnumValues(StringBuilder description, ProtobufField field,
 | 
					  private void appendEnumValues(
 | 
				
			||||||
      Map<String, String> enumValuesWithComments) {
 | 
					      StringBuilder description, ProtobufField field, Map<String, String> enumValuesWithComments) {
 | 
				
			||||||
    enumValuesWithComments.forEach((name, comment) -> {
 | 
					    enumValuesWithComments.forEach(
 | 
				
			||||||
      field.getEnumValues().stream().filter(v -> v.getName().equals(name)).findFirst().ifPresent(value -> {
 | 
					        (name, comment) -> {
 | 
				
			||||||
        description.append(String.format("%d: %s", value.getNumber(), name));
 | 
					          field.getEnumValues().stream()
 | 
				
			||||||
        if (!comment.isEmpty()) {
 | 
					              .filter(v -> v.getName().equals(name))
 | 
				
			||||||
          description.append(" - ").append(comment);
 | 
					              .findFirst()
 | 
				
			||||||
        }
 | 
					              .ifPresent(
 | 
				
			||||||
        description.append("\n");
 | 
					                  value -> {
 | 
				
			||||||
      });
 | 
					                    description.append(String.format("%d: %s", value.getNumber(), name));
 | 
				
			||||||
    });
 | 
					                    if (!comment.isEmpty()) {
 | 
				
			||||||
 | 
					                      description.append(" - ").append(comment);
 | 
				
			||||||
 | 
					                    }
 | 
				
			||||||
 | 
					                    description.append("\n");
 | 
				
			||||||
 | 
					                  });
 | 
				
			||||||
 | 
					        });
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  private List<TagAssociation> getTagAssociations(ProtobufField field, VisitContext context) {
 | 
					  private List<TagAssociation> getTagAssociations(ProtobufField field, VisitContext context) {
 | 
				
			||||||
    Stream<TagAssociation> fieldTags =
 | 
					    Stream<TagAssociation> fieldTags =
 | 
				
			||||||
        ProtobufExtensionUtil.extractTagPropertiesFromOptions(getFieldOptions(field.getFieldProto()),
 | 
					        ProtobufExtensionUtil.extractTagPropertiesFromOptions(
 | 
				
			||||||
            context.getGraph().getRegistry()).map(tag -> new TagAssociation().setTag(new TagUrn(tag.getName())));
 | 
					                getFieldOptions(field.getFieldProto()), context.getGraph().getRegistry())
 | 
				
			||||||
 | 
					            .map(tag -> new TagAssociation().setTag(new TagUrn(tag.getName())));
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    Stream<TagAssociation> promotedTags =
 | 
					    Stream<TagAssociation> promotedTags =
 | 
				
			||||||
        promotedTags(field, context).map(tag -> new TagAssociation().setTag(new TagUrn(tag.getName())));
 | 
					        promotedTags(field, context)
 | 
				
			||||||
 | 
					            .map(tag -> new TagAssociation().setTag(new TagUrn(tag.getName())));
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    return Stream.concat(fieldTags, promotedTags)
 | 
					    return Stream.concat(fieldTags, promotedTags)
 | 
				
			||||||
        .distinct()
 | 
					        .distinct()
 | 
				
			||||||
@ -100,10 +117,11 @@ public class ProtobufExtensionFieldVisitor extends SchemaFieldVisitor {
 | 
				
			|||||||
        .collect(Collectors.toList());
 | 
					        .collect(Collectors.toList());
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  private List<GlossaryTermAssociation> getGlossaryTermAssociations(ProtobufField field, VisitContext context) {
 | 
					  private List<GlossaryTermAssociation> getGlossaryTermAssociations(
 | 
				
			||||||
 | 
					      ProtobufField field, VisitContext context) {
 | 
				
			||||||
    Stream<GlossaryTermAssociation> fieldTerms =
 | 
					    Stream<GlossaryTermAssociation> fieldTerms =
 | 
				
			||||||
        ProtobufExtensionUtil.extractTermAssociationsFromOptions(getFieldOptions(field.getFieldProto()),
 | 
					        ProtobufExtensionUtil.extractTermAssociationsFromOptions(
 | 
				
			||||||
            context.getGraph().getRegistry());
 | 
					            getFieldOptions(field.getFieldProto()), context.getGraph().getRegistry());
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    Stream<GlossaryTermAssociation> promotedTerms = promotedTerms(field, context);
 | 
					    Stream<GlossaryTermAssociation> promotedTerms = promotedTerms(field, context);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@ -120,11 +138,12 @@ public class ProtobufExtensionFieldVisitor extends SchemaFieldVisitor {
 | 
				
			|||||||
   */
 | 
					   */
 | 
				
			||||||
  private Stream<TagProperties> promotedTags(ProtobufField field, VisitContext context) {
 | 
					  private Stream<TagProperties> promotedTags(ProtobufField field, VisitContext context) {
 | 
				
			||||||
    if (field.isMessage()) {
 | 
					    if (field.isMessage()) {
 | 
				
			||||||
      return context.getGraph()
 | 
					      return context.getGraph().outgoingEdgesOf(field).stream()
 | 
				
			||||||
          .outgoingEdgesOf(field)
 | 
					          .flatMap(
 | 
				
			||||||
          .stream()
 | 
					              e ->
 | 
				
			||||||
          .flatMap(e -> ProtobufExtensionUtil.extractTagPropertiesFromOptions(
 | 
					                  ProtobufExtensionUtil.extractTagPropertiesFromOptions(
 | 
				
			||||||
              getMessageOptions(e.getEdgeTarget().messageProto()), context.getGraph().getRegistry()))
 | 
					                      getMessageOptions(e.getEdgeTarget().messageProto()),
 | 
				
			||||||
 | 
					                      context.getGraph().getRegistry()))
 | 
				
			||||||
          .distinct();
 | 
					          .distinct();
 | 
				
			||||||
    } else {
 | 
					    } else {
 | 
				
			||||||
      return Stream.of();
 | 
					      return Stream.of();
 | 
				
			||||||
@ -138,11 +157,12 @@ public class ProtobufExtensionFieldVisitor extends SchemaFieldVisitor {
 | 
				
			|||||||
   */
 | 
					   */
 | 
				
			||||||
  private Stream<GlossaryTermAssociation> promotedTerms(ProtobufField field, VisitContext context) {
 | 
					  private Stream<GlossaryTermAssociation> promotedTerms(ProtobufField field, VisitContext context) {
 | 
				
			||||||
    if (field.isMessage()) {
 | 
					    if (field.isMessage()) {
 | 
				
			||||||
      return context.getGraph()
 | 
					      return context.getGraph().outgoingEdgesOf(field).stream()
 | 
				
			||||||
          .outgoingEdgesOf(field)
 | 
					          .flatMap(
 | 
				
			||||||
          .stream()
 | 
					              e ->
 | 
				
			||||||
          .flatMap(e -> ProtobufExtensionUtil.extractTermAssociationsFromOptions(
 | 
					                  ProtobufExtensionUtil.extractTermAssociationsFromOptions(
 | 
				
			||||||
              getMessageOptions(e.getEdgeTarget().messageProto()), context.getGraph().getRegistry()))
 | 
					                      getMessageOptions(e.getEdgeTarget().messageProto()),
 | 
				
			||||||
 | 
					                      context.getGraph().getRegistry()))
 | 
				
			||||||
          .distinct();
 | 
					          .distinct();
 | 
				
			||||||
    } else {
 | 
					    } else {
 | 
				
			||||||
      return Stream.of();
 | 
					      return Stream.of();
 | 
				
			||||||
 | 
				
			|||||||
@ -48,15 +48,16 @@ public class ProtobufUtilsTest {
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
  @Test
 | 
					  @Test
 | 
				
			||||||
  public void testCollapseLocationCommentsWithUTF8() {
 | 
					  public void testCollapseLocationCommentsWithUTF8() {
 | 
				
			||||||
    DescriptorProtos.SourceCodeInfo.Location location = DescriptorProtos.SourceCodeInfo.Location.newBuilder()
 | 
					    DescriptorProtos.SourceCodeInfo.Location location =
 | 
				
			||||||
        .addAllLeadingDetachedComments(Arrays.asList("/* Emoji 😊 */", "/* Accented é */"))
 | 
					        DescriptorProtos.SourceCodeInfo.Location.newBuilder()
 | 
				
			||||||
        .setLeadingComments("/* Chinese 你好 */\n// Russian Привет")
 | 
					            .addAllLeadingDetachedComments(Arrays.asList("/* Emoji 😊 */", "/* Accented é */"))
 | 
				
			||||||
        .setTrailingComments("// Korean 안녕")
 | 
					            .setLeadingComments("/* Chinese 你好 */\n// Russian Привет")
 | 
				
			||||||
        .build();
 | 
					            .setTrailingComments("// Korean 안녕")
 | 
				
			||||||
 | 
					            .build();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    String actual = ProtobufUtils.collapseLocationComments(location);
 | 
					    String actual = ProtobufUtils.collapseLocationComments(location);
 | 
				
			||||||
    String expected = "Emoji 😊 */\nAccented é */\nChinese 你好 */\nRussian Привет\nKorean 안녕";
 | 
					    String expected = "Emoji 😊 */\nAccented é */\nChinese 你好 */\nRussian Привет\nKorean 안녕";
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    assertEquals(expected, actual);
 | 
					    assertEquals(expected, actual);
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
				
			|||||||
@ -361,14 +361,22 @@ public class ProtobufFieldTest {
 | 
				
			|||||||
    ProtobufDataset test = getTestProtobufDataset("extended_protobuf", "messageE");
 | 
					    ProtobufDataset test = getTestProtobufDataset("extended_protobuf", "messageE");
 | 
				
			||||||
    SchemaMetadata testMetadata = test.getSchemaMetadata();
 | 
					    SchemaMetadata testMetadata = test.getSchemaMetadata();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    SchemaField timestampField = testMetadata.getFields()
 | 
					    SchemaField timestampField =
 | 
				
			||||||
        .stream()
 | 
					        testMetadata.getFields().stream()
 | 
				
			||||||
        .filter(v -> v.getFieldPath()
 | 
					            .filter(
 | 
				
			||||||
            .equals("[version=2.0].[type=extended_protobuf_TimestampUnitMessage].[type=enum].timestamp_unit_type"))
 | 
					                v ->
 | 
				
			||||||
        .findFirst()
 | 
					                    v.getFieldPath()
 | 
				
			||||||
        .orElseThrow();
 | 
					                        .equals(
 | 
				
			||||||
 | 
					                            "[version=2.0].[type=extended_protobuf_TimestampUnitMessage].[type=enum].timestamp_unit_type"))
 | 
				
			||||||
 | 
					            .findFirst()
 | 
				
			||||||
 | 
					            .orElseThrow();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    assertEquals("timestamp unit\n" + "\n" + "0: MILLISECOND - 10^-3 seconds\n" + "1: MICROSECOND - 10^-6 seconds\n"
 | 
					    assertEquals(
 | 
				
			||||||
        + "2: NANOSECOND - 10^-9 seconds\n", timestampField.getDescription());
 | 
					        "timestamp unit\n"
 | 
				
			||||||
 | 
					            + "\n"
 | 
				
			||||||
 | 
					            + "0: MILLISECOND - 10^-3 seconds\n"
 | 
				
			||||||
 | 
					            + "1: MICROSECOND - 10^-6 seconds\n"
 | 
				
			||||||
 | 
					            + "2: NANOSECOND - 10^-9 seconds\n",
 | 
				
			||||||
 | 
					        timestampField.getDescription());
 | 
				
			||||||
  }
 | 
					  }
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
				
			|||||||
@ -52,7 +52,7 @@ import org.opensearch.index.query.functionscore.ScoreFunctionBuilders;
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
@Slf4j
 | 
					@Slf4j
 | 
				
			||||||
public class SearchQueryBuilder {
 | 
					public class SearchQueryBuilder {
 | 
				
			||||||
  public static final String STRUCTURED_QUERY_PREFIX = "\\\\/q ";
 | 
					  public static final String STRUCTURED_QUERY_PREFIX = "\\/q ";
 | 
				
			||||||
  private final ExactMatchConfiguration exactMatchConfiguration;
 | 
					  private final ExactMatchConfiguration exactMatchConfiguration;
 | 
				
			||||||
  private final PartialConfiguration partialConfiguration;
 | 
					  private final PartialConfiguration partialConfiguration;
 | 
				
			||||||
  private final WordGramConfiguration wordGramConfiguration;
 | 
					  private final WordGramConfiguration wordGramConfiguration;
 | 
				
			||||||
 | 
				
			|||||||
		Loading…
	
	
			
			x
			
			
		
	
		Reference in New Issue
	
	Block a user