From aba388190328b4d6dc012dded8f01b886d6627fb Mon Sep 17 00:00:00 2001 From: seoju Date: Tue, 10 Jan 2023 09:22:37 +0900 Subject: [PATCH] feat(ingest): add import descriptions of two or more nested messages (#6959) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 서주현[G플레이스데이터개발] --- .../protobuf/model/ProtobufElement.java | 3 +- .../datahub/protobuf/model/ProtobufField.java | 54 +++++++++++++++++- .../datahub/protobuf/model/ProtobufGraph.java | 8 +++ .../protobuf/model/ProtobufOneOfField.java | 2 +- .../protobuf/model/ProtobufFieldTest.java | 29 ++++++++++ .../extended_protobuf/messageC.proto | 14 +++++ .../extended_protobuf/messageC.protoc | Bin 0 -> 681 bytes 7 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 metadata-integration/java/datahub-protobuf/src/test/resources/extended_protobuf/messageC.proto create mode 100644 metadata-integration/java/datahub-protobuf/src/test/resources/extended_protobuf/messageC.protoc diff --git a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufElement.java b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufElement.java index 91c76fe16b..977c0b841d 100644 --- a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufElement.java +++ b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufElement.java @@ -24,8 +24,7 @@ public interface ProtobufElement { List fileLocations = fileProto().getSourceCodeInfo().getLocationList(); return fileLocations.stream() .filter(loc -> loc.getPathCount() > 1 - && loc.getPath(0) == FileDescriptorProto.MESSAGE_TYPE_FIELD_NUMBER - && messageProto() == fileProto().getMessageType(loc.getPath(1))); + && loc.getPath(0) == FileDescriptorProto.MESSAGE_TYPE_FIELD_NUMBER); } Stream accept(ProtobufModelVisitor v, VisitContext context); diff --git a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufField.java b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufField.java index 584fdc77a9..48748a0540 100644 --- a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufField.java +++ b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufField.java @@ -21,6 +21,8 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -37,6 +39,7 @@ public class ProtobufField implements ProtobufElement { private final String fieldPathType; private final Boolean isMessageType; private final SchemaFieldDataType schemaFieldDataType; + private final Boolean isNestedType; public OneofDescriptorProto oneOfProto() { if (fieldProto.hasOneofIndex()) { @@ -208,14 +211,59 @@ public class ProtobufField implements ProtobufElement { @Override public String comment() { return messageLocations() - .filter(loc -> loc.getPathCount() > 3 - && loc.getPath(2) == DescriptorProto.FIELD_FIELD_NUMBER - && fieldProto == messageProto().getField(loc.getPath(3))) + .filter(location -> location.getPathCount() > 3) + .filter(location -> !ProtobufUtils.collapseLocationComments(location).isEmpty() + && !isEnumType(location.getPathList())) + .filter(location -> { + List pathList = location.getPathList(); + DescriptorProto messageType = fileProto().getMessageType(pathList.get(1)); + + if (!isNestedType + && location.getPath(2) == DescriptorProto.FIELD_FIELD_NUMBER + && fieldProto == messageType.getField(location.getPath(3))) { + return true; + } else if (isNestedType + && location.getPath(2) == DescriptorProto.NESTED_TYPE_FIELD_NUMBER + && fieldProto == getNestedTypeFields(pathList, messageType)) { + return true; + } + return false; + }) .map(ProtobufUtils::collapseLocationComments) .collect(Collectors.joining("\n")) .trim(); } + private FieldDescriptorProto getNestedTypeFields(List pathList, DescriptorProto messageType) { + int pathSize = pathList.size(); + List nestedValues = new ArrayList<>(pathSize); + + for (int index = 0; index < pathSize; index++) { + if (index > 1 + && index % 2 == 0 + && pathList.get(index) == DescriptorProto.NESTED_TYPE_FIELD_NUMBER) { + nestedValues.add(pathList.get(index + 1)); + } + } + + for (Integer value : nestedValues) { + messageType = messageType.getNestedType(value); + } + + return messageType.getField(pathList.get(pathList.size() - 1)); + } + + private boolean isEnumType(List pathList) { + for (int index = 0; index < pathList.size(); index++) { + if (index > 1 + && index % 2 == 0 + && pathList.get(index) == DescriptorProto.ENUM_TYPE_FIELD_NUMBER) { + return true; + } + } + return false; + } + @Override public Stream accept(ProtobufModelVisitor visitor, VisitContext context) { return visitor.visitField(this, context); diff --git a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufGraph.java b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufGraph.java index c9d9709aa2..ae2319af85 100644 --- a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufGraph.java +++ b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/model/ProtobufGraph.java @@ -170,6 +170,7 @@ public class ProtobufGraph extends DefaultDirectedGraph { ProtobufMessage nestedMessageVertex = ProtobufMessage.builder() .fileProto(fileProto) @@ -222,6 +227,7 @@ public class ProtobufGraph extends DefaultDirectedGraph f.getFieldPath() + .equals("[version=2.0].[type=extended_protobuf_UserMsg].[type=extended_protobuf_UserMsg_UserInfo].user_info.[type=string].nickname")) + .findFirst() + .orElseThrow(); + + assertEquals("nickname info", nicknameField.getDescription()); + + SchemaField profileUrlField = testMetadata.getFields() + .stream().filter(f -> f.getFieldPath() + .equals("[version=2.0].[type=extended_protobuf_UserMsg].[type=extended_protobuf_UserMsg_UserInfo].user_info.[type=string].profile_url")) + .findFirst() + .orElseThrow(); + + assertEquals("profile url info", profileUrlField.getDescription()); + } } diff --git a/metadata-integration/java/datahub-protobuf/src/test/resources/extended_protobuf/messageC.proto b/metadata-integration/java/datahub-protobuf/src/test/resources/extended_protobuf/messageC.proto new file mode 100644 index 0000000000..7ff5320525 --- /dev/null +++ b/metadata-integration/java/datahub-protobuf/src/test/resources/extended_protobuf/messageC.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; +package extended_protobuf; + + +message UserMsg { + message UserInfo { + string nickname = 1; // nickname info + string profile_url = 2; // profile url info + } + + string id = 1; // user id + string name = 2; // user name + UserInfo user_info = 3; // user info +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-protobuf/src/test/resources/extended_protobuf/messageC.protoc b/metadata-integration/java/datahub-protobuf/src/test/resources/extended_protobuf/messageC.protoc new file mode 100644 index 0000000000000000000000000000000000000000..e87d4e7aaa646fd9269ac5b7bd3045d8e5551321 GIT binary patch literal 681 zcmZXS%T60X5JmenuiJwS6=V;NluU*VD=(5Ch{OUBi4Bt1mW{_~gs~$VOMDUEz=nUL zY900DnVw_Fjyt!IyAX6jB|({ChuUOuhX%S~q4@ITMT|6#{1vi@H4(%V>eNj17c zEUwfvLcFf-9_mw?a}S!n!*j}IwJN9k?Wzrdwl&$V{=8X)g^bJj7M5!C}V YCKRh98fK*av}G7(q{D0;)<5B|cksh#a{vGU literal 0 HcmV?d00001