mirror of
https://github.com/datahub-project/datahub.git
synced 2025-08-18 14:16:48 +00:00
feat(openapi): entity registry api (#13878)
This commit is contained in:
david-leifker
GitHub
parent
b162d6f365
commit
ddb4e17772
@ -7,7 +7,7 @@ import javax.annotation.Nonnull;
|
||||
import lombok.Value;
|
||||
|
||||
@Value
|
||||
public class UrnValidationFieldSpec {
|
||||
public class UrnValidationFieldSpec implements FieldSpec {
|
||||
@Nonnull PathSpec path;
|
||||
@Nonnull UrnValidationAnnotation urnValidationAnnotation;
|
||||
@Nonnull DataSchema pegasusSchema;
|
||||
|
||||
@ -16,6 +16,8 @@ dependencies {
|
||||
implementation externalDependency.opentelemetryApi
|
||||
implementation externalDependency.opentelemetrySdkTrace
|
||||
implementation externalDependency.opentelemetrySdkMetrics
|
||||
api externalDependency.jacksonDataBind
|
||||
api externalDependency.jacksonJDK8
|
||||
compileOnly externalDependency.lombok
|
||||
|
||||
annotationProcessor externalDependency.lombok
|
||||
|
||||
@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.core.StreamReadConstraints;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
|
||||
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
|
||||
import com.linkedin.metadata.Constants;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
@ -20,6 +21,7 @@ public class ObjectMapperContext implements ContextInterface {
|
||||
|
||||
static {
|
||||
defaultMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
|
||||
defaultMapper.registerModule(new Jdk8Module());
|
||||
|
||||
for (ObjectMapper mapper : List.of(defaultMapper, defaultYamlMapper)) {
|
||||
int maxSize =
|
||||
|
||||
@ -5,6 +5,7 @@ import static com.linkedin.metadata.Constants.MAX_JACKSON_STRING_SIZE;
|
||||
|
||||
import com.fasterxml.jackson.core.StreamReadConstraints;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@ -20,6 +21,7 @@ public class ObjectMapperFactory {
|
||||
objectMapper
|
||||
.getFactory()
|
||||
.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build());
|
||||
objectMapper.registerModule(new Jdk8Module());
|
||||
return objectMapper;
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,14 @@
|
||||
package io.datahubproject.openapi.v1.models.registry;
|
||||
|
||||
import java.util.Map;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public class AspectAnnotationDto {
|
||||
private String name;
|
||||
private boolean timeseries;
|
||||
private boolean autoRender;
|
||||
private Map<String, Object> renderSpec;
|
||||
}
|
||||
@ -0,0 +1,118 @@
|
||||
package io.datahubproject.openapi.v1.models.registry;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.linkedin.metadata.models.AspectSpec;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Data
|
||||
@Builder
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class AspectSpecDto {
|
||||
private AspectAnnotationDto aspectAnnotation;
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_EMPTY)
|
||||
private Map<String, FieldSpecDto> searchableFieldSpec;
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_EMPTY)
|
||||
private Map<String, FieldSpecDto> searchableRefFieldSpec;
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_EMPTY)
|
||||
private Map<String, FieldSpecDto> searchScoreFieldSpec;
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_EMPTY)
|
||||
private Map<String, FieldSpecDto> relationshipFieldSpec;
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_EMPTY)
|
||||
private Map<String, FieldSpecDto> timeseriesFieldSpec;
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_EMPTY)
|
||||
private Map<String, FieldSpecDto> timeseriesFieldCollectionSpec;
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_EMPTY)
|
||||
private Map<String, FieldSpecDto> urnValidationFieldSpec;
|
||||
|
||||
private String registryName;
|
||||
private String registryVersion;
|
||||
|
||||
public static AspectSpecDto fromAspectSpec(AspectSpec aspectSpec) {
|
||||
// Get the aspect annotation - need to use reflection since it's private
|
||||
AspectAnnotationDto aspectAnnotationDto = null;
|
||||
try {
|
||||
// The AspectSpec class has getName(), isTimeseries(), isAutoRender(), getRenderSpec() methods
|
||||
// that delegate to the AspectAnnotation
|
||||
aspectAnnotationDto =
|
||||
AspectAnnotationDto.builder()
|
||||
.name(aspectSpec.getName())
|
||||
.timeseries(aspectSpec.isTimeseries())
|
||||
.autoRender(aspectSpec.isAutoRender())
|
||||
.renderSpec(convertDataMap(aspectSpec.getRenderSpec()))
|
||||
.build();
|
||||
} catch (Exception e) {
|
||||
log.error("Error extracting aspect annotation: {}", e.getMessage());
|
||||
}
|
||||
|
||||
return AspectSpecDto.builder()
|
||||
.aspectAnnotation(aspectAnnotationDto)
|
||||
|
||||
// Convert Maps using the generic approach
|
||||
.searchableFieldSpec(convertFieldSpecMap(aspectSpec.getSearchableFieldSpecMap()))
|
||||
.searchScoreFieldSpec(convertFieldSpecMap(aspectSpec.getSearchScoreFieldSpecMap()))
|
||||
.relationshipFieldSpec(convertFieldSpecMap(aspectSpec.getRelationshipFieldSpecMap()))
|
||||
.timeseriesFieldSpec(convertFieldSpecMap(aspectSpec.getTimeseriesFieldSpecMap()))
|
||||
.timeseriesFieldCollectionSpec(
|
||||
convertFieldSpecMap(aspectSpec.getTimeseriesFieldCollectionSpecMap()))
|
||||
.searchableRefFieldSpec(
|
||||
convertFieldSpecMapFromList(aspectSpec.getSearchableRefFieldSpecs()))
|
||||
.urnValidationFieldSpec(convertFieldSpecMap(aspectSpec.getUrnValidationFieldSpecMap()))
|
||||
.registryName(aspectSpec.getRegistryName())
|
||||
.registryVersion(aspectSpec.getRegistryVersion().toString())
|
||||
.build();
|
||||
}
|
||||
|
||||
private static <T extends com.linkedin.metadata.models.FieldSpec>
|
||||
Map<String, FieldSpecDto> convertFieldSpecMap(Map<String, T> fieldSpecMap) {
|
||||
if (fieldSpecMap == null) {
|
||||
return new HashMap<>();
|
||||
}
|
||||
Map<String, FieldSpecDto> result = new HashMap<>();
|
||||
fieldSpecMap.forEach(
|
||||
(key, value) -> {
|
||||
FieldSpecDto converted = FieldSpecDto.fromFieldSpec(value);
|
||||
if (converted != null) {
|
||||
result.put(key, converted);
|
||||
}
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
// Helper for searchableRefFieldSpecs which only has a List getter
|
||||
private static Map<String, FieldSpecDto> convertFieldSpecMapFromList(
|
||||
List<? extends com.linkedin.metadata.models.FieldSpec> fieldSpecs) {
|
||||
if (fieldSpecs == null || fieldSpecs.isEmpty()) {
|
||||
return null; // Return null instead of empty map
|
||||
}
|
||||
Map<String, FieldSpecDto> result = new HashMap<>();
|
||||
for (com.linkedin.metadata.models.FieldSpec spec : fieldSpecs) {
|
||||
FieldSpecDto converted = FieldSpecDto.fromFieldSpec(spec);
|
||||
if (converted != null) {
|
||||
result.put(spec.getPath().toString(), converted);
|
||||
}
|
||||
}
|
||||
return result.isEmpty() ? null : result; // Return null if result is empty
|
||||
}
|
||||
|
||||
private static Map<String, Object> convertDataMap(com.linkedin.data.DataMap dataMap) {
|
||||
if (dataMap == null) {
|
||||
return null;
|
||||
}
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
dataMap.forEach(result::put);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
package io.datahubproject.openapi.v1.models.registry;
|
||||
|
||||
import com.linkedin.metadata.models.annotation.EntityAnnotation;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public class EntityAnnotationDto {
|
||||
private String name;
|
||||
private String keyAspect;
|
||||
|
||||
public static EntityAnnotationDto fromEntityAnnotation(EntityAnnotation annotation) {
|
||||
if (annotation == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return EntityAnnotationDto.builder()
|
||||
.name(annotation.getName())
|
||||
.keyAspect(annotation.getKeyAspect())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,112 @@
|
||||
package io.datahubproject.openapi.v1.models.registry;
|
||||
|
||||
import com.linkedin.metadata.models.EntitySpec;
|
||||
import com.linkedin.metadata.models.annotation.SearchableAnnotation;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public class EntitySpecDto {
|
||||
private String name;
|
||||
private EntityAnnotationDto entityAnnotation;
|
||||
private String keyAspectName;
|
||||
private AspectSpecDto keyAspectSpec;
|
||||
private List<AspectSpecDto> aspectSpecs;
|
||||
private Map<String, AspectSpecDto> aspectSpecMap;
|
||||
|
||||
private List<FieldSpecDto> searchableFieldSpecs;
|
||||
private Map<String, Set<String>> searchableFieldTypes;
|
||||
private List<FieldSpecDto> searchScoreFieldSpecs;
|
||||
private List<FieldSpecDto> relationshipFieldSpecs;
|
||||
private List<FieldSpecDto> searchableRefFieldSpecs;
|
||||
|
||||
// Schema information
|
||||
private String snapshotSchemaName;
|
||||
private String aspectTyperefSchemaName;
|
||||
|
||||
public static EntitySpecDto fromEntitySpec(EntitySpec entitySpec) {
|
||||
if (entitySpec == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return EntitySpecDto.builder()
|
||||
.name(entitySpec.getName())
|
||||
.entityAnnotation(
|
||||
EntityAnnotationDto.fromEntityAnnotation(entitySpec.getEntityAnnotation()))
|
||||
.keyAspectName(entitySpec.getKeyAspectName())
|
||||
.keyAspectSpec(AspectSpecDto.fromAspectSpec(entitySpec.getKeyAspectSpec()))
|
||||
.aspectSpecs(
|
||||
entitySpec.getAspectSpecs().stream()
|
||||
.map(AspectSpecDto::fromAspectSpec)
|
||||
.collect(Collectors.toList()))
|
||||
.aspectSpecMap(
|
||||
entitySpec.getAspectSpecMap().entrySet().stream()
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
Map.Entry::getKey,
|
||||
entry -> AspectSpecDto.fromAspectSpec(entry.getValue()))))
|
||||
.searchableFieldSpecs(
|
||||
entitySpec.getSearchableFieldSpecs().stream()
|
||||
.map(FieldSpecDto::fromFieldSpec)
|
||||
.collect(Collectors.toList()))
|
||||
.searchableFieldTypes(convertSearchableFieldTypes(entitySpec.getSearchableFieldTypes()))
|
||||
.searchScoreFieldSpecs(
|
||||
entitySpec.getSearchScoreFieldSpecs().stream()
|
||||
.map(FieldSpecDto::fromFieldSpec)
|
||||
.collect(Collectors.toList()))
|
||||
.relationshipFieldSpecs(
|
||||
entitySpec.getRelationshipFieldSpecs().stream()
|
||||
.map(FieldSpecDto::fromFieldSpec)
|
||||
.collect(Collectors.toList()))
|
||||
.searchableRefFieldSpecs(
|
||||
entitySpec.getSearchableRefFieldSpecs().stream()
|
||||
.map(FieldSpecDto::fromFieldSpec)
|
||||
.collect(Collectors.toList()))
|
||||
|
||||
// Schema information - handle potential UnsupportedOperationException
|
||||
.snapshotSchemaName(getSnapshotSchemaName(entitySpec))
|
||||
.aspectTyperefSchemaName(getAspectTyperefSchemaName(entitySpec))
|
||||
.build();
|
||||
}
|
||||
|
||||
private static Map<String, Set<String>> convertSearchableFieldTypes(
|
||||
Map<String, Set<SearchableAnnotation.FieldType>> fieldTypes) {
|
||||
if (fieldTypes == null) {
|
||||
return new HashMap<>();
|
||||
}
|
||||
|
||||
return fieldTypes.entrySet().stream()
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
Map.Entry::getKey,
|
||||
entry -> entry.getValue().stream().map(Enum::name).collect(Collectors.toSet())));
|
||||
}
|
||||
|
||||
private static String getSnapshotSchemaName(EntitySpec entitySpec) {
|
||||
try {
|
||||
return entitySpec.getSnapshotSchema() != null
|
||||
? entitySpec.getSnapshotSchema().getFullName()
|
||||
: null;
|
||||
} catch (UnsupportedOperationException e) {
|
||||
// Config-based entities don't have snapshot schemas
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static String getAspectTyperefSchemaName(EntitySpec entitySpec) {
|
||||
try {
|
||||
return entitySpec.getAspectTyperefSchema() != null
|
||||
? entitySpec.getAspectTyperefSchema().getFullName()
|
||||
: null;
|
||||
} catch (UnsupportedOperationException e) {
|
||||
// Config-based entities don't have aspect typeref schemas
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
package io.datahubproject.openapi.v1.models.registry;
|
||||
|
||||
import com.linkedin.metadata.models.EntitySpec;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public class EntitySpecResponse {
|
||||
private String name;
|
||||
private List<AspectSpecDto> aspectSpecs;
|
||||
|
||||
public static EntitySpecResponse fromEntitySpec(EntitySpec entitySpec) {
|
||||
return EntitySpecResponse.builder()
|
||||
.name(entitySpec.getName())
|
||||
.aspectSpecs(
|
||||
entitySpec.getAspectSpecs().stream()
|
||||
.map(AspectSpecDto::fromAspectSpec)
|
||||
.collect(Collectors.toList()))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
package io.datahubproject.openapi.v1.models.registry;
|
||||
|
||||
import com.linkedin.metadata.models.annotation.EventAnnotation;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public class EventAnnotationDto {
|
||||
private String name;
|
||||
|
||||
public static EventAnnotationDto fromEventAnnotation(EventAnnotation annotation) {
|
||||
if (annotation == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return EventAnnotationDto.builder().name(annotation.getName()).build();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
package io.datahubproject.openapi.v1.models.registry;
|
||||
|
||||
import com.linkedin.metadata.models.EventSpec;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public class EventSpecDto {
|
||||
private String name;
|
||||
private EventAnnotationDto eventAnnotation;
|
||||
private String pegasusSchemaName;
|
||||
private String pegasusSchemaDoc;
|
||||
|
||||
public static EventSpecDto fromEventSpec(EventSpec eventSpec) {
|
||||
if (eventSpec == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return EventSpecDto.builder()
|
||||
.name(eventSpec.getName())
|
||||
.eventAnnotation(EventAnnotationDto.fromEventAnnotation(eventSpec.getEventAnnotation()))
|
||||
.pegasusSchemaName(
|
||||
eventSpec.getPegasusSchema() != null
|
||||
? eventSpec.getPegasusSchema().getFullName()
|
||||
: null)
|
||||
.pegasusSchemaDoc(
|
||||
eventSpec.getPegasusSchema() != null ? eventSpec.getPegasusSchema().getDoc() : null)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,208 @@
|
||||
package io.datahubproject.openapi.v1.models.registry;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonGenerator;
|
||||
import com.fasterxml.jackson.databind.JsonSerializer;
|
||||
import com.fasterxml.jackson.databind.SerializerProvider;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
import com.linkedin.data.schema.DataSchema;
|
||||
import com.linkedin.metadata.models.*;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Data
|
||||
@Builder
|
||||
public class FieldSpecDto {
|
||||
private List<String> pathComponents;
|
||||
private String fieldType;
|
||||
|
||||
@JsonSerialize(using = DataSchemaSerializer.class)
|
||||
private DataSchema pegasusSchema;
|
||||
|
||||
private Map<String, Object> annotations; // All annotation data
|
||||
private Map<String, Object> properties; // Other properties
|
||||
|
||||
public static class DataSchemaSerializer extends JsonSerializer<DataSchema> {
|
||||
@Override
|
||||
public void serialize(DataSchema value, JsonGenerator gen, SerializerProvider serializers)
|
||||
throws IOException {
|
||||
if (value != null) {
|
||||
gen.writeRawValue(value.toString());
|
||||
} else {
|
||||
gen.writeNull();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static FieldSpecDto fromFieldSpec(FieldSpec spec) {
|
||||
if (spec == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
FieldSpecDtoBuilder builder =
|
||||
FieldSpecDto.builder()
|
||||
.pathComponents(spec.getPath().getPathComponents())
|
||||
.fieldType(spec.getClass().getSimpleName())
|
||||
.pegasusSchema(spec.getPegasusSchema());
|
||||
|
||||
Map<String, Object> annotations = new HashMap<>();
|
||||
Map<String, Object> properties = new HashMap<>();
|
||||
|
||||
// Use reflection to find all getter methods
|
||||
Method[] methods = spec.getClass().getMethods();
|
||||
for (Method method : methods) {
|
||||
String methodName = method.getName();
|
||||
|
||||
// Skip common interface methods we've already handled
|
||||
if (methodName.equals("getPath")
|
||||
|| methodName.equals("getPegasusSchema")
|
||||
|| methodName.equals("getClass")
|
||||
|| methodName.equals("hashCode")
|
||||
|| methodName.equals("equals")
|
||||
|| methodName.equals("toString")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Process getter methods
|
||||
if ((methodName.startsWith("get") || methodName.startsWith("is"))
|
||||
&& method.getParameterCount() == 0) {
|
||||
|
||||
try {
|
||||
Object value = method.invoke(spec);
|
||||
if (value != null) {
|
||||
String propertyName = extractPropertyName(methodName);
|
||||
|
||||
// Sanitize the value to prevent circular references
|
||||
value = sanitizeValue(value);
|
||||
|
||||
// If it's an annotation object, serialize it separately
|
||||
if (methodName.contains("Annotation")) {
|
||||
annotations.put(propertyName, value);
|
||||
} else {
|
||||
properties.put(propertyName, value);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
// Log error but continue processing
|
||||
System.err.println("Error processing method " + methodName + ": " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
builder.annotations(annotations);
|
||||
builder.properties(properties);
|
||||
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private static String extractPropertyName(String methodName) {
|
||||
if (methodName.startsWith("get")) {
|
||||
return Character.toLowerCase(methodName.charAt(3)) + methodName.substring(4);
|
||||
} else if (methodName.startsWith("is")) {
|
||||
return Character.toLowerCase(methodName.charAt(2)) + methodName.substring(3);
|
||||
}
|
||||
return methodName;
|
||||
}
|
||||
|
||||
private static Map<String, Object> serializeAnnotation(Object annotation) {
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
|
||||
// Use reflection to extract all properties from the annotation
|
||||
Method[] methods = annotation.getClass().getMethods();
|
||||
for (Method method : methods) {
|
||||
String methodName = method.getName();
|
||||
|
||||
if ((methodName.startsWith("get") || methodName.startsWith("is"))
|
||||
&& method.getParameterCount() == 0
|
||||
&& !methodName.equals("getClass")
|
||||
&& !methodName.equals("hashCode")
|
||||
&& !methodName.equals("equals")
|
||||
&& !methodName.equals("toString")) {
|
||||
|
||||
try {
|
||||
Object value = method.invoke(annotation);
|
||||
if (value != null) {
|
||||
|
||||
String propertyName = extractPropertyName(methodName);
|
||||
|
||||
// Sanitize the value to prevent circular references
|
||||
value = sanitizeValue(value);
|
||||
|
||||
result.put(propertyName, value);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
// Log error but continue
|
||||
log.error("Error processing annotation method {}: {}", methodName, e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitizes values to prevent circular references and other serialization issues. This method
|
||||
* handles special types that can cause problems during JSON serialization.
|
||||
*/
|
||||
private static Object sanitizeValue(Object value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Handle PathSpec to avoid circular references
|
||||
if (value instanceof com.linkedin.data.schema.PathSpec) {
|
||||
com.linkedin.data.schema.PathSpec pathSpec = (com.linkedin.data.schema.PathSpec) value;
|
||||
Map<String, Object> pathSpecMap = new HashMap<>();
|
||||
pathSpecMap.put("pathComponents", pathSpec.getPathComponents());
|
||||
// Explicitly don't include parent to avoid circular reference
|
||||
return pathSpecMap;
|
||||
}
|
||||
|
||||
// Handle FieldSpec objects (including TimeseriesFieldSpec, RelationshipFieldSpec, etc.)
|
||||
if (value instanceof com.linkedin.metadata.models.FieldSpec) {
|
||||
// Convert to FieldSpecDto to avoid circular references
|
||||
return FieldSpecDto.fromFieldSpec((com.linkedin.metadata.models.FieldSpec) value);
|
||||
}
|
||||
|
||||
// Handle DataMap
|
||||
if (value instanceof com.linkedin.data.DataMap) {
|
||||
Map<String, Object> dataMapCopy = new HashMap<>();
|
||||
((com.linkedin.data.DataMap) value).forEach(dataMapCopy::put);
|
||||
return dataMapCopy;
|
||||
}
|
||||
|
||||
// Handle Maps recursively
|
||||
if (value instanceof Map) {
|
||||
Map<String, Object> sanitizedMap = new HashMap<>();
|
||||
((Map<?, ?>) value)
|
||||
.forEach(
|
||||
(k, v) -> {
|
||||
String key = k != null ? k.toString() : null;
|
||||
if (key != null) {
|
||||
sanitizedMap.put(key, sanitizeValue(v));
|
||||
}
|
||||
});
|
||||
return sanitizedMap;
|
||||
}
|
||||
|
||||
// Handle Collections recursively
|
||||
if (value instanceof List) {
|
||||
return ((List<?>) value)
|
||||
.stream().map(FieldSpecDto::sanitizeValue).collect(java.util.stream.Collectors.toList());
|
||||
}
|
||||
|
||||
// If it's an annotation object (contains "Annotation" in class name), serialize it
|
||||
if (value.getClass().getName().contains("Annotation")) {
|
||||
return serializeAnnotation(value);
|
||||
}
|
||||
|
||||
// Return primitive types and strings as-is
|
||||
return value;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,61 @@
|
||||
package io.datahubproject.openapi.v1.models.registry;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude.Include;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@JsonInclude(Include.NON_NULL)
|
||||
public class PaginatedResponse<T> {
|
||||
private List<T> elements;
|
||||
private Integer start;
|
||||
private Integer count;
|
||||
private Integer total;
|
||||
|
||||
/** Creates a paginated response for DTOs that need custom mapping */
|
||||
public static <K extends Comparable<K>, V, D> PaginatedResponse<D> fromMap(
|
||||
Map<K, V> items,
|
||||
Integer start,
|
||||
Integer count,
|
||||
java.util.function.Function<Map.Entry<K, V>, D> mapper) {
|
||||
|
||||
if (items == null || items.isEmpty()) {
|
||||
return PaginatedResponse.<D>builder().elements(List.of()).start(0).count(0).total(0).build();
|
||||
}
|
||||
|
||||
// Sort by key and map to DTOs
|
||||
List<D> sortedItems =
|
||||
items.entrySet().stream()
|
||||
.sorted(Map.Entry.comparingByKey())
|
||||
.map(mapper)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
// Apply pagination
|
||||
int actualStart = start != null ? start : 0;
|
||||
int actualCount = count != null ? count : sortedItems.size();
|
||||
|
||||
// Ensure start is within bounds
|
||||
if (actualStart > (sortedItems.size() - 1)) {
|
||||
throw new IllegalArgumentException(
|
||||
String.format("Start offset %s exceeds total %s", actualStart, sortedItems.size() - 1));
|
||||
}
|
||||
|
||||
// Calculate end index
|
||||
int endIndex = Math.min(actualStart + actualCount, sortedItems.size());
|
||||
|
||||
// Get the page
|
||||
List<D> pageItems = sortedItems.subList(actualStart, endIndex);
|
||||
|
||||
return PaginatedResponse.<D>builder()
|
||||
.elements(pageItems)
|
||||
.start(actualStart)
|
||||
.count(pageItems.size())
|
||||
.total(sortedItems.size())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,350 @@
|
||||
package io.datahubproject.openapi.v1.registry;
|
||||
|
||||
import com.datahub.authentication.Authentication;
|
||||
import com.datahub.authentication.AuthenticationContext;
|
||||
import com.datahub.authorization.AuthUtil;
|
||||
import com.datahub.authorization.AuthorizerChain;
|
||||
import com.linkedin.metadata.authorization.PoliciesConfig;
|
||||
import com.linkedin.metadata.models.AspectSpec;
|
||||
import com.linkedin.metadata.models.EntitySpec;
|
||||
import com.linkedin.metadata.models.EventSpec;
|
||||
import io.datahubproject.metadata.context.OperationContext;
|
||||
import io.datahubproject.metadata.context.RequestContext;
|
||||
import io.datahubproject.openapi.v1.models.registry.AspectSpecDto;
|
||||
import io.datahubproject.openapi.v1.models.registry.EntitySpecDto;
|
||||
import io.datahubproject.openapi.v1.models.registry.EntitySpecResponse;
|
||||
import io.datahubproject.openapi.v1.models.registry.EventSpecDto;
|
||||
import io.datahubproject.openapi.v1.models.registry.PaginatedResponse;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.media.Content;
|
||||
import io.swagger.v3.oas.annotations.responses.ApiResponse;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/openapi/v1/registry/models")
|
||||
@Slf4j
|
||||
@Tag(name = "Entity Registry API", description = "An API to expose the Entity Registry")
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class EntityRegistryController {
|
||||
@Autowired private AuthorizerChain authorizerChain;
|
||||
@Autowired private OperationContext systemOperationContext;
|
||||
|
||||
@GetMapping(path = "/entity/specifications", produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
@Operation(
|
||||
description = "Retrieves all entity specs. Requires MANAGE_SYSTEM_OPERATIONS_PRIVILEGE.",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
responseCode = "200",
|
||||
description = "Successfully returned entity specs",
|
||||
content = @Content(mediaType = MediaType.APPLICATION_JSON_VALUE)),
|
||||
@ApiResponse(
|
||||
responseCode = "403",
|
||||
description = "Caller not authorized to access the entity registry")
|
||||
})
|
||||
public ResponseEntity<PaginatedResponse<EntitySpecDto>> getEntitySpecs(
|
||||
HttpServletRequest request,
|
||||
@Parameter(description = "Start index for pagination", example = "0")
|
||||
@RequestParam(name = "start", required = false, defaultValue = "0")
|
||||
Integer start,
|
||||
@Parameter(name = "count", description = "Number of items to return", example = "5")
|
||||
@RequestParam(name = "count", required = false, defaultValue = "5")
|
||||
Integer count) {
|
||||
|
||||
Authentication authentication = AuthenticationContext.getAuthentication();
|
||||
String actorUrnStr = authentication.getActor().toUrnStr();
|
||||
OperationContext opContext =
|
||||
systemOperationContext.asSession(
|
||||
RequestContext.builder()
|
||||
.buildOpenapi(actorUrnStr, request, "getEntitySpecs", Collections.emptyList()),
|
||||
authorizerChain,
|
||||
authentication);
|
||||
|
||||
if (!AuthUtil.isAPIOperationsAuthorized(
|
||||
opContext, PoliciesConfig.MANAGE_SYSTEM_OPERATIONS_PRIVILEGE)) {
|
||||
log.error("{} is not authorized to get entity", actorUrnStr);
|
||||
return ResponseEntity.status(HttpStatus.FORBIDDEN).body(null);
|
||||
}
|
||||
|
||||
Map<String, EntitySpec> entitySpecs =
|
||||
systemOperationContext.getEntityRegistry().getEntitySpecs();
|
||||
|
||||
PaginatedResponse<EntitySpecDto> response =
|
||||
PaginatedResponse.fromMap(
|
||||
entitySpecs,
|
||||
start,
|
||||
count,
|
||||
entry -> {
|
||||
EntitySpecDto dto = EntitySpecDto.fromEntitySpec(entry.getValue());
|
||||
// Ensure the name is set from the map key
|
||||
if (dto.getName() == null) {
|
||||
dto.setName(entry.getKey());
|
||||
}
|
||||
return dto;
|
||||
});
|
||||
|
||||
return ResponseEntity.ok(response);
|
||||
}
|
||||
|
||||
@GetMapping(
|
||||
path = "/entity/specifications/{entityName}",
|
||||
produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
@Operation(
|
||||
description =
|
||||
"Retrieves entity spec for entity. Requires MANAGE_SYSTEM_OPERATIONS_PRIVILEGE.",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
responseCode = "200",
|
||||
description = "Successfully returned entity spec",
|
||||
content = @Content(mediaType = MediaType.APPLICATION_JSON_VALUE)),
|
||||
@ApiResponse(
|
||||
responseCode = "403",
|
||||
description = "Caller not authorized to access the entity registry")
|
||||
})
|
||||
public ResponseEntity<EntitySpecResponse> getEntitySpec(
|
||||
HttpServletRequest request, @PathVariable("entityName") String entityName) {
|
||||
|
||||
Authentication authentication = AuthenticationContext.getAuthentication();
|
||||
String actorUrnStr = authentication.getActor().toUrnStr();
|
||||
OperationContext opContext =
|
||||
systemOperationContext.asSession(
|
||||
RequestContext.builder()
|
||||
.buildOpenapi(actorUrnStr, request, "getEntitySpec", Collections.emptyList()),
|
||||
authorizerChain,
|
||||
authentication);
|
||||
|
||||
if (!AuthUtil.isAPIOperationsAuthorized(
|
||||
opContext, PoliciesConfig.MANAGE_SYSTEM_OPERATIONS_PRIVILEGE)) {
|
||||
log.error("{} is not authorized to get entity", actorUrnStr);
|
||||
return ResponseEntity.status(HttpStatus.FORBIDDEN).body(null);
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
EntitySpec entitySpec = systemOperationContext.getEntityRegistry().getEntitySpec(entityName);
|
||||
|
||||
return ResponseEntity.ok(EntitySpecResponse.fromEntitySpec(entitySpec));
|
||||
|
||||
} catch (IllegalArgumentException e) {
|
||||
if (e.getMessage() != null && e.getMessage().contains("Failed to find entity with name")) {
|
||||
return ResponseEntity.notFound().build();
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
@GetMapping(
|
||||
path = "/entity/specifications/{entityName}/aspects",
|
||||
produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
@Operation(
|
||||
description =
|
||||
"Retrieves aspect specs for the entity. Requires MANAGE_SYSTEM_OPERATIONS_PRIVILEGE.",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
responseCode = "200",
|
||||
description = "Successfully returned aspect specs",
|
||||
content = @Content(mediaType = MediaType.APPLICATION_JSON_VALUE)),
|
||||
@ApiResponse(
|
||||
responseCode = "403",
|
||||
description = "Caller not authorized to access the entity registry")
|
||||
})
|
||||
public ResponseEntity<Map<String, AspectSpecDto>> getEntityAspecSpecs(
|
||||
HttpServletRequest request, @PathVariable("entityName") String entityName) {
|
||||
|
||||
Authentication authentication = AuthenticationContext.getAuthentication();
|
||||
String actorUrnStr = authentication.getActor().toUrnStr();
|
||||
OperationContext opContext =
|
||||
systemOperationContext.asSession(
|
||||
RequestContext.builder()
|
||||
.buildOpenapi(actorUrnStr, request, "getEntityAspecSpecs", Collections.emptyList()),
|
||||
authorizerChain,
|
||||
authentication);
|
||||
|
||||
if (!AuthUtil.isAPIOperationsAuthorized(
|
||||
opContext, PoliciesConfig.MANAGE_SYSTEM_OPERATIONS_PRIVILEGE)) {
|
||||
log.error("{} is not authorized to get entity model", actorUrnStr);
|
||||
return ResponseEntity.status(HttpStatus.FORBIDDEN).body(null);
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
Map<String, AspectSpec> aspectSpecs =
|
||||
systemOperationContext.getEntityRegistry().getEntitySpec(entityName).getAspectSpecMap();
|
||||
Map<String, AspectSpecDto> aspectSpecDtos =
|
||||
aspectSpecs.entrySet().stream()
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
Map.Entry::getKey, entry -> AspectSpecDto.fromAspectSpec(entry.getValue())));
|
||||
return ResponseEntity.ok(aspectSpecDtos);
|
||||
|
||||
} catch (IllegalArgumentException e) {
|
||||
if (e.getMessage() != null && e.getMessage().contains("Failed to find entity with name")) {
|
||||
return ResponseEntity.notFound().build();
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
@GetMapping(path = "/aspect/specifications", produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
@Operation(
|
||||
description = "Retrieves all entity specs. Requires MANAGE_SYSTEM_OPERATIONS_PRIVILEGE.",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
responseCode = "200",
|
||||
description = "Successfully returned entity specs",
|
||||
content = @Content(mediaType = MediaType.APPLICATION_JSON_VALUE)),
|
||||
@ApiResponse(
|
||||
responseCode = "403",
|
||||
description = "Caller not authorized to access the entity registry")
|
||||
})
|
||||
public ResponseEntity<PaginatedResponse<AspectSpecDto>> getAspectSpecs(
|
||||
HttpServletRequest request,
|
||||
@Parameter(description = "Start index for pagination", example = "0")
|
||||
@RequestParam(name = "start", required = false, defaultValue = "0")
|
||||
Integer start,
|
||||
@Parameter(name = "count", description = "Number of items to return", example = "20")
|
||||
@RequestParam(name = "count", required = false, defaultValue = "20")
|
||||
Integer count) {
|
||||
|
||||
Authentication authentication = AuthenticationContext.getAuthentication();
|
||||
String actorUrnStr = authentication.getActor().toUrnStr();
|
||||
OperationContext opContext =
|
||||
systemOperationContext.asSession(
|
||||
RequestContext.builder()
|
||||
.buildOpenapi(actorUrnStr, request, "getAspectSpecs", Collections.emptyList()),
|
||||
authorizerChain,
|
||||
authentication);
|
||||
|
||||
if (!AuthUtil.isAPIOperationsAuthorized(
|
||||
opContext, PoliciesConfig.MANAGE_SYSTEM_OPERATIONS_PRIVILEGE)) {
|
||||
log.error("{} is not authorized to get aspect models", actorUrnStr);
|
||||
return ResponseEntity.status(HttpStatus.FORBIDDEN).body(null);
|
||||
}
|
||||
|
||||
Map<String, AspectSpec> aspectSpecs =
|
||||
systemOperationContext.getEntityRegistry().getAspectSpecs();
|
||||
|
||||
PaginatedResponse<AspectSpecDto> response =
|
||||
PaginatedResponse.fromMap(
|
||||
aspectSpecs, start, count, entry -> AspectSpecDto.fromAspectSpec(entry.getValue()));
|
||||
|
||||
return ResponseEntity.ok(response);
|
||||
}
|
||||
|
||||
@GetMapping(
|
||||
path = "/aspect/specifications/{aspectName}",
|
||||
produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
@Operation(
|
||||
description =
|
||||
"Retrieves aspect spec for the specified aspect. Requires MANAGE_SYSTEM_OPERATIONS_PRIVILEGE.",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
responseCode = "200",
|
||||
description = "Successfully returned aspect spec",
|
||||
content = @Content(mediaType = MediaType.APPLICATION_JSON_VALUE)),
|
||||
@ApiResponse(
|
||||
responseCode = "403",
|
||||
description = "Caller not authorized to access the entity registry")
|
||||
})
|
||||
public ResponseEntity<AspectSpecDto> getAspectSpec(
|
||||
HttpServletRequest request, @PathVariable("aspectName") String aspectName) {
|
||||
|
||||
Authentication authentication = AuthenticationContext.getAuthentication();
|
||||
String actorUrnStr = authentication.getActor().toUrnStr();
|
||||
OperationContext opContext =
|
||||
systemOperationContext.asSession(
|
||||
RequestContext.builder()
|
||||
.buildOpenapi(actorUrnStr, request, "getAspectSpec", Collections.emptyList()),
|
||||
authorizerChain,
|
||||
authentication);
|
||||
|
||||
if (!AuthUtil.isAPIOperationsAuthorized(
|
||||
opContext, PoliciesConfig.MANAGE_SYSTEM_OPERATIONS_PRIVILEGE)) {
|
||||
log.error("{} is not authorized to get aspect models", actorUrnStr);
|
||||
return ResponseEntity.status(HttpStatus.FORBIDDEN).body(null);
|
||||
}
|
||||
|
||||
if (systemOperationContext.getEntityRegistry().getAspectSpecs().get(aspectName) != null) {
|
||||
|
||||
return ResponseEntity.ok(
|
||||
AspectSpecDto.fromAspectSpec(
|
||||
systemOperationContext.getEntityRegistry().getAspectSpecs().get(aspectName)));
|
||||
|
||||
} else {
|
||||
return ResponseEntity.notFound().build();
|
||||
}
|
||||
}
|
||||
|
||||
@GetMapping(path = "/event/specifications", produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
@Operation(
|
||||
description = "Retrieves all event specs. Requires MANAGE_SYSTEM_OPERATIONS_PRIVILEGE.",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
responseCode = "200",
|
||||
description = "Successfully returned event specs",
|
||||
content = @Content(mediaType = MediaType.APPLICATION_JSON_VALUE)),
|
||||
@ApiResponse(
|
||||
responseCode = "403",
|
||||
description = "Caller not authorized to access the entity registry")
|
||||
})
|
||||
public ResponseEntity<PaginatedResponse<EventSpecDto>> getEventSpecs(
|
||||
HttpServletRequest request,
|
||||
@Parameter(description = "Start index for pagination", example = "0")
|
||||
@RequestParam(name = "start", required = false, defaultValue = "0")
|
||||
Integer start,
|
||||
@Parameter(name = "count", description = "Number of items to return", example = "20")
|
||||
@RequestParam(name = "count", required = false, defaultValue = "20")
|
||||
Integer count) {
|
||||
|
||||
Authentication authentication = AuthenticationContext.getAuthentication();
|
||||
String actorUrnStr = authentication.getActor().toUrnStr();
|
||||
OperationContext opContext =
|
||||
systemOperationContext.asSession(
|
||||
RequestContext.builder()
|
||||
.buildOpenapi(actorUrnStr, request, "getEventSpecs", Collections.emptyList()),
|
||||
authorizerChain,
|
||||
authentication);
|
||||
|
||||
if (!AuthUtil.isAPIOperationsAuthorized(
|
||||
opContext, PoliciesConfig.MANAGE_SYSTEM_OPERATIONS_PRIVILEGE)) {
|
||||
log.error("{} is not authorized to get event models", actorUrnStr);
|
||||
return ResponseEntity.status(HttpStatus.FORBIDDEN).body(null);
|
||||
}
|
||||
|
||||
Map<String, EventSpec> eventSpecs = systemOperationContext.getEntityRegistry().getEventSpecs();
|
||||
|
||||
// Use the PaginatedResponse helper to handle pagination and sorting
|
||||
PaginatedResponse<EventSpecDto> response =
|
||||
PaginatedResponse.fromMap(
|
||||
eventSpecs,
|
||||
start,
|
||||
count,
|
||||
entry -> {
|
||||
EventSpecDto dto = EventSpecDto.fromEventSpec(entry.getValue());
|
||||
// Ensure the name is set from the map key
|
||||
if (dto.getName() == null) {
|
||||
dto.setName(entry.getKey());
|
||||
}
|
||||
return dto;
|
||||
});
|
||||
|
||||
return ResponseEntity.ok(response);
|
||||
}
|
||||
}
|
||||
@ -111,7 +111,7 @@ public class LineageRegistryController {
|
||||
@GetMapping(path = "/edges/{entityName}", produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
@Operation(
|
||||
description =
|
||||
"Retrieves lineage lineage edges for entity. Requires MANAGE_SYSTEM_OPERATIONS_PRIVILEGE.",
|
||||
"Retrieves lineage edges for entity. Requires MANAGE_SYSTEM_OPERATIONS_PRIVILEGE.",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
responseCode = "200",
|
||||
@ -156,7 +156,7 @@ public class LineageRegistryController {
|
||||
@GetMapping(path = "/edges/{entityName}/{direction}", produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
@Operation(
|
||||
description =
|
||||
"Retrieves lineage lineage edges for entity in the provided direction. Requires MANAGE_SYSTEM_OPERATIONS_PRIVILEGE.",
|
||||
"Retrieves lineage edges for entity in the provided direction. Requires MANAGE_SYSTEM_OPERATIONS_PRIVILEGE.",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
responseCode = "200",
|
||||
|
||||
@ -0,0 +1,510 @@
|
||||
package io.datahubproject.openapi.v1.registry;
|
||||
|
||||
import static io.datahubproject.test.metadata.context.TestOperationContexts.TEST_USER_AUTH;
|
||||
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
|
||||
import static org.hamcrest.Matchers.hasSize;
|
||||
import static org.hamcrest.Matchers.is;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
|
||||
|
||||
import com.datahub.authentication.AuthenticationContext;
|
||||
import com.datahub.authorization.AuthUtil;
|
||||
import com.datahub.authorization.AuthorizerChain;
|
||||
import com.linkedin.metadata.authorization.PoliciesConfig;
|
||||
import io.datahubproject.metadata.context.OperationContext;
|
||||
import io.datahubproject.test.metadata.context.TestOperationContexts;
|
||||
import jakarta.servlet.ServletException;
|
||||
import org.mockito.MockedStatic;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
public class EntityRegistryControllerTest {
|
||||
private MockMvc mockMvc;
|
||||
private EntityRegistryController controller;
|
||||
private AuthorizerChain mockAuthorizerChain;
|
||||
private OperationContext operationContext;
|
||||
private MockedStatic<AuthUtil> authUtilMock;
|
||||
private MockedStatic<AuthenticationContext> authContextMock;
|
||||
|
||||
private static final String TEST_ENTITY_NAME = "dataset";
|
||||
private static final String TEST_ASPECT_NAME = "datasetProperties";
|
||||
private static final String NON_EXISTENT_ENTITY = "nonExistentEntity";
|
||||
private static final String NON_EXISTENT_ASPECT = "nonExistentAspect";
|
||||
|
||||
@BeforeMethod
|
||||
public void setup() {
|
||||
// Create mocks
|
||||
mockAuthorizerChain = mock(AuthorizerChain.class);
|
||||
|
||||
operationContext =
|
||||
TestOperationContexts.userContextNoSearchAuthorization(mockAuthorizerChain, TEST_USER_AUTH);
|
||||
|
||||
authContextMock = Mockito.mockStatic(AuthenticationContext.class);
|
||||
authContextMock.when(AuthenticationContext::getAuthentication).thenReturn(TEST_USER_AUTH);
|
||||
|
||||
// Create controller
|
||||
controller = new EntityRegistryController(mockAuthorizerChain, operationContext);
|
||||
|
||||
// Setup MockMvc
|
||||
mockMvc = MockMvcBuilders.standaloneSetup(controller).build();
|
||||
|
||||
// Mock AuthUtil static methods
|
||||
authUtilMock = Mockito.mockStatic(AuthUtil.class);
|
||||
}
|
||||
|
||||
@AfterMethod
|
||||
public void tearDown() {
|
||||
authUtilMock.close();
|
||||
authContextMock.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetEntitySpecsWithAuthorizationDefaultPagination() throws Exception {
|
||||
// Setup authorization to return true
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(true);
|
||||
|
||||
// Execute test - default pagination (start=0, count=20)
|
||||
mockMvc
|
||||
.perform(
|
||||
get("/openapi/v1/registry/models/entity/specifications")
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.start", is(0)))
|
||||
.andExpect(jsonPath("$.count", is(5)))
|
||||
.andExpect(jsonPath("$.total", greaterThanOrEqualTo(50)))
|
||||
.andExpect(jsonPath("$.elements", hasSize(5)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetEntitySpecsWithCustomPagination() throws Exception {
|
||||
// Setup authorization to return true
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(true);
|
||||
|
||||
// Execute test with custom pagination
|
||||
mockMvc
|
||||
.perform(
|
||||
get("/openapi/v1/registry/models/entity/specifications")
|
||||
.param("start", "10")
|
||||
.param("count", "5")
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.start", is(10)))
|
||||
.andExpect(jsonPath("$.count", is(5)))
|
||||
.andExpect(jsonPath("$.total", greaterThanOrEqualTo(50)))
|
||||
.andExpect(jsonPath("$.elements", hasSize(5)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetEntitySpecsWithLargeCount() throws Exception {
|
||||
// Setup authorization to return true
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(true);
|
||||
|
||||
// Execute test with count larger than total
|
||||
mockMvc
|
||||
.perform(
|
||||
get("/openapi/v1/registry/models/entity/specifications")
|
||||
.param("start", "0")
|
||||
.param("count", "100")
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.start", is(0)))
|
||||
.andExpect(jsonPath("$.count", greaterThanOrEqualTo(50)))
|
||||
.andExpect(jsonPath("$.total", greaterThanOrEqualTo(50)))
|
||||
.andExpect(jsonPath("$.elements.length()", greaterThanOrEqualTo(50)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetEntitySpecsUnauthorized() throws Exception {
|
||||
// Setup authorization to return false
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(false);
|
||||
|
||||
// Execute test
|
||||
mockMvc
|
||||
.perform(
|
||||
get("/openapi/v1/registry/models/entity/specifications")
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isForbidden());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetEntitySpecByNameWithAuthorization() throws Exception {
|
||||
// Setup authorization to return true
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(true);
|
||||
|
||||
// Execute test
|
||||
mockMvc
|
||||
.perform(
|
||||
get("/openapi/v1/registry/models/entity/specifications/{entityName}", TEST_ENTITY_NAME)
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.name", is(TEST_ENTITY_NAME)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetEntitySpecByNameNotFound() throws Exception {
|
||||
// Setup authorization to return true
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(true);
|
||||
|
||||
// Execute test
|
||||
mockMvc
|
||||
.perform(
|
||||
get(
|
||||
"/openapi/v1/registry/models/entity/specifications/{entityName}",
|
||||
NON_EXISTENT_ENTITY)
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isNotFound());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetEntitySpecByNameUnauthorized() throws Exception {
|
||||
// Setup authorization to return false
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(false);
|
||||
|
||||
// Execute test
|
||||
mockMvc
|
||||
.perform(
|
||||
get("/openapi/v1/registry/models/entity/specifications/{entityName}", TEST_ENTITY_NAME)
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isForbidden());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetEntityAspectSpecsWithAuthorization() throws Exception {
|
||||
// Setup authorization to return true
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(true);
|
||||
|
||||
// Execute test
|
||||
mockMvc
|
||||
.perform(
|
||||
get(
|
||||
"/openapi/v1/registry/models/entity/specifications/{entityName}/aspects",
|
||||
TEST_ENTITY_NAME)
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$." + TEST_ASPECT_NAME).exists());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetEntityAspectSpecsNotFound() throws Exception {
|
||||
// Setup authorization to return true
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(true);
|
||||
|
||||
// Execute test
|
||||
mockMvc
|
||||
.perform(
|
||||
get(
|
||||
"/openapi/v1/registry/models/entity/specifications/{entityName}/aspects",
|
||||
NON_EXISTENT_ENTITY)
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isNotFound());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetEntityAspectSpecsUnauthorized() throws Exception {
|
||||
// Setup authorization to return false
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(false);
|
||||
|
||||
// Execute test
|
||||
mockMvc
|
||||
.perform(
|
||||
get(
|
||||
"/openapi/v1/registry/models/entity/specifications/{entityName}/aspects",
|
||||
TEST_ENTITY_NAME)
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isForbidden());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAspectSpecsWithAuthorizationDefaultPagination() throws Exception {
|
||||
// Setup authorization to return true
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(true);
|
||||
|
||||
// Execute test
|
||||
mockMvc
|
||||
.perform(
|
||||
get("/openapi/v1/registry/models/aspect/specifications")
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.start", is(0)))
|
||||
.andExpect(jsonPath("$.count", is(20)))
|
||||
.andExpect(jsonPath("$.total", greaterThanOrEqualTo(200)))
|
||||
.andExpect(jsonPath("$.elements", hasSize(20)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAspectSpecsWithCustomPagination() throws Exception {
|
||||
// Setup authorization to return true
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(true);
|
||||
|
||||
// Execute test
|
||||
mockMvc
|
||||
.perform(
|
||||
get("/openapi/v1/registry/models/aspect/specifications")
|
||||
.param("start", "5")
|
||||
.param("count", "10")
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.start", is(5)))
|
||||
.andExpect(jsonPath("$.count", is(10)))
|
||||
.andExpect(jsonPath("$.total", greaterThanOrEqualTo(200)))
|
||||
.andExpect(jsonPath("$.elements", hasSize(10)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAspectSpecsUnauthorized() throws Exception {
|
||||
// Setup authorization to return false
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(false);
|
||||
|
||||
// Execute test
|
||||
mockMvc
|
||||
.perform(
|
||||
get("/openapi/v1/registry/models/aspect/specifications")
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isForbidden());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAspectSpecByNameWithAuthorization() throws Exception {
|
||||
// Setup authorization to return true
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(true);
|
||||
|
||||
// Execute test
|
||||
mockMvc
|
||||
.perform(
|
||||
get("/openapi/v1/registry/models/aspect/specifications/{aspectName}", TEST_ASPECT_NAME)
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.aspectAnnotation.name", is(TEST_ASPECT_NAME)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAspectSpecByNameNotFound() throws Exception {
|
||||
// Setup authorization to return true
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(true);
|
||||
|
||||
// Execute test
|
||||
mockMvc
|
||||
.perform(
|
||||
get(
|
||||
"/openapi/v1/registry/models/aspect/specifications/{aspectName}",
|
||||
NON_EXISTENT_ASPECT)
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isNotFound());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAspectSpecByNameUnauthorized() throws Exception {
|
||||
// Setup authorization to return false
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(false);
|
||||
|
||||
// Execute test
|
||||
mockMvc
|
||||
.perform(
|
||||
get("/openapi/v1/registry/models/aspect/specifications/{aspectName}", TEST_ASPECT_NAME)
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isForbidden());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetEventSpecsWithAuthorizationNoPagination() throws Exception {
|
||||
// TODO: Revisit when there are events
|
||||
|
||||
// Setup authorization to return true
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(true);
|
||||
|
||||
// Execute test - no pagination params should return all
|
||||
mockMvc
|
||||
.perform(
|
||||
get("/openapi/v1/registry/models/event/specifications")
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.start", is(0)))
|
||||
.andExpect(jsonPath("$.count", is(0)))
|
||||
.andExpect(jsonPath("$.total", greaterThanOrEqualTo(0)))
|
||||
.andExpect(jsonPath("$.elements.length()", greaterThanOrEqualTo(0)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetEventSpecsWithPagination() throws Exception {
|
||||
// TODO: Revisit when we have these specs
|
||||
|
||||
// Setup authorization to return true
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(true);
|
||||
|
||||
// Execute test
|
||||
mockMvc
|
||||
.perform(
|
||||
get("/openapi/v1/registry/models/event/specifications")
|
||||
.param("start", "2")
|
||||
.param("count", "3")
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.start", is(0)))
|
||||
.andExpect(jsonPath("$.count", is(0)))
|
||||
.andExpect(jsonPath("$.total", greaterThanOrEqualTo(0)))
|
||||
.andExpect(jsonPath("$.elements.length()", greaterThanOrEqualTo(0)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetEventSpecsUnauthorized() throws Exception {
|
||||
// Setup authorization to return false
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(false);
|
||||
|
||||
// Execute test
|
||||
mockMvc
|
||||
.perform(
|
||||
get("/openapi/v1/registry/models/event/specifications")
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isForbidden());
|
||||
}
|
||||
|
||||
@Test(
|
||||
expectedExceptions = ServletException.class,
|
||||
expectedExceptionsMessageRegExp =
|
||||
"Request processing failed: java.lang.IllegalArgumentException: Start offset 1000000000 exceeds total .*")
|
||||
public void testPaginationWithStartBeyondTotal() throws Exception {
|
||||
// Setup authorization to return true
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(true);
|
||||
|
||||
// Execute test with start beyond total items
|
||||
mockMvc
|
||||
.perform(
|
||||
get("/openapi/v1/registry/models/aspect/specifications")
|
||||
.param("start", "1000000000")
|
||||
.param("count", "20")
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isBadRequest());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPaginationWithZeroCount() throws Exception {
|
||||
// Setup authorization to return true
|
||||
authUtilMock
|
||||
.when(
|
||||
() ->
|
||||
AuthUtil.isAPIOperationsAuthorized(
|
||||
any(OperationContext.class), any(PoliciesConfig.Privilege.class)))
|
||||
.thenReturn(true);
|
||||
|
||||
// Execute test with count = 0
|
||||
mockMvc
|
||||
.perform(
|
||||
get("/openapi/v1/registry/models/aspect/specifications")
|
||||
.param("start", "0")
|
||||
.param("count", "0")
|
||||
.accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.start", is(0)))
|
||||
.andExpect(jsonPath("$.count", is(0)))
|
||||
.andExpect(jsonPath("$.total", greaterThanOrEqualTo(200)))
|
||||
.andExpect(jsonPath("$.elements", hasSize(0)));
|
||||
}
|
||||
}
|
||||
@ -32,7 +32,6 @@ public class LineageRegistryControllerTest {
|
||||
private MockedStatic<AuthenticationContext> authContextMock;
|
||||
|
||||
private static final String TEST_ENTITY_NAME = "dataset";
|
||||
private static final String TEST_DOWNSTREAM_ENTITY = "chart";
|
||||
|
||||
@BeforeMethod
|
||||
public void setup() {
|
||||
|
||||
@ -14,6 +14,7 @@ import com.fasterxml.jackson.core.StreamReadConstraints;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies;
|
||||
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
|
||||
import com.linkedin.r2.transport.http.server.RAPJakartaServlet;
|
||||
import com.linkedin.restli.server.RestliHandlerServlet;
|
||||
import io.datahubproject.iceberg.catalog.rest.common.IcebergJsonConverter;
|
||||
@ -148,6 +149,7 @@ public class ServletConfig implements WebMvcConfigurer {
|
||||
.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build());
|
||||
objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
|
||||
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
objectMapper.registerModule(new Jdk8Module());
|
||||
MappingJackson2HttpMessageConverter jsonConverter =
|
||||
new MappingJackson2HttpMessageConverter(objectMapper);
|
||||
messageConverters.add(jsonConverter);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user