Fixes #1984 - Remove unused code in Json related utils (#1985)

This commit is contained in:
Suresh Srinivas 2021-12-31 11:52:11 -08:00 committed by GitHub
parent c272ff7254
commit a649e723cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 12 additions and 456 deletions

View File

@ -13,7 +13,6 @@
package org.openmetadata.catalog.jdbi3;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.net.URI;
import java.util.Date;
@ -40,12 +39,6 @@ public class BotsRepository extends EntityRepository<Bots> {
Fields.EMPTY_FIELDS);
}
public Bots insert(Bots bots) throws JsonProcessingException {
bots.setHref(null);
daoCollection.botsDAO().insert(bots);
return bots;
}
@Override
public Bots setFields(Bots entity, Fields fields) {
return entity;

View File

@ -1,35 +0,0 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public final class EventUtils {
private static final Logger LOG = LoggerFactory.getLogger(EventUtils.class);
private EventUtils() {}
public static void publishEntityCreatedEvent(String entity, String entityName, String event) {
String print = String.format("Entity Created: [%s] Name: [%s] Event: [%s]", entity, entityName, event);
LOG.info(print);
}
public static void publishEntityUpdatedEvent(String entity, String entityName, String oldEvent, String newEvent) {
String diff = JsonUtils.diffTwoJson(oldEvent, newEvent);
String print = String.format("Entity Updated: [%s] Name: [%s] DiffString: [%s]", entity, entityName, diff);
LOG.info(print);
}
}

View File

@ -14,27 +14,17 @@
package org.openmetadata.catalog.util;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.type.TypeFactory;
import com.fasterxml.jackson.datatype.jsr353.JSR353Module;
import com.networknt.schema.JsonMetaSchema;
import com.networknt.schema.JsonSchema;
import com.networknt.schema.JsonSchemaFactory;
import com.networknt.schema.ValidationMessage;
import com.networknt.schema.urn.URNFactory;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.io.StringWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.json.Json;
import javax.json.JsonArray;
import javax.json.JsonArrayBuilder;
@ -42,17 +32,10 @@ import javax.json.JsonObject;
import javax.json.JsonPatch;
import javax.json.JsonStructure;
import javax.json.JsonValue;
import javax.json.JsonWriter;
import javax.json.JsonWriterFactory;
import javax.json.stream.JsonGenerator;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public final class JsonUtils {
public static final MediaType DEFAULT_MEDIA_TYPE = MediaType.APPLICATION_JSON_TYPE;
private static final Logger LOG = LoggerFactory.getLogger(JsonUtils.class);
private static final ObjectMapper OBJECT_MAPPER;
static {
@ -65,64 +48,6 @@ public final class JsonUtils {
private JsonUtils() {}
public static Set<ValidationMessage> validate(InputStream schemaStream, String jsonPayload) throws IOException {
return validate(schemaStream, jsonPayload, null);
}
public static Set<ValidationMessage> validate(InputStream schemaStream, String jsonPayload, URNFactory urnFactory)
throws IOException {
JsonSchemaFactory.Builder builder = new JsonSchemaFactory.Builder();
JsonMetaSchema metaSchema = JsonMetaSchema.getV7();
builder.defaultMetaSchemaURI(metaSchema.getUri()).addMetaSchema(metaSchema);
if (urnFactory != null) {
builder.addUrnFactory(urnFactory);
}
JsonSchemaFactory factory = builder.build();
JsonSchema schema = factory.getSchema(schemaStream);
ObjectMapper mapper = new ObjectMapper();
JsonNode node = mapper.readTree(jsonPayload);
return schema.validate(node);
}
public static String diffTwoJson(String v1, String v2) {
JsonValue source = Json.createReader(new StringReader(v1)).readValue();
JsonValue dest = Json.createReader(new StringReader(v2)).readValue();
JsonPatch diff = Json.createDiff(source.asJsonObject(), dest.asJsonObject());
return formatJson(diff.toJsonArray());
}
public static String formatJson(JsonValue jsonValue) {
StringWriter stringWriter = new StringWriter();
prettyPrintString(jsonValue, stringWriter);
return stringWriter.toString();
}
public static void prettyPrintString(JsonValue jsonValue, Writer writer) {
Map<String, Object> config = Collections.singletonMap(JsonGenerator.PRETTY_PRINTING, true);
JsonWriterFactory writerFactory = Json.createWriterFactory(config);
try (JsonWriter jsonWriter = writerFactory.createWriter(writer)) {
jsonWriter.write(jsonValue);
}
}
public static <T> T getEntity(WebTarget target, Class<T> clazz) {
return getEntity(target, DEFAULT_MEDIA_TYPE, clazz);
}
public static <T> T getEntity(WebTarget target, MediaType mediaType, Class<T> clazz) {
try {
String response = target.request(mediaType).get(String.class);
JsonNode node = OBJECT_MAPPER.readTree(response);
return OBJECT_MAPPER.treeToValue(node, clazz);
} catch (Exception ex) {
LOG.error("Error while Calling URI : {}", target.getUri());
throw new RuntimeException(ex);
}
}
public static String pojoToJson(Object o) throws JsonProcessingException {
if (o == null) {
return null;
@ -234,4 +159,12 @@ public final class JsonUtils {
public static <T> T convertValue(JsonValue patched, Class<T> clz) {
return OBJECT_MAPPER.convertValue(patched, clz);
}
public static JsonPatch getJsonPatch(String v1, String v2) {
System.out.println(v1);
System.out.println(v2);
JsonValue source = Json.createReader(new StringReader(v1)).readValue();
JsonValue dest = Json.createReader(new StringReader(v2)).readValue();
return Json.createDiff(source.asJsonObject(), dest.asJsonObject());
}
}

View File

@ -108,7 +108,6 @@ import org.openmetadata.catalog.util.JsonUtils;
import org.openmetadata.catalog.util.RestUtil;
import org.openmetadata.catalog.util.ResultList;
import org.openmetadata.catalog.util.TestUtils;
import org.openmetadata.common.utils.JsonSchemaUtil;
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public abstract class EntityResourceTest<T> extends CatalogApplicationTest {
@ -861,7 +860,7 @@ public abstract class EntityResourceTest<T> extends CatalogApplicationTest {
protected final T patchEntity(UUID id, String originalJson, T updated, Map<String, String> authHeaders)
throws JsonProcessingException, HttpResponseException {
String updatedEntityJson = JsonUtils.pojoToJson(updated);
JsonPatch patch = JsonSchemaUtil.getJsonPatch(originalJson, updatedEntityJson);
JsonPatch patch = JsonUtils.getJsonPatch(originalJson, updatedEntityJson);
return TestUtils.patch(getResource(id), patch, entityClass, authHeaders);
}

View File

@ -45,7 +45,6 @@ import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.JsonUtils;
import org.openmetadata.catalog.util.TestUtils;
import org.openmetadata.common.utils.JsonSchemaUtil;
public class RoleResourceTest extends EntityResourceTest<Role> {
@ -166,7 +165,7 @@ public class RoleResourceTest extends EntityResourceTest<Role> {
private Role patchRole(UUID roleId, String originalJson, Role updated, Map<String, String> authHeaders)
throws JsonProcessingException, HttpResponseException {
String updatedJson = JsonUtils.pojoToJson(updated);
JsonPatch patch = JsonSchemaUtil.getJsonPatch(originalJson, updatedJson);
JsonPatch patch = JsonUtils.getJsonPatch(originalJson, updatedJson);
return TestUtils.patch(CatalogApplicationTest.getResource("roles/" + roleId), patch, Role.class, authHeaders);
}

View File

@ -59,7 +59,6 @@ import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.JsonUtils;
import org.openmetadata.catalog.util.TestUtils;
import org.openmetadata.common.utils.JsonSchemaUtil;
public class TeamResourceTest extends EntityResourceTest<Team> {
final Profile PROFILE = new Profile().withImages(new ImageList().withImage(URI.create("http://image.com")));
@ -316,7 +315,7 @@ public class TeamResourceTest extends EntityResourceTest<Team> {
private Team patchTeam(UUID teamId, String originalJson, Team updated, Map<String, String> authHeaders)
throws JsonProcessingException, HttpResponseException {
String updatedJson = JsonUtils.pojoToJson(updated);
JsonPatch patch = JsonSchemaUtil.getJsonPatch(originalJson, updatedJson);
JsonPatch patch = JsonUtils.getJsonPatch(originalJson, updatedJson);
return TestUtils.patch(CatalogApplicationTest.getResource("teams/" + teamId), patch, Team.class, authHeaders);
}

View File

@ -77,7 +77,6 @@ import org.openmetadata.catalog.util.JsonUtils;
import org.openmetadata.catalog.util.ResultList;
import org.openmetadata.catalog.util.TestUtils;
import org.openmetadata.catalog.util.TestUtils.UpdateType;
import org.openmetadata.common.utils.JsonSchemaUtil;
public class UserResourceTest extends EntityResourceTest<User> {
final Profile PROFILE = new Profile().withImages(new ImageList().withImage(URI.create("http://image.com")));
@ -489,7 +488,7 @@ public class UserResourceTest extends EntityResourceTest<User> {
private User patchUser(UUID userId, String originalJson, User updated, Map<String, String> headers)
throws JsonProcessingException, HttpResponseException {
String updatedJson = JsonUtils.pojoToJson(updated);
JsonPatch patch = JsonSchemaUtil.getJsonPatch(originalJson, updatedJson);
JsonPatch patch = JsonUtils.getJsonPatch(originalJson, updatedJson);
return TestUtils.patch(CatalogApplicationTest.getResource("users/" + userId), patch, User.class, headers);
}

View File

@ -1,122 +0,0 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.common.utils;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.kjetland.jackson.jsonSchema.JsonSchemaConfig;
import com.kjetland.jackson.jsonSchema.JsonSchemaDraft;
import com.kjetland.jackson.jsonSchema.JsonSchemaGenerator;
import com.networknt.schema.JsonMetaSchema;
import com.networknt.schema.JsonSchema;
import com.networknt.schema.JsonSchemaFactory;
import com.networknt.schema.ValidationMessage;
import com.networknt.schema.uri.ClasspathURLFactory;
import com.networknt.schema.urn.URNFactory;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.io.StringWriter;
import java.io.Writer;
import java.net.URL;
import java.util.Collections;
import java.util.Map;
import java.util.Set;
import javax.json.Json;
import javax.json.JsonPatch;
import javax.json.JsonValue;
import javax.json.JsonWriter;
import javax.json.JsonWriterFactory;
import javax.json.stream.JsonGenerator;
public final class JsonSchemaUtil {
private JsonSchemaUtil() {}
/** Return JSON schema from a POJO annotated appropriately */
public static <T> String jsonSchemaForClass(Class<T> c) throws JsonProcessingException {
ObjectMapper mapper = new ObjectMapper();
JsonSchemaConfig config = JsonSchemaConfig.vanillaJsonSchemaDraft4().withJsonSchemaDraft(JsonSchemaDraft.DRAFT_07);
JsonSchemaGenerator schemaGen = new JsonSchemaGenerator(mapper, config);
JsonNode jsonSchema = schemaGen.generateJsonSchema(c);
return mapper.writeValueAsString(jsonSchema);
}
/** URN factory that maps the json schema URL to internal resource based URL to be used for testing purposes */
public static URNFactory getUrnFactory() {
return urn -> {
try {
// Turn urn in relative path format "../type/common.json into absolute path /json/type/common.json
urn = urn.replace("../type", "json/type");
URL absoluteURL =
ClasspathURLFactory.convert(new ClasspathURLFactory().create(String.format("resource:/%s", urn)));
return absoluteURL.toURI();
} catch (Exception ex) {
return null;
}
};
}
public static Set<ValidationMessage> validate(InputStream schemaStream, String jsonPayload) throws IOException {
return validate(schemaStream, jsonPayload, null);
}
public static Set<ValidationMessage> validate(InputStream schemaStream, String jsonPayload, URNFactory urnFactory)
throws IOException {
JsonSchemaFactory.Builder builder = new JsonSchemaFactory.Builder();
JsonMetaSchema metaSchema = JsonMetaSchema.getV7();
builder.defaultMetaSchemaURI(metaSchema.getUri()).addMetaSchema(metaSchema);
if (urnFactory != null) {
builder.addUrnFactory(urnFactory);
}
JsonSchemaFactory factory = builder.build();
JsonSchema schema = factory.getSchema(schemaStream);
ObjectMapper mapper = new ObjectMapper();
JsonNode node = mapper.readTree(jsonPayload);
return schema.validate(node);
}
public static JsonPatch getJsonPatch(String v1, String v2) {
System.out.println(v1);
System.out.println(v2);
JsonValue source = Json.createReader(new StringReader(v1)).readValue();
JsonValue dest = Json.createReader(new StringReader(v2)).readValue();
return Json.createDiff(source.asJsonObject(), dest.asJsonObject());
}
public static String diffTwoJson(String v1, String v2) {
JsonValue source = Json.createReader(new StringReader(v1)).readValue();
JsonValue dest = Json.createReader(new StringReader(v2)).readValue();
JsonPatch diff = Json.createDiff(source.asJsonObject(), dest.asJsonObject());
return formatJson(diff.toJsonArray());
}
public static String formatJson(JsonValue jsonValue) {
StringWriter stringWriter = new StringWriter();
prettyPrintString(jsonValue, stringWriter);
return stringWriter.toString();
}
public static void prettyPrintString(JsonValue jsonValue, Writer writer) {
Map<String, Object> config = Collections.singletonMap(JsonGenerator.PRETTY_PRINTING, true);
JsonWriterFactory writerFactory = Json.createWriterFactory(config);
try (JsonWriter jsonWriter = writerFactory.createWriter(writer)) {
jsonWriter.write(jsonValue);
}
}
}

View File

@ -1,36 +0,0 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.common.utils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Test;
public class JsonDiffTest {
@Test
public void generateAndPrintDiff() throws IOException {
String v1 =
IOUtils.toString(
JsonDiffTest.class.getClassLoader().getResourceAsStream("json/dim_location/v1.json"),
StandardCharsets.UTF_8);
String v2 =
IOUtils.toString(
JsonDiffTest.class.getClassLoader().getResourceAsStream("json/dim_location/v2.json"),
StandardCharsets.UTF_8);
String diff = JsonSchemaUtil.diffTwoJson(v1, v2);
System.out.println(diff);
}
}

View File

@ -1,173 +0,0 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.common.utils;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonPropertyDescription;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.networknt.schema.ValidationMessage;
import com.networknt.schema.urn.URNFactory;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/**
* This test provides examples of how to use: - JSON schema to validate the JSON payload - Generate JSON schema from
* POJO
*/
public class JsonSchemaTest {
private final UUID TEST_UUID = UUID.randomUUID();
private final URI TEST_URI = URI.create("http://test.com");
private URNFactory urnFactory;
@BeforeEach
public void setup() {
urnFactory = JsonSchemaUtil.getUrnFactory();
}
private java.util.Set<com.networknt.schema.ValidationMessage> validate(InputStream in, String jsonPayload)
throws IOException {
System.out.println("Validating " + jsonPayload);
Set<ValidationMessage> errors = JsonSchemaUtil.validate(in, jsonPayload, urnFactory);
System.out.println("Errors " + errors);
return errors;
}
/** Validate a conforming JSON payload using JSON schema */
@Test
public void validJson() throws IOException {
// Valid jsonPayload
InputStream in = JsonSchemaTest.class.getClassLoader().getResourceAsStream("json/entity/testEntity.json");
Map<String, Object> objectTypeMap =
Map.of(
"ot1", "ot1",
"ot2", "ot2");
Map<String, Object> map =
Map.of(
"stringProperty", "property1",
"uriProperty", TEST_URI,
"uuidProperty", TEST_UUID,
"objectTypeProperty", objectTypeMap);
String jsonPayload = new ObjectMapper().writeValueAsString(map);
assertEquals(0, validate(in, jsonPayload).size());
}
/** Validate a non-conforming JSON payload that is missing a required field using JSON schema */
@Test
public void missingField() throws IOException {
InputStream in = JsonSchemaTest.class.getClassLoader().getResourceAsStream("json/entity/testEntity.json");
// No mandatory field "stringProperty"
Map<String, Object> objectTypeMap =
Map.of(
"ot1", "ot1",
"ot2", "ot2");
Map<String, Object> map =
Map.of(
// Missing stringProperty1
"uriProperty", TEST_URI,
"uuidProperty", TEST_UUID,
"objectTypeProperty", objectTypeMap);
String jsonPayload = new ObjectMapper().writeValueAsString(map);
Set<ValidationMessage> errors = validate(in, jsonPayload);
assertEquals(1, errors.size());
assertTrue(errors.iterator().next().getMessage().contains("stringProperty: is missing"));
}
/** Validate a non-conforming JSON payload that is missing a required inner field using JSON schema */
@Test
public void missingInnerField() throws IOException {
// No mandatory inner field "objectType.ot1"
InputStream in = JsonSchemaTest.class.getClassLoader().getResourceAsStream("json/entity/testEntity.json");
Map<String, Object> objectTypeMap =
Map.of(
// Missing inner field ot1
"ot2", "ot2");
Map<String, Object> map =
Map.of(
"stringProperty", "property1",
"uriProperty", TEST_URI,
"uuidProperty", TEST_UUID,
"objectTypeProperty", objectTypeMap);
String jsonPayload = new ObjectMapper().writeValueAsString(map);
Set<ValidationMessage> errors = validate(in, jsonPayload);
assertEquals(1, errors.size());
assertTrue(errors.iterator().next().getMessage().contains("ot1: is missing"));
}
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonPropertyOrder({"name", "id", "connection"})
/** Validate a non-conforming JSON payload that has invalid data */
@Test
public void invalidJsonData() throws IOException {
// Invalid value that is not a URI
InputStream in = JsonSchemaTest.class.getClassLoader().getResourceAsStream("json/entity/testEntity.json");
Map<String, Object> objectTypeMap = Map.of("ot1", "ot2", "ot2", "ot2");
Map<String, Object> map =
Map.of(
"stringProperty",
"property1",
"uriProperty",
"invalidUri", // Invalid URI
"uuidProperty",
TEST_UUID,
"objectTypeProperty",
objectTypeMap);
String jsonPayload = new ObjectMapper().writeValueAsString(map);
Set<ValidationMessage> errors = validate(in, jsonPayload);
assertEquals(1, errors.size());
assertTrue(errors.iterator().next().getMessage().contains("uriProperty: does not match the uri pattern"));
}
/** Test POJO to JSON schema */
@Test
public void pojoToJsonSchema() throws IOException {
// From POJO class generate json schema
String jsonData = JsonSchemaUtil.jsonSchemaForClass(Pojo.class);
ObjectMapper mapper = new ObjectMapper();
Map<String, Object> jsonMap = mapper.readValue(jsonData, new TypeReference<>() {});
System.out.println(jsonMap);
assertEquals("http://json-schema.org/draft-07/schema#", jsonMap.get("$schema"));
assertEquals("Pojo", jsonMap.get("title"));
assertEquals("object", jsonMap.get("type"));
// Check properties of the object
@SuppressWarnings("unchecked")
Map<String, Object> propertiesMap = (Map<String, Object>) jsonMap.get("properties");
assertEquals("{type=string, description=TODO}", propertiesMap.get("name").toString());
}
static class Pojo {
@JsonProperty("name")
@JsonPropertyDescription("TODO")
private String name;
@JsonProperty("id")
@JsonPropertyDescription("Type used for UUID")
private UUID id;
}
}