From 6b137a862be757e8af9416246e61ba847233cfe2 Mon Sep 17 00:00:00 2001 From: Kerem Sahin Date: Wed, 4 Dec 2019 17:51:52 -0800 Subject: [PATCH 1/3] Removing unnecessary classes for mxe-registration --- gms/impl/build.gradle | 2 +- .../linkedin/metadata/dao/utils/ESUtils.java | 7 +- .../metadata/dao/utils/SearchUtils.java | 2 +- .../com/linkedin/metadata/dao/Neo4jUtil.java | 16 ++--- .../DatasetSnapshotRequestBuilderTest.java | 2 +- .../metadata/dao/utils/QueryUtilTest.java | 2 +- metadata-events/mxe-registration/build.gradle | 62 ----------------- .../main/java/com/linkedin/mxe/Configs.java | 41 ------------ .../com/linkedin/mxe/RegisterSchemas.java | 42 ------------ .../main/java/com/linkedin/mxe/Topics.java | 4 -- .../mxe/SchemaCompatibilityTests.java | 66 ------------------- 11 files changed, 14 insertions(+), 232 deletions(-) delete mode 100644 metadata-events/mxe-registration/src/main/java/com/linkedin/mxe/Configs.java delete mode 100644 metadata-events/mxe-registration/src/main/java/com/linkedin/mxe/RegisterSchemas.java delete mode 100644 metadata-events/mxe-registration/src/validation/java/com/linkedin/mxe/SchemaCompatibilityTests.java diff --git a/gms/impl/build.gradle b/gms/impl/build.gradle index 47db026238..d876449c63 100644 --- a/gms/impl/build.gradle +++ b/gms/impl/build.gradle @@ -28,4 +28,4 @@ pegasus.main.idlOptions.addIdlItem([ 'com.linkedin.identity.rest' ]) -ext.apiProject = project(':gms:api') +ext.apiProject = project(':gms:api') \ No newline at end of file diff --git a/metadata-dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/utils/ESUtils.java b/metadata-dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/utils/ESUtils.java index 2382d59fcd..882a2eec93 100644 --- a/metadata-dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/utils/ESUtils.java +++ b/metadata-dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/utils/ESUtils.java @@ -3,8 +3,6 @@ package com.linkedin.metadata.dao.utils; import java.util.Arrays; import java.util.Map; import javax.annotation.Nonnull; - -import com.linkedin.common.urn.Urn; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -28,11 +26,10 @@ public class ESUtils { BoolQueryBuilder boolFilter = new BoolQueryBuilder(); for (Map.Entry entry : requestMap.entrySet()) { BoolQueryBuilder filters = new BoolQueryBuilder(); - // TODO: Remove checking for urn after solving META-10102 - Arrays.stream(Urn.isUrn(entry.getValue()) ? new String[]{entry.getValue()} : entry.getValue().split(",")) + Arrays.stream(entry.getValue().split(",")) .forEach(elem -> filters.should(QueryBuilders.matchQuery(entry.getKey(), elem))); boolFilter.must(filters); } return boolFilter; } -} \ No newline at end of file +} diff --git a/metadata-dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/utils/SearchUtils.java b/metadata-dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/utils/SearchUtils.java index 16eefc51a9..88f62ba997 100644 --- a/metadata-dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/utils/SearchUtils.java +++ b/metadata-dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/utils/SearchUtils.java @@ -45,7 +45,7 @@ public class SearchUtils { } /** - * Convert a requestMap to a filter + * Converts a requestMap to a filter * * @param requestMap a map of fields and values * @return the search filter diff --git a/metadata-dao-impl/neo4j-dao/src/main/java/com/linkedin/metadata/dao/Neo4jUtil.java b/metadata-dao-impl/neo4j-dao/src/main/java/com/linkedin/metadata/dao/Neo4jUtil.java index bcda6af598..5b23f4caf8 100644 --- a/metadata-dao-impl/neo4j-dao/src/main/java/com/linkedin/metadata/dao/Neo4jUtil.java +++ b/metadata-dao-impl/neo4j-dao/src/main/java/com/linkedin/metadata/dao/Neo4jUtil.java @@ -256,7 +256,7 @@ public class Neo4jUtil { */ @Nonnull public static RelationshipFilter createRelationshipFilter(@Nonnull Filter filter, - @Nonnull RelationshipDirection relationshipDirection) { + @Nonnull RelationshipDirection relationshipDirection) { return new RelationshipFilter().setCriteria(filter.getCriteria()).setDirection(relationshipDirection); } @@ -270,7 +270,7 @@ public class Neo4jUtil { */ @Nonnull public static RelationshipFilter createRelationshipFilter(@Nonnull String field, @Nonnull String value, - @Nonnull RelationshipDirection relationshipDirection) { + @Nonnull RelationshipDirection relationshipDirection) { return createRelationshipFilter(createFilter(field, value), relationshipDirection); } @@ -284,10 +284,10 @@ public class Neo4jUtil { @Nonnull public static Filter createFilter(@Nonnull String field, @Nonnull String value) { return new Filter() - .setCriteria( - new CriterionArray(Collections.singletonList( - new Criterion().setField(field).setValue(value).setCondition(Condition.EQUAL) - )) - ); + .setCriteria( + new CriterionArray(Collections.singletonList( + new Criterion().setField(field).setValue(value).setCondition(Condition.EQUAL) + )) + ); } -} +} \ No newline at end of file diff --git a/metadata-dao-impl/restli-dao/src/test/java/com/linkedin/metadata/dao/DatasetSnapshotRequestBuilderTest.java b/metadata-dao-impl/restli-dao/src/test/java/com/linkedin/metadata/dao/DatasetSnapshotRequestBuilderTest.java index 1898b241fe..a32329bad9 100644 --- a/metadata-dao-impl/restli-dao/src/test/java/com/linkedin/metadata/dao/DatasetSnapshotRequestBuilderTest.java +++ b/metadata-dao-impl/restli-dao/src/test/java/com/linkedin/metadata/dao/DatasetSnapshotRequestBuilderTest.java @@ -24,4 +24,4 @@ public class DatasetSnapshotRequestBuilderTest extends BaseSnapshotRequestBuilde assertEquals(builder.urnClass(), DatasetUrn.class); } -} +} \ No newline at end of file diff --git a/metadata-dao/src/test/java/com/linkedin/metadata/dao/utils/QueryUtilTest.java b/metadata-dao/src/test/java/com/linkedin/metadata/dao/utils/QueryUtilTest.java index 838ebe855c..fd4854d2f0 100644 --- a/metadata-dao/src/test/java/com/linkedin/metadata/dao/utils/QueryUtilTest.java +++ b/metadata-dao/src/test/java/com/linkedin/metadata/dao/utils/QueryUtilTest.java @@ -87,4 +87,4 @@ public class QueryUtilTest { private AspectVersion makeAspectVersion(String aspectName, long version) { return new AspectVersion().setAspect(aspectName).setVersion(version); } -} +} \ No newline at end of file diff --git a/metadata-events/mxe-registration/build.gradle b/metadata-events/mxe-registration/build.gradle index 6e7baa2cbe..bbfeb03c22 100644 --- a/metadata-events/mxe-registration/build.gradle +++ b/metadata-events/mxe-registration/build.gradle @@ -1,63 +1 @@ apply plugin: 'java' - -configurations { - avro17 - - validationCompile - validationCompile.extendsFrom testCompile - - validationRuntime - validationRuntime.extendsFrom testRuntime -} - -dependencies { - compile externalDependency.kafkaSchemaRegistry - - compileOnly project(':metadata-events:mxe-avro-1.7') - - validationCompile externalDependency.avroMigrationHelper - validationCompile externalDependency.testng - - avro17 project(':metadata-events:mxe-avro-1.7') -} - -////////////////////////////////////// -// Schema compatibility validation -////////////////////////////////////// -sourceSets { - validation { - java { - compileClasspath += main.output + test.output - runtimeClasspath += main.output + test.output - srcDir file('src/validation/java') - } - } -} - -idea { - module { - testSourceDirs += file("src/validation/java") - } -} - -task validate17(type: Test, dependsOn: configurations.avro17) { - testLogging.events 'standard_out' - testClassesDirs = sourceSets.validation.output.classesDirs - classpath = files(sourceSets.validation.runtimeClasspath, configurations.avro17) - - // Prevent gradle from caching each validation run - outputs.upToDateWhen { false } -} - -// Make sure validate17 are run as part of gradle build -check.dependsOn += [validate17] - -////////////////////////////////////// -// Schema registration -////////////////////////////////////// -def mainClassName = 'com.linkedin.mxe.RegisterSchemas' - -task register17(type: JavaExec, dependsOn: [configurations.compile, configurations.avro17]) { - main = mainClassName - classpath = files(sourceSets.main.runtimeClasspath, configurations.avro17) -} diff --git a/metadata-events/mxe-registration/src/main/java/com/linkedin/mxe/Configs.java b/metadata-events/mxe-registration/src/main/java/com/linkedin/mxe/Configs.java deleted file mode 100644 index 714702821c..0000000000 --- a/metadata-events/mxe-registration/src/main/java/com/linkedin/mxe/Configs.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.linkedin.mxe; - -import com.linkedin.pegasus2avro.mxe.FailedMetadataChangeEvent; -import com.linkedin.pegasus2avro.mxe.MetadataAuditEvent; -import com.linkedin.pegasus2avro.mxe.MetadataChangeEvent; -import com.linkedin.pegasus2avro.mxe.MetadataGraphEvent; -import com.linkedin.pegasus2avro.mxe.MetadataSearchEvent; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import org.apache.avro.Schema; - - -public class Configs { - - public static final Map FABRIC_SCHEMA_REGISTRY_MAP = - Collections.unmodifiableMap(new HashMap() { - { - put("ei", "http://ei4-schemaregistry-vip-1.int.linkedin.com:10252"); - put("corp", "http://lca1-schema-registry-vip-1.corp.linkedin.com:10252"); - } - }); - - public static final Map TOPIC_SCHEMA_MAP = Collections.unmodifiableMap(new HashMap() { - { - put(Topics.METADATA_AUDIT_EVENT, MetadataAuditEvent.SCHEMA$); - put(Topics.METADATA_CHANGE_EVENT, MetadataChangeEvent.SCHEMA$); - put(Topics.FAILED_METADATA_CHANGE_EVENT, FailedMetadataChangeEvent.SCHEMA$); - put(Topics.METADATA_GRAPH_EVENT, MetadataGraphEvent.SCHEMA$); - put(Topics.METADATA_SEARCH_EVENT, MetadataSearchEvent.SCHEMA$); - - put(Topics.DEV_METADATA_AUDIT_EVENT, MetadataAuditEvent.SCHEMA$); - put(Topics.DEV_METADATA_CHANGE_EVENT, MetadataChangeEvent.SCHEMA$); - put(Topics.DEV_FAILED_METADATA_CHANGE_EVENT, FailedMetadataChangeEvent.SCHEMA$); - } - }); - - private Configs() { - // Util class - } -} diff --git a/metadata-events/mxe-registration/src/main/java/com/linkedin/mxe/RegisterSchemas.java b/metadata-events/mxe-registration/src/main/java/com/linkedin/mxe/RegisterSchemas.java deleted file mode 100644 index f849c9b14c..0000000000 --- a/metadata-events/mxe-registration/src/main/java/com/linkedin/mxe/RegisterSchemas.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.linkedin.mxe; - -import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; -import org.apache.avro.Schema; - - -public class RegisterSchemas { - - public static final String DEFAULT_SCHEMA_REGISTRY_URL = "http://localhost:8081"; - - private RegisterSchemas() { - } - - static CachedSchemaRegistryClient createClient(String url) { - return new CachedSchemaRegistryClient(url, Integer.MAX_VALUE); - } - - private static void registerSchema(String topic, Schema schema, CachedSchemaRegistryClient client) { - try { - client.register(topic, schema); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public static void registerSchema(String topic, String schemaRegistryUrl) { - CachedSchemaRegistryClient client = createClient(schemaRegistryUrl); - Schema schema = Configs.TOPIC_SCHEMA_MAP.get(topic); - System.out.println(String.format("Registering %s using registry %s (size: %d)", topic, schemaRegistryUrl, - schema.toString(false).length())); - registerSchema(topic, schema, client); - } - - public static void main(final String[] args) { - final String url = args.length == 1 ? args[0] : DEFAULT_SCHEMA_REGISTRY_URL; - Configs.TOPIC_SCHEMA_MAP.forEach((topic, schema) -> { - System.out.println(String.format("Registering %s using registry %s (size: %d)", topic, - url, schema.toString(false).length())); - registerSchema(topic, url); - }); - } -} diff --git a/metadata-events/mxe-registration/src/main/java/com/linkedin/mxe/Topics.java b/metadata-events/mxe-registration/src/main/java/com/linkedin/mxe/Topics.java index bfa01b81cc..eb4d7e9ffd 100644 --- a/metadata-events/mxe-registration/src/main/java/com/linkedin/mxe/Topics.java +++ b/metadata-events/mxe-registration/src/main/java/com/linkedin/mxe/Topics.java @@ -7,10 +7,6 @@ public class Topics { public static final String METADATA_GRAPH_EVENT = "MetadataGraphEvent"; public static final String METADATA_SEARCH_EVENT = "MetadataSearchEvent"; - public static final String DEV_METADATA_AUDIT_EVENT = "MetadataAuditEvent_dev"; - public static final String DEV_METADATA_CHANGE_EVENT = "MetadataChangeEvent_dev"; - public static final String DEV_FAILED_METADATA_CHANGE_EVENT = "FailedMetadataChangeEvent_dev"; - private Topics() { // Util class } diff --git a/metadata-events/mxe-registration/src/validation/java/com/linkedin/mxe/SchemaCompatibilityTests.java b/metadata-events/mxe-registration/src/validation/java/com/linkedin/mxe/SchemaCompatibilityTests.java deleted file mode 100644 index 993ab2e9d3..0000000000 --- a/metadata-events/mxe-registration/src/validation/java/com/linkedin/mxe/SchemaCompatibilityTests.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.linkedin.mxe; - -import com.linkedin.avro.legacy.LegacyAvroSchema; -import io.confluent.kafka.schemaregistry.avro.AvroCompatibilityChecker; -import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; -import io.confluent.kafka.schemaregistry.client.SchemaMetadata; -import org.apache.avro.*; -import org.testng.Reporter; -import org.testng.annotations.Test; - -import java.util.Random; - -import static com.linkedin.mxe.RegisterSchemas.DEFAULT_SCHEMA_REGISTRY_URL; -import static org.testng.Assert.*; - - -public class SchemaCompatibilityTests { - - private static final String FORWARD_INCOMPATIBILITY_WARNING = "" // format - + "************************************************************************\n" //format - + " New schema is forward incompatible with the current schema.\n" // format - + " Consider bumping up the major version.\n" // format - + "************************************************************************\n"; - - private static final AvroCompatibilityChecker AVRO_BACKWARD_COMPATIBILITY_CHECKER = - AvroCompatibilityChecker.BACKWARD_CHECKER; - - private static final AvroCompatibilityChecker AVRO_FORWARD_COMPATIBILITY_CHECKER = - AvroCompatibilityChecker.FORWARD_CHECKER; - - @Test - public void testBackwardCompatibility() { - final CachedSchemaRegistryClient client = RegisterSchemas.createClient(DEFAULT_SCHEMA_REGISTRY_URL); - - Configs.TOPIC_SCHEMA_MAP.forEach((topic, schema) -> { - Schema olderSchema = findLastRegisteredSchemaMetadata(topic, client); - if (olderSchema == null) { - Reporter.log("Unable to find registered schema for " + topic + true); - return; - } - - // Check backward compatibility, i.e. can new schema fits old data - assertTrue(AVRO_BACKWARD_COMPATIBILITY_CHECKER.isCompatible(schema, olderSchema), - "New schema is backward incompatible with the current schema \n\n"); - - // Check forward compatibility, i.e. can new data fits old schema - assertTrue(AVRO_FORWARD_COMPATIBILITY_CHECKER.isCompatible(schema, olderSchema), - FORWARD_INCOMPATIBILITY_WARNING + "\n\n"); - }); - } - - private Schema findLastRegisteredSchemaMetadata(String topic, CachedSchemaRegistryClient client) { - try { - SchemaMetadata metadata = client.getLatestSchemaMetadata(topic); - return client.getBySubjectAndID(topic, metadata.getId()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private SchemaMetadata makeSchemaMetadata(String topic, Schema schema) { - LegacyAvroSchema legacyAvroSchema = new LegacyAvroSchema(topic, schema.toString(false)); - Random rand = new Random(); - return new SchemaMetadata(rand.nextInt(), rand.nextInt(), legacyAvroSchema.toString()); - } -} From c302cd592f3d870d43634586a5834c8f17225037 Mon Sep 17 00:00:00 2001 From: Kerem Sahin Date: Thu, 5 Dec 2019 10:37:45 -0800 Subject: [PATCH 2/3] Move DatasetBrowseConfig to gms/impl --- .../com/linkedin/dataset/factory/DatasetBrowseDAOFactory.java | 2 +- .../com/linkedin/metadata/configs}/DatasetBrowseConfig.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) rename {metadata-dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/browse => gms/impl/src/main/java/com/linkedin/metadata/configs}/DatasetBrowseConfig.java (68%) diff --git a/gms/factories/src/main/java/com/linkedin/dataset/factory/DatasetBrowseDAOFactory.java b/gms/factories/src/main/java/com/linkedin/dataset/factory/DatasetBrowseDAOFactory.java index 32c5bdcce3..2eca49ca5d 100644 --- a/gms/factories/src/main/java/com/linkedin/dataset/factory/DatasetBrowseDAOFactory.java +++ b/gms/factories/src/main/java/com/linkedin/dataset/factory/DatasetBrowseDAOFactory.java @@ -1,6 +1,6 @@ package com.linkedin.dataset.factory; -import com.linkedin.metadata.dao.browse.DatasetBrowseConfig; +import com.linkedin.metadata.configs.DatasetBrowseConfig; import com.linkedin.metadata.dao.browse.ESBrowseDAO; import org.elasticsearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; diff --git a/metadata-dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/browse/DatasetBrowseConfig.java b/gms/impl/src/main/java/com/linkedin/metadata/configs/DatasetBrowseConfig.java similarity index 68% rename from metadata-dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/browse/DatasetBrowseConfig.java rename to gms/impl/src/main/java/com/linkedin/metadata/configs/DatasetBrowseConfig.java index 3fbfdf5d62..501c59eb21 100644 --- a/metadata-dao-impl/elasticsearch-dao/src/main/java/com/linkedin/metadata/dao/browse/DatasetBrowseConfig.java +++ b/gms/impl/src/main/java/com/linkedin/metadata/configs/DatasetBrowseConfig.java @@ -1,5 +1,6 @@ -package com.linkedin.metadata.dao.browse; +package com.linkedin.metadata.configs; +import com.linkedin.metadata.dao.browse.BaseBrowseConfig; import com.linkedin.metadata.search.DatasetDocument; public class DatasetBrowseConfig extends BaseBrowseConfig { From b7f305cbb44a5d6171b09c277692d7d636694f19 Mon Sep 17 00:00:00 2001 From: Kerem Sahin Date: Thu, 5 Dec 2019 11:20:50 -0800 Subject: [PATCH 3/3] Stop using PaginationContext --- .../linkedin/identity/client/CorpUsers.java | 8 ++--- .../events/search/PaginationContext.pdsc | 35 ------------------- 2 files changed, 4 insertions(+), 39 deletions(-) delete mode 100644 li-utils/src/main/pegasus/com/linkedin/avro2pegasus/events/search/PaginationContext.pdsc diff --git a/gms/client/src/main/java/com/linkedin/identity/client/CorpUsers.java b/gms/client/src/main/java/com/linkedin/identity/client/CorpUsers.java index 1dc9b1ea3e..87163bf4f8 100644 --- a/gms/client/src/main/java/com/linkedin/identity/client/CorpUsers.java +++ b/gms/client/src/main/java/com/linkedin/identity/client/CorpUsers.java @@ -1,6 +1,5 @@ package com.linkedin.identity.client; -import com.linkedin.avro2pegasus.events.search.PaginationContext; import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.identity.CorpUser; import com.linkedin.identity.CorpUserEditableInfo; @@ -85,15 +84,16 @@ public class CorpUsers extends BaseClient implements SearchableClient /** * Get all {@link CorpUser} models of the corp users * - * @param context pagination param + * @param start offset to start + * @param count number of max {@link CorpUser}s to return * @return {@link CorpUser} models of the corp user * @throws RemoteInvocationException */ @Nonnull - public List getAll(@Nonnull PaginationContext context) + public List getAll(int start, int count) throws RemoteInvocationException { final GetAllRequest getAllRequest = CORP_USERS_REQUEST_BUILDERS.getAll() - .paginate(context.getStart(), context.getNumToReturn()) + .paginate(start, count) .build(); return _client.sendRequest(getAllRequest).getResponseEntity().getElements(); } diff --git a/li-utils/src/main/pegasus/com/linkedin/avro2pegasus/events/search/PaginationContext.pdsc b/li-utils/src/main/pegasus/com/linkedin/avro2pegasus/events/search/PaginationContext.pdsc deleted file mode 100644 index 6d73078b44..0000000000 --- a/li-utils/src/main/pegasus/com/linkedin/avro2pegasus/events/search/PaginationContext.pdsc +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "PaginationContext", - "namespace": "com.linkedin.avro2pegasus.events.search", - "doc": "Data about pagination", - "type": "record", - "fields": [ - { - "name": "numToReturn", - "doc": "The number of entities to return after applying the filters described by entityFilter", - "optional": true, - "type": "int" - }, - { - "name": "entityFilter", - "doc": "List of URNs of specific entities to be filtered (eg. urn:li:member:1)", - "optional": true, - "type": { - "type": "array", - "items": "string" - } - }, - { - "name": "start", - "doc": "This field has been deprecated in favor of entityFilter and startFilter.", - "optional": true, - "type": "int" - }, - { - "name": "startFilter", - "doc": "The number of top scoring results to filter after applying entityFilter.", - "optional": true, - "type": "int" - } - ] -} \ No newline at end of file