yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-24 16:38:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(GQL Queries): Productionalizing GraphQL Part 1: Dataset Query support + adding shared GraphQL module (#2066)

Browse Source 
* Productionalizing GraphQL Part 1: Dataset Query support + introducing common datahub-graphql-core module.

Co-authored-by: John Joyce <jjoyce0510@gmail.com>
This commit is contained in:
John Joyce 2021-01-22 15:44:00 -08:00 committed by GitHub
parent f9d33f5519
commit 50cec65f57
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
39 changed files with 2040 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
build.gradle
View File

@ -30,6 +30,8 @@ project.ext.spec = [
]
project.ext.externalDependency = [
'antlr4Runtime': 'org.antlr:antlr4-runtime:4.7.2',
'antlr4': 'org.antlr:antlr4:4.7.2',
'assertJ': 'org.assertj:assertj-core:3.11.1',
'avro_1_7': 'org.apache.avro:avro:1.7.7',
'avroCompiler_1_7': 'org.apache.avro:avro-compiler:1.7.7',
@ -53,6 +55,7 @@ project.ext.externalDependency = [
'gmaTestModels': "com.linkedin.datahub-gma:test-models:$gmaVersion",
'gmaTestModelsDataTemplate': "com.linkedin.datahub-gma:test-models-data-template:$gmaVersion",
'gmaValidators': "com.linkedin.datahub-gma:validators:$gmaVersion",
'graphqlJava': 'com.graphql-java:graphql-java:16.1',
'gson': 'com.google.code.gson:gson:2.8.6',
'guice': 'com.google.inject:guice:4.2.2',
'guava': 'com.google.guava:guava:27.0.1-jre',

104
datahub-frontend/app/controllers/api/v2/GraphQLController.java Normal file
View File

@ -0,0 +1,104 @@
package controllers.api.v2;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.linkedin.datahub.graphql.GmsGraphQLEngine;
import com.linkedin.datahub.graphql.GraphQLEngine;
import com.typesafe.config.Config;
import graphql.ExecutionResult;
import graphql.PlayQueryContext;
import graphql.resolvers.mutation.LogInResolver;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import javax.annotation.Nonnull;
import javax.inject.Inject;
import org.apache.commons.io.IOUtils;
import play.api.Environment;
import play.mvc.Controller;
import play.mvc.Result;
public class GraphQLController extends Controller {
private static final String FRONTEND_SCHEMA_NAME = "datahub-frontend.graphql";
private static final String MUTATION_TYPE = "Mutation";
private static final String LOG_IN_MUTATION = "logIn";
private static final String QUERY = "query";
private static final String VARIABLES = "variables";
private final GraphQLEngine _engine;
private final Config _config;
@Inject
public GraphQLController(@Nonnull Environment environment, @Nonnull Config config) {
/*
* Fetch path to custom GraphQL Type System Definition
*/
String schemaString;
try {
final InputStream is = environment.resourceAsStream(FRONTEND_SCHEMA_NAME).get();
schemaString = IOUtils.toString(is, StandardCharsets.UTF_8);
is.close();
} catch (IOException e) {
throw new RuntimeException("Failed to find GraphQL Schema with name " + FRONTEND_SCHEMA_NAME, e);
}
/*
* Instantiate engine
*/
_engine = GmsGraphQLEngine.builder()
.addSchema(schemaString)
.configureRuntimeWiring(builder ->
builder.type(MUTATION_TYPE,
typeWiring -> typeWiring.dataFetcher(LOG_IN_MUTATION, new LogInResolver())))
.build();
_config = config;
}
@Nonnull
public Result execute() throws Exception {
JsonNode bodyJson = request().body().asJson();
if (bodyJson == null) {
return badRequest();
}
/*
* Extract "query" field
*/
JsonNode queryJson = bodyJson.get(QUERY);
if (queryJson == null) {
return badRequest();
}
/*
* Extract "variables" map
*/
JsonNode variablesJson = bodyJson.get(VARIABLES);
Map<String, Object> variables = null;
if (variablesJson != null) {
variables = new ObjectMapper().convertValue(variablesJson, new TypeReference<Map<String, Object>>(){ });
}
/*
* Init QueryContext
*/
PlayQueryContext context = new PlayQueryContext(session(), _config);
/*
* Execute GraphQL Query
*/
ExecutionResult executionResult = _engine.execute(queryJson.asText(), variables, context);
/*
* Format & Return Response
*/
return ok(new ObjectMapper().writeValueAsString(executionResult.toSpecification()));
}
}

72
datahub-frontend/app/graphql/PlayQueryContext.java Normal file
View File

@ -0,0 +1,72 @@
package graphql;
import com.typesafe.config.Config;
import play.mvc.Http;
import com.linkedin.datahub.graphql.QueryContext;
import static security.AuthConstants.AUTH_TOKEN;
import static security.AuthConstants.USER;
/**
* Provides session context to components of the GraphQL Engine at runtime.
*/
public class PlayQueryContext implements QueryContext {
private final Http.Session _session;
private final Config _appConfig;
public PlayQueryContext(Http.Session session) {
this(session, null);
}
public PlayQueryContext(Http.Session session, Config appConfig) {
_session = session;
_appConfig = appConfig;
}
/**
* Returns true if the current user is authenticated, false otherwise.
*/
@Override
public boolean isAuthenticated() {
return getSession().containsKey(AUTH_TOKEN); // TODO: Compute this once by validating the signed auth token.
}
/**
* Returns the currently logged in user string
*/
@Override
public String getActor() {
return _session.get(USER);
}
/**
* Retrieves the {@link Http.Session} object associated with the current user.
*/
public Http.Session getSession() {
return _session;
}
/**
* Retrieves the {@link Config} object associated with the play application.
*/
public Config getAppConfig() {
return _appConfig;
}
/**
* Retrieves the user name associated with the current user.
*/
public String getUserName() {
return _session.get(USER);
}
/**
* Retrieves the hashed auth token associated with the current user.
*/
public String getAuthToken() {
return _session.get(AUTH_TOKEN);
}
}

76
datahub-frontend/app/graphql/resolvers/mutation/LogInResolver.java Normal file
View File

@ -0,0 +1,76 @@
package graphql.resolvers.mutation;
import com.linkedin.common.urn.CorpuserUrn;
import com.linkedin.datahub.graphql.exception.AuthenticationException;
import com.linkedin.datahub.graphql.exception.ValidationException;
import com.linkedin.datahub.graphql.generated.CorpUser;
import com.linkedin.datahub.graphql.loaders.CorpUserLoader;
import com.linkedin.datahub.graphql.mappers.CorpUserMapper;
import graphql.PlayQueryContext;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import org.apache.commons.lang3.StringUtils;
import org.dataloader.DataLoader;
import play.Logger;
import security.AuthUtil;
import security.AuthenticationManager;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.concurrent.CompletableFuture;
import static security.AuthConstants.*;
/**
* Resolver responsible for authenticating a user
*/
public class LogInResolver implements DataFetcher<CompletableFuture<CorpUser>> {
@Override
public CompletableFuture<CorpUser> get(DataFetchingEnvironment environment) throws Exception {
/*
Extract arguments
*/
final String username = environment.getArgument(USER_NAME);
final String password = environment.getArgument(PASSWORD);
if (StringUtils.isBlank(username)) {
throw new ValidationException("username must not be empty");
}
PlayQueryContext context = environment.getContext();
context.getSession().clear();
// Create a uuid string for this session if one doesn't already exist
String uuid = context.getSession().get(UUID);
if (uuid == null) {
uuid = java.util.UUID.randomUUID().toString();
context.getSession().put(UUID, uuid);
}
try {
AuthenticationManager.authenticateUser(username, password);
} catch (javax.naming.AuthenticationException e) {
throw new AuthenticationException("Failed to authenticate user", e);
}
context.getSession().put(USER, username);
String secretKey = context.getAppConfig().getString(SECRET_KEY_PROPERTY);
try {
// store hashed username within PLAY_SESSION cookie
String hashedUserName = AuthUtil.generateHash(username, secretKey.getBytes());
context.getSession().put(AUTH_TOKEN, hashedUserName);
} catch (NoSuchAlgorithmException | InvalidKeyException e) {
throw new RuntimeException("Failed to hash username", e);
}
/*
Fetch the latest version of the logged in user. (via CorpUser entity)
*/
final DataLoader<String, com.linkedin.identity.CorpUser> userLoader = environment.getDataLoader(CorpUserLoader.NAME);
return userLoader.load(new CorpuserUrn(username).toString())
.thenApply(CorpUserMapper::map);
}
}

16
datahub-frontend/app/security/AuthConstants.java Normal file
View File

@ -0,0 +1,16 @@
package security;
public class AuthConstants {
public static final String SECRET_KEY_PROPERTY = "play.http.secret.key";
public static final String USER_NAME = "username";
public static final String PASSWORD = "password";
public static final String USER = "user";
public static final String UUID = "uuid";
public static final String AUTH_TOKEN = "auth_token";
private AuthConstants() { }
}

5
datahub-frontend/conf/datahub-frontend.graphql Normal file
View File

@ -0,0 +1,5 @@
# This will host GQL schema extensions specific to the frontend.
extend type Mutation {
logIn(username: String!, password: String!): CorpUser
}

2
datahub-frontend/conf/routes
View File

@ -37,6 +37,8 @@ GET /api/v2/datasets/:urn/snapshot co
GET /api/v2/datasets/:urn/upstreams controllers.api.v2.Dataset.getDatasetUpstreams(urn: String)
GET /api/v2/list/platforms controllers.api.v2.Dataset.getDataPlatforms
GET /api/v2/search controllers.api.v2.Search.search()
POST /api/v2/graphql controllers.api.v2.GraphQLController.execute()
GET /api/*path controllers.Application.apiNotFound(path)
POST /api/*path controllers.Application.apiNotFound(path)

5
datahub-frontend/play.gradle
View File

@ -16,6 +16,11 @@ dependencies {
assets project(path: ':datahub-web', configuration: 'assets')
play project(":datahub-dao")
play project(":datahub-graphql-core")
play externalDependency.graphqlJava
play externalDependency.antlr4Runtime
play externalDependency.antlr4
play externalDependency.jerseyCore
play externalDependency.jerseyGuava

61
datahub-graphql-core/README.md Normal file
View File

@ -0,0 +1,61 @@
Notice: `datahub-graphql-core` is currently in beta, and as such is currently subject to backwards incompatible changes.
# DataHub GraphQL Core
DataHub GraphQL API is a shared lib module containing a GraphQL API on top of the GMS service layer. It exposes a graph-based representation
permitting reads and writes against the entities and aspects on the Metadata Graph, including Datasets, CorpUsers, & more.
Contained within this module are
1. **GMS Schema**: A GQL schema that based on GMS models, located under `resources/gms.graphql`.
2. **GMS Data Fetchers**: Components used by the GraphQL engine to resolve individual fields in the GQL schema.
3. **GMS Data Loaders**: Components used by the GraphQL engine to fetch data from downstream sources efficiently (by batching).
4. **GraphQLEngine**: A wrapper on top of the default `GraphQL` object provided by `graphql-java`. Provides a way to configure all of the important stuff using a simple `Builder API`.
5. **GMSGraphQLEngine**: An engine capable of resolving the GMS schema using the data fetchers + loaders mentioned above (with no additional configuration required).
We've chosen to place these components in a library module so that GraphQL servers can be deployed in multiple "modes":
1. **Standalone**: GraphQL facade, mainly used for programmatic access to the GMS graph from a non-Java environment
2. **Embedded**: Leverageable within another Java server to surface an extended GraphQL schema. For example, we use this to extend the GMS GraphQL schema in `datahub-frontend`
## Extending the Graph
### Near Term
When extending the GMS graph, the following steps should be followed:
1. Extend `gms.graphql` schema with new `types` (Queries) or `inputs` (Mutations).
These should generally mirror the GMS models exactly, with notable exceptions:
- Maps: the GQL model must instead contain a list of { key, value } objects (e.g. Dataset.pdl 'properties' field)
- Foreign-Keys: Foreign-key references embedded in GMS models should be resolved if the referenced entity exists in the GQL schema,
replacing the key with the actual entity model. (Example: replacing the 'owner' urn field in 'Ownership' with an actual `CorpUser` type)
2. Implement `DataLoaders` for any `Query` data
- DataLoaders should simply wrap GMS-provided clients to fetch data from GMS API.
3. Implement `Mappers` to transform Pegasus model returned by GMS to an auto-generated GQL POJO. (under `/mainGeneratedGraphQL`, generated on `./gradlew datahub-graphql-core:build`)
- If you've followed the guidance above, these mappers should be simple, mainly
providing identity mappings for fields that exist in both the GQL + Pegasus POJOs.
- In some cases, you'll need to perform small lambdas (unions, maps) to materialize the GQL object.
4. Implement `DataFetchers` for any entity-type fields
- Each field which resolvers a full entity from a particular downstream GMS API should have it's owner resolver,
which leverages any DataLoaders implemented in step 2 in the case of `Queries`.
- Resolvers should always return an auto-generated GQL POJO (under `/mainGeneratedGraphQL`) to minimize the risk of runtime exceptions
5. Implement `DataFetcher` unit tests
### Long Term
Eventually, much of this is intended to be automatically generated from GMS models, including
- Generation of the primary entities on the GQL graph
- Generation of Pegasus to GQL mapper logic
- Generation of DataLoaders
- Generatation of DataFetchers (Resolvers)

37
datahub-graphql-core/build.gradle Normal file
View File

@ -0,0 +1,37 @@
plugins {
id "io.github.kobylynskyi.graphql.codegen" version "4.1.1"
}
apply plugin: 'java'
dependencies {
compile project(':gms:client')
compile project(':metadata-utils')
compile externalDependency.graphqlJava
compile externalDependency.antlr4Runtime
compile externalDependency.antlr4
compile externalDependency.guava
compileOnly externalDependency.lombok
annotationProcessor externalDependency.lombok
testCompile externalDependency.mockito
}
graphqlCodegen {
// For options: https://github.com/kobylynskyi/graphql-java-codegen/blob/master/docs/codegen-options.md
graphqlSchemaPaths = ["$projectDir/src/main/resources/gms.graphql".toString()]
outputDir = new File("$projectDir/src/mainGeneratedGraphQL/java")
packageName = "com.linkedin.datahub.graphql.generated"
generateApis = true
modelValidationAnnotation = "@javax.annotation.Nonnull"
customTypesMapping = [
Long: "Long",
]
}
tasks.withType(Checkstyle) {
exclude "**/generated/**"
}
compileJava.dependsOn 'graphqlCodegen'
sourceSets.main.java.srcDir "$projectDir/src/mainGeneratedGraphQL/java"

29
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java Normal file
View File

@ -0,0 +1,29 @@
package com.linkedin.datahub.graphql;
/**
* Constants relating to GraphQL type system & execution.
*/
public class Constants {
private Constants() { };
/**
* Type Names.
*/
public static final String QUERY_TYPE_NAME = "Query";
public static final String CORP_USER_TYPE_NAME = "CorpUser";
public static final String OWNER_TYPE_NAME = "Owner";
/**
* Field Names.
*/
public static final String URN_FIELD_NAME = "urn";
public static final String DATASETS_FIELD_NAME = "dataset";
public static final String MANAGER_FIELD_NAME = "manager";
public static final String OWNER_FIELD_NAME = "owner";
/**
* Misc.
*/
public static final String GMS_SCHEMA_FILE = "gms.graphql";
}

84
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java Normal file
View File

@ -0,0 +1,84 @@
package com.linkedin.datahub.graphql;
import com.google.common.collect.ImmutableMap;
import com.linkedin.datahub.graphql.loaders.CorpUserLoader;
import com.linkedin.datahub.graphql.loaders.DatasetLoader;
import com.linkedin.datahub.graphql.loaders.GmsClientFactory;
import com.linkedin.datahub.graphql.resolvers.AuthenticatedResolver;
import com.linkedin.datahub.graphql.resolvers.corpuser.ManagerResolver;
import com.linkedin.datahub.graphql.resolvers.ownership.OwnerResolver;
import com.linkedin.datahub.graphql.resolvers.query.DatasetResolver;
import graphql.schema.idl.RuntimeWiring;
import org.apache.commons.io.IOUtils;
import org.dataloader.DataLoader;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.function.Supplier;
import static com.linkedin.datahub.graphql.Constants.*;
import static graphql.Scalars.GraphQLLong;
/**
* A {@link GraphQLEngine} configured to provide access to the entities and aspects on the the GMS graph.
*/
public class GmsGraphQLEngine {
private static GraphQLEngine _engine;
public static String schema() {
String defaultSchemaString;
try {
InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(GMS_SCHEMA_FILE);
defaultSchemaString = IOUtils.toString(is, StandardCharsets.UTF_8);
is.close();
} catch (IOException e) {
throw new RuntimeException("Failed to find GraphQL Schema with name " + GMS_SCHEMA_FILE, e);
}
return defaultSchemaString;
}
public static Map<String, Supplier<DataLoader<?, ?>>> loaderSuppliers() {
return ImmutableMap.of(
DatasetLoader.NAME, () -> DataLoader.newDataLoader(new DatasetLoader(GmsClientFactory.getDatasetsClient())),
CorpUserLoader.NAME, () -> DataLoader.newDataLoader(new CorpUserLoader(GmsClientFactory.getCorpUsersClient()))
);
}
public static void configureRuntimeWiring(final RuntimeWiring.Builder builder) {
builder
.type(QUERY_TYPE_NAME, typeWiring -> typeWiring
.dataFetcher(DATASETS_FIELD_NAME, new AuthenticatedResolver<>(new DatasetResolver()))
)
.type(OWNER_TYPE_NAME, typeWiring -> typeWiring
.dataFetcher(OWNER_FIELD_NAME, new AuthenticatedResolver<>(new OwnerResolver()))
)
.type(CORP_USER_TYPE_NAME, typeWiring -> typeWiring
.dataFetcher(MANAGER_FIELD_NAME, new AuthenticatedResolver<>(new ManagerResolver()))
)
.scalar(GraphQLLong);
}
public static GraphQLEngine.Builder builder() {
return GraphQLEngine.builder()
.addSchema(schema())
.addDataLoaders(loaderSuppliers())
.configureRuntimeWiring(GmsGraphQLEngine::configureRuntimeWiring);
}
public static GraphQLEngine get() {
if (_engine == null) {
synchronized (GmsGraphQLEngine.class) {
if (_engine == null) {
_engine = builder().build();
}
}
}
return _engine;
}
private GmsGraphQLEngine() { }
}

163
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GraphQLEngine.java Normal file
View File

@ -0,0 +1,163 @@
package com.linkedin.datahub.graphql;
import graphql.ExecutionInput;
import graphql.ExecutionResult;
import graphql.GraphQL;
import graphql.schema.GraphQLSchema;
import graphql.schema.idl.RuntimeWiring;
import graphql.schema.idl.SchemaGenerator;
import graphql.schema.idl.SchemaParser;
import graphql.schema.idl.TypeDefinitionRegistry;
import org.dataloader.DataLoader;
import org.dataloader.DataLoaderRegistry;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Supplier;
import static graphql.schema.idl.RuntimeWiring.newRuntimeWiring;
/**
* Simple wrapper around a {@link GraphQL} instance providing APIs for building an engine and executing
* GQL queries.
* <p>This class provides a {@link Builder} builder for constructing {@link GraphQL} instances provided one or more
* schemas, {@link DataLoader}s, & a configured {@link RuntimeWiring}.
*
* <p>In addition, it provides a simplified 'execute' API that accepts a 1) query string and 2) set of variables.
*/
public class GraphQLEngine {
private final GraphQL _engine;
private final Map<String, Supplier<DataLoader<?, ?>>> _dataLoaderSuppliers;
private GraphQLEngine(@Nonnull final List<String> schemas,
@Nonnull final RuntimeWiring runtimeWiring,
@Nonnull final Map<String, Supplier<DataLoader<?, ?>>> dataLoaderSuppliers) {
_dataLoaderSuppliers = dataLoaderSuppliers;
/*
* Parse schema
*/
SchemaParser schemaParser = new SchemaParser();
TypeDefinitionRegistry typeDefinitionRegistry = new TypeDefinitionRegistry();
schemas.forEach(schema -> typeDefinitionRegistry.merge(schemaParser.parse(schema)));
/*
* Configure resolvers (data fetchers)
*/
SchemaGenerator schemaGenerator = new SchemaGenerator();
GraphQLSchema graphQLSchema = schemaGenerator.makeExecutableSchema(typeDefinitionRegistry, runtimeWiring);
/*
* Instantiate engine
*/
_engine = GraphQL.newGraphQL(graphQLSchema).build();
}
public ExecutionResult execute(@Nonnull final String query,
@Nullable final Map<String, Object> variables,
@Nonnull final QueryContext context) {
/*
* Init DataLoaderRegistry - should be created for each request.
*/
DataLoaderRegistry register = createDataLoaderRegistry(_dataLoaderSuppliers);
/*
* Construct execution input
*/
ExecutionInput executionInput = ExecutionInput.newExecutionInput()
.query(query)
.variables(variables)
.dataLoaderRegistry(register)
.context(context)
.build();
/*
* Execute GraphQL Query
*/
return _engine.execute(executionInput);
}
public static Builder builder() {
return new Builder();
}
/**
* Used to construct a {@link GraphQLEngine}.
*/
public static class Builder {
private final List<String> _schemas = new ArrayList<>();
private final Map<String, Supplier<DataLoader<?, ?>>> _loaderSuppliers = new HashMap<>();
private final RuntimeWiring.Builder _runtimeWiringBuilder = newRuntimeWiring();
/**
* Used to add a schema file containing the GQL types resolved by the engine.
*
* If multiple files are provided, their schemas will be merged together.
*/
public Builder addSchema(final String schema) {
_schemas.add(schema);
return this;
}
/**
* Used to register a {@link DataLoader} to be used within the configured resolvers.
*
* The {@link Supplier} provided is expected to return a new instance of {@link DataLoader} when invoked.
*
* If multiple loaders are registered with the name, the latter will override the former.
*/
public Builder addDataLoader(final String name, final Supplier<DataLoader<?, ?>> dataLoaderSupplier) {
_loaderSuppliers.put(name, dataLoaderSupplier);
return this;
}
/**
* Used to register multiple {@link DataLoader}s for use within the configured resolvers.
*
* The included {@link Supplier} provided is expected to return a new instance of {@link DataLoader} when invoked.
*
* If multiple loaders are registered with the name, the latter will override the former.
*/
public Builder addDataLoaders(Map<String, Supplier<DataLoader<?, ?>>> dataLoaderSuppliers) {
_loaderSuppliers.putAll(dataLoaderSuppliers);
return this;
}
/**
* Used to configure the runtime wiring (data fetchers & type resolvers)
* used in resolving the Graph QL schema.
*
* The {@link Consumer} provided accepts a {@link RuntimeWiring.Builder} and should register any required
* data + type resolvers.
*/
public Builder configureRuntimeWiring(final Consumer<RuntimeWiring.Builder> builderFunc) {
builderFunc.accept(_runtimeWiringBuilder);
return this;
}
/**
* Builds a {@link GraphQLEngine}.
*/
public GraphQLEngine build() {
return new GraphQLEngine(_schemas, _runtimeWiringBuilder.build(), _loaderSuppliers);
}
}
private DataLoaderRegistry createDataLoaderRegistry(final Map<String, Supplier<DataLoader<?, ?>>> dataLoaderSuppliers) {
final DataLoaderRegistry registry = new DataLoaderRegistry();
for (String key : dataLoaderSuppliers.keySet()) {
registry.register(key, dataLoaderSuppliers.get(key).get());
}
return registry;
}
}

17
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/QueryContext.java Normal file
View File

@ -0,0 +1,17 @@
package com.linkedin.datahub.graphql;
/**
* Provided as input to GraphQL resolvers; used to carry information about GQL request context.
*/
public interface QueryContext {
/**
* Returns true if the current actor is authenticated, false otherwise.
*/
boolean isAuthenticated();
/**
* Returns the current authenticated actor, null if there is none.
*/
String getActor();
}

17
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/exception/AuthenticationException.java Normal file
View File

@ -0,0 +1,17 @@
package com.linkedin.datahub.graphql.exception;
import graphql.GraphQLException;
/**
* Exception thrown when authentication fails.
*/
public class AuthenticationException extends GraphQLException {
public AuthenticationException(String message) {
super(message);
}
public AuthenticationException(String message, Throwable cause) {
super(message, cause);
}
}

17
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/exception/ValidationException.java Normal file
View File

@ -0,0 +1,17 @@
package com.linkedin.datahub.graphql.exception;
import graphql.GraphQLException;
/**
* Exception thrown when an unexpected value is provided by the client.
*/
public class ValidationException extends GraphQLException {
public ValidationException(String message) {
super(message);
}
public ValidationException(String message, Throwable cause) {
super(message, cause);
}
}

61
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/loaders/CorpUserLoader.java Normal file
View File

@ -0,0 +1,61 @@
package com.linkedin.datahub.graphql.loaders;
import com.linkedin.common.urn.CorpuserUrn;
import com.linkedin.identity.CorpUser;
import com.linkedin.identity.client.CorpUsers;
import org.dataloader.BatchLoader;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.stream.Collectors;
/**
* Responsible for fetching {@link CorpUser} objects from the downstream GMS, leveraging
* the public clients.
*/
public class CorpUserLoader implements BatchLoader<String, CorpUser> {
public static final String NAME = "corpUserLoader";
private final CorpUsers _corpUsersClient;
public CorpUserLoader(final CorpUsers corpUsersClient) {
_corpUsersClient = corpUsersClient;
}
@Override
public CompletionStage<List<CorpUser>> load(List<String> keys) {
return CompletableFuture.supplyAsync(() -> {
try {
final List<CorpuserUrn> corpUserUrns = keys
.stream()
.map(this::getCorpUserUrn)
.collect(Collectors.toList());
final Map<CorpuserUrn, CorpUser> corpUserMap = _corpUsersClient
.batchGet(new HashSet<>(corpUserUrns));
final List<CorpUser> results = new ArrayList<>();
for (CorpuserUrn urn : corpUserUrns) {
results.add(corpUserMap.getOrDefault(urn, null));
}
return results;
} catch (Exception e) {
throw new RuntimeException("Failed to batch load CorpUsers", e);
}
});
}
private CorpuserUrn getCorpUserUrn(String urnStr) {
try {
return CorpuserUrn.createFromString(urnStr);
} catch (URISyntaxException e) {
throw new RuntimeException(String.format("Failed to retrieve user with urn %s, invalid urn", urnStr));
}
}
}

62
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/loaders/DatasetLoader.java Normal file
View File

@ -0,0 +1,62 @@
package com.linkedin.datahub.graphql.loaders;
import com.linkedin.common.urn.DatasetUrn;
import com.linkedin.dataset.Dataset;
import com.linkedin.dataset.client.Datasets;
import org.dataloader.BatchLoader;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.stream.Collectors;
/**
* Responsible for fetching {@link Dataset} objects from the downstream GMS, leveraging
* the public clients.
*/
public class DatasetLoader implements BatchLoader<String, Dataset> {
public static final String NAME = "datasetLoader";
private final Datasets _datasetsClient;
public DatasetLoader(final Datasets datasetsClient) {
_datasetsClient = datasetsClient;
}
@Override
public CompletionStage<List<Dataset>> load(List<String> keys) {
return CompletableFuture.supplyAsync(() -> {
try {
List<DatasetUrn> datasetUrns = keys.stream()
.map(this::getDatasetUrn)
.collect(Collectors.toList());
Map<DatasetUrn, Dataset> datasetMap = _datasetsClient.batchGet(datasetUrns
.stream()
.filter(Objects::nonNull)
.collect(Collectors.toSet()));
List<Dataset> results = new ArrayList<>();
for (DatasetUrn urn : datasetUrns) {
results.add(datasetMap.getOrDefault(urn, null));
}
return results;
} catch (Exception e) {
throw new RuntimeException("Failed to batch load Datasets", e);
}
});
}
private DatasetUrn getDatasetUrn(String urnStr) {
try {
return DatasetUrn.createFromString(urnStr);
} catch (URISyntaxException e) {
throw new RuntimeException(String.format("Failed to retrieve dataset with urn %s, invalid urn", urnStr));
}
}
}

52
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/loaders/GmsClientFactory.java Normal file
View File

@ -0,0 +1,52 @@
package com.linkedin.datahub.graphql.loaders;
import com.linkedin.dataset.client.Datasets;
import com.linkedin.identity.client.CorpUsers;
import com.linkedin.metadata.restli.DefaultRestliClientFactory;
import com.linkedin.restli.client.Client;
import com.linkedin.util.Configuration;
/**
* Provides access to clients for use in fetching data from downstream GMS services.
*/
public class GmsClientFactory {
/**
* The following environment variables are expected to be provided.
* They are used in establishing the connection to the downstream GMS.
* Currently, only 1 downstream GMS is supported.
*/
private static final String GMS_HOST_ENV_VAR = "DATAHUB_GMS_HOST";
private static final String GMS_PORT_ENV_VAR = "DATAHUB_GMS_PORT";
private static final Client REST_CLIENT = DefaultRestliClientFactory.getRestLiClient(
Configuration.getEnvironmentVariable(GMS_HOST_ENV_VAR),
Integer.valueOf(Configuration.getEnvironmentVariable(GMS_PORT_ENV_VAR)));
private static CorpUsers _corpUsers;
private static Datasets _datasets;
private GmsClientFactory() { }
public static CorpUsers getCorpUsersClient() {
if (_corpUsers == null) {
synchronized (GmsClientFactory.class) {
if (_corpUsers == null) {
_corpUsers = new CorpUsers(REST_CLIENT);
}
}
}
return _corpUsers;
}
public static Datasets getDatasetsClient() {
if (_datasets == null) {
synchronized (GmsClientFactory.class) {
if (_datasets == null) {
_datasets = new Datasets(REST_CLIENT);
}
}
}
return _datasets;
}
}

28
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/mappers/AuditStampMapper.java Normal file
View File

@ -0,0 +1,28 @@
package com.linkedin.datahub.graphql.mappers;
import com.linkedin.datahub.graphql.generated.AuditStamp;
import javax.annotation.Nonnull;
/**
* Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema.
*
* To be replaced by auto-generated mappers implementations
*/
public class AuditStampMapper implements ModelMapper<com.linkedin.common.AuditStamp, AuditStamp> {
public static final AuditStampMapper INSTANCE = new AuditStampMapper();
public static AuditStamp map(@Nonnull final com.linkedin.common.AuditStamp auditStamp) {
return INSTANCE.apply(auditStamp);
}
@Override
public AuditStamp apply(@Nonnull final com.linkedin.common.AuditStamp auditStamp) {
final AuditStamp result = new AuditStamp();
result.setActor(auditStamp.getActor().toString());
result.setTime(auditStamp.getTime());
return result;
}
}

31
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/mappers/CorpUserEditableInfoMapper.java Normal file
View File

@ -0,0 +1,31 @@
package com.linkedin.datahub.graphql.mappers;
import com.linkedin.datahub.graphql.generated.CorpUserEditableInfo;
import javax.annotation.Nonnull;
/**
* Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema.
*
* To be replaced by auto-generated mappers implementations
*/
public class CorpUserEditableInfoMapper implements ModelMapper<com.linkedin.identity.CorpUserEditableInfo, CorpUserEditableInfo> {
public static final CorpUserEditableInfoMapper INSTANCE = new CorpUserEditableInfoMapper();
public static CorpUserEditableInfo map(@Nonnull final com.linkedin.identity.CorpUserEditableInfo info) {
return INSTANCE.apply(info);
}
@Override
public CorpUserEditableInfo apply(@Nonnull final com.linkedin.identity.CorpUserEditableInfo info) {
final CorpUserEditableInfo result = new CorpUserEditableInfo();
result.setAboutMe(info.getAboutMe());
result.setSkills(info.getSkills());
result.setTeams(info.getTeams());
if (info.hasPictureLink()) {
result.setPictureLink(info.getPictureLink().toString());
}
return result;
}
}

39
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/mappers/CorpUserInfoMapper.java Normal file
View File

@ -0,0 +1,39 @@
package com.linkedin.datahub.graphql.mappers;
import com.linkedin.datahub.graphql.generated.CorpUser;
import com.linkedin.datahub.graphql.generated.CorpUserInfo;
import javax.annotation.Nonnull;
/**
* Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema.
*
* To be replaced by auto-generated mappers implementations
*/
public class CorpUserInfoMapper implements ModelMapper<com.linkedin.identity.CorpUserInfo, CorpUserInfo> {
public static final CorpUserInfoMapper INSTANCE = new CorpUserInfoMapper();
public static CorpUserInfo map(@Nonnull final com.linkedin.identity.CorpUserInfo corpUserInfo) {
return INSTANCE.apply(corpUserInfo);
}
@Override
public CorpUserInfo apply(@Nonnull final com.linkedin.identity.CorpUserInfo info) {
final CorpUserInfo result = new CorpUserInfo();
result.setActive(info.isActive());
result.setCountryCode(info.getCountryCode());
result.setDepartmentId(info.getDepartmentId());
result.setDepartmentName(info.getDepartmentName());
result.setEmail(info.getEmail());
result.setDisplayName(info.getDisplayName());
result.setFirstName(info.getFirstName());
result.setLastName(info.getLastName());
result.setFullName(info.getFullName());
result.setTitle(info.getTitle());
if (info.hasManagerUrn()) {
result.setManager(new CorpUser.Builder().setUrn(info.getManagerUrn().toString()).build());
}
return result;
}
}

34
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/mappers/CorpUserMapper.java Normal file
View File

@ -0,0 +1,34 @@
package com.linkedin.datahub.graphql.mappers;
import com.linkedin.common.urn.CorpuserUrn;
import com.linkedin.datahub.graphql.generated.CorpUser;
import javax.annotation.Nonnull;
/**
* Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema.
*
* To be replaced by auto-generated mappers implementations
*/
public class CorpUserMapper implements ModelMapper<com.linkedin.identity.CorpUser, CorpUser> {
public static final CorpUserMapper INSTANCE = new CorpUserMapper();
public static CorpUser map(@Nonnull final com.linkedin.identity.CorpUser corpUser) {
return INSTANCE.apply(corpUser);
}
@Override
public CorpUser apply(@Nonnull final com.linkedin.identity.CorpUser corpUser) {
final CorpUser result = new CorpUser();
result.setUrn(new CorpuserUrn(corpUser.getUsername()).toString());
result.setUsername(corpUser.getUsername());
if (corpUser.hasInfo()) {
result.setInfo(CorpUserInfoMapper.map(corpUser.getInfo()));
}
if (corpUser.hasEditableInfo()) {
result.setEditableInfo(CorpUserEditableInfoMapper.map(corpUser.getEditableInfo()));
}
return result;
}
}

54
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/mappers/DatasetMapper.java Normal file
View File

@ -0,0 +1,54 @@
package com.linkedin.datahub.graphql.mappers;
import com.linkedin.datahub.graphql.generated.Dataset;
import com.linkedin.datahub.graphql.generated.FabricType;
import com.linkedin.datahub.graphql.generated.PlatformNativeType;
import javax.annotation.Nonnull;
/**
* Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema.
*
* To be replaced by auto-generated mappers implementations
*/
public class DatasetMapper implements ModelMapper<com.linkedin.dataset.Dataset, Dataset> {
public static final DatasetMapper INSTANCE = new DatasetMapper();
public static Dataset map(@Nonnull final com.linkedin.dataset.Dataset dataset) {
return INSTANCE.apply(dataset);
}
@Override
public Dataset apply(@Nonnull final com.linkedin.dataset.Dataset dataset) {
com.linkedin.datahub.graphql.generated.Dataset result = new com.linkedin.datahub.graphql.generated.Dataset();
result.setUrn(dataset.getUrn().toString());
result.setName(dataset.getName());
result.setDescription(dataset.getDescription());
result.setPlatform(dataset.getPlatform().toString());
result.setOrigin(Enum.valueOf(FabricType.class, dataset.getOrigin().name()));
result.setTags(dataset.getTags());
// TODO: Modify GMS to return created, lastModified at the top level as contract requires.
if (dataset.hasSchemaMetadata()) {
result.setCreated(AuditStampMapper.map(dataset.getSchemaMetadata().getCreated()));
result.setLastModified(AuditStampMapper.map(dataset.getSchemaMetadata().getLastModified()));
if (dataset.getSchemaMetadata().hasDeleted()) {
result.setDeleted(AuditStampMapper.map(dataset.getSchemaMetadata().getDeleted()));
}
}
if (dataset.hasPlatformNativeType()) {
result.setPlatformNativeType(Enum.valueOf(PlatformNativeType.class, dataset.getPlatformNativeType().name()));
}
if (dataset.hasUri()) {
result.setUri(dataset.getUri().toString());
}
if (dataset.hasProperties()) {
result.setProperties(StringMapMapper.map(dataset.getProperties()));
}
if (dataset.hasOwnership()) {
result.setOwnership(OwnershipMapper.map(dataset.getOwnership()));
}
return result;
}
}

9
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/mappers/ModelMapper.java Normal file
View File

@ -0,0 +1,9 @@
package com.linkedin.datahub.graphql.mappers;
/**
* Simple interface for classes capable of mapping an input of type I to
* an output of type O.
*/
public interface ModelMapper<I, O> {
O apply(final I input);
}

34
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/mappers/OwnerMapper.java Normal file
View File

@ -0,0 +1,34 @@
package com.linkedin.datahub.graphql.mappers;
import com.linkedin.datahub.graphql.generated.CorpUser;
import com.linkedin.datahub.graphql.generated.Owner;
import com.linkedin.datahub.graphql.generated.OwnershipType;
import javax.annotation.Nonnull;
/**
* Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema.
*
* To be replaced by auto-generated mappers implementations
*/
public class OwnerMapper implements ModelMapper<com.linkedin.common.Owner, Owner> {
public static final OwnerMapper INSTANCE = new OwnerMapper();
public static Owner map(@Nonnull final com.linkedin.common.Owner owner) {
return INSTANCE.apply(owner);
}
@Override
public Owner apply(@Nonnull final com.linkedin.common.Owner owner) {
final Owner result = new Owner();
result.setType(Enum.valueOf(OwnershipType.class, owner.getType().toString()));
CorpUser partialOwner = new CorpUser();
partialOwner.setUrn(owner.getOwner().toString());
result.setOwner(partialOwner);
if (owner.hasSource()) {
result.setSource(OwnershipSourceMapper.map(owner.getSource()));
}
return result;
}
}

31
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/mappers/OwnershipMapper.java Normal file
View File

@ -0,0 +1,31 @@
package com.linkedin.datahub.graphql.mappers;
import com.linkedin.datahub.graphql.generated.Ownership;
import javax.annotation.Nonnull;
import java.util.stream.Collectors;
/**
* Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema.
*
* To be replaced by auto-generated mappers implementations
*/
public class OwnershipMapper implements ModelMapper<com.linkedin.common.Ownership, Ownership> {
public static final OwnershipMapper INSTANCE = new OwnershipMapper();
public static Ownership map(@Nonnull final com.linkedin.common.Ownership ownership) {
return INSTANCE.apply(ownership);
}
@Override
public Ownership apply(@Nonnull final com.linkedin.common.Ownership ownership) {
final Ownership result = new Ownership();
result.setLastModified(AuditStampMapper.map(ownership.getLastModified()));
result.setOwners(ownership.getOwners()
.stream()
.map(OwnerMapper::map)
.collect(Collectors.toList()));
return result;
}
}

29
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/mappers/OwnershipSourceMapper.java Normal file
View File

@ -0,0 +1,29 @@
package com.linkedin.datahub.graphql.mappers;
import com.linkedin.datahub.graphql.generated.OwnershipSource;
import com.linkedin.datahub.graphql.generated.OwnershipSourceType;
import javax.annotation.Nonnull;
/**
* Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema.
*
* To be replaced by auto-generated mappers implementations
*/
public class OwnershipSourceMapper implements ModelMapper<com.linkedin.common.OwnershipSource, OwnershipSource> {
public static final OwnershipSourceMapper INSTANCE = new OwnershipSourceMapper();
public static OwnershipSource map(@Nonnull final com.linkedin.common.OwnershipSource ownershipSource) {
return INSTANCE.apply(ownershipSource);
}
@Override
public OwnershipSource apply(@Nonnull final com.linkedin.common.OwnershipSource ownershipSource) {
final OwnershipSource result = new OwnershipSource();
result.setUrl(ownershipSource.getUrl());
result.setType(Enum.valueOf(OwnershipSourceType.class, ownershipSource.getType().toString()));
return result;
}
}

35
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/mappers/StringMapMapper.java Normal file
View File

@ -0,0 +1,35 @@
package com.linkedin.datahub.graphql.mappers;
import com.linkedin.datahub.graphql.generated.StringMapEntry;
import javax.annotation.Nonnull;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema.
*
* To be replaced by auto-generated mappers implementations
*/
public class StringMapMapper implements ModelMapper<Map<String, String>, List<StringMapEntry>> {
public static final StringMapMapper INSTANCE = new StringMapMapper();
public static List<StringMapEntry> map(@Nonnull final Map<String, String> input) {
return INSTANCE.apply(input);
}
@Override
public List<StringMapEntry> apply(@Nonnull final Map<String, String> input) {
List<StringMapEntry> results = new ArrayList<>();
for (String key : input.keySet()) {
final StringMapEntry entry = new StringMapEntry();
entry.setKey(key);
entry.setValue(input.get(key));
results.add(entry);
}
return results;
}
}

29
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/AuthenticatedResolver.java Normal file
View File

@ -0,0 +1,29 @@
package com.linkedin.datahub.graphql.resolvers;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.exception.AuthenticationException;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
/**
* Checks whether the user is currently authenticated & if so delegates execution to a child resolver.
*/
public final class AuthenticatedResolver<T> implements DataFetcher<T> {
private final DataFetcher<T> _resolver;
public AuthenticatedResolver(final DataFetcher<T> resolver) {
_resolver = resolver;
}
@Override
public final T get(DataFetchingEnvironment environment) throws Exception {
final QueryContext context = environment.getContext();
if (context.isAuthenticated()) {
return _resolver.get(environment);
}
throw new AuthenticationException("Failed to authenticate the current user.");
}
}

24
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/corpuser/ManagerResolver.java Normal file
View File

@ -0,0 +1,24 @@
package com.linkedin.datahub.graphql.resolvers.corpuser;
import com.linkedin.datahub.graphql.generated.CorpUser;
import com.linkedin.datahub.graphql.generated.CorpUserInfo;
import com.linkedin.datahub.graphql.loaders.CorpUserLoader;
import com.linkedin.datahub.graphql.mappers.CorpUserMapper;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import org.dataloader.DataLoader;
import java.util.concurrent.CompletableFuture;
/**
* Resolver responsible for resolving the 'manager' field of the CorpUser type.
*/
public class ManagerResolver implements DataFetcher<CompletableFuture<CorpUser>> {
@Override
public CompletableFuture<CorpUser> get(DataFetchingEnvironment environment) throws Exception {
final CorpUserInfo parent = environment.getSource();
final DataLoader<String, com.linkedin.identity.CorpUser> dataLoader = environment.getDataLoader(CorpUserLoader.NAME);
return dataLoader.load(parent.getManager().getUrn())
.thenApply(corpUser -> corpUser != null ? CorpUserMapper.map(corpUser) : null);
}
}

25
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ownership/OwnerResolver.java Normal file
View File

@ -0,0 +1,25 @@
package com.linkedin.datahub.graphql.resolvers.ownership;
import com.linkedin.datahub.graphql.generated.CorpUser;
import com.linkedin.datahub.graphql.generated.Owner;
import com.linkedin.datahub.graphql.loaders.CorpUserLoader;
import com.linkedin.datahub.graphql.mappers.CorpUserMapper;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import org.dataloader.DataLoader;
import java.util.concurrent.CompletableFuture;
/**
* Resolver responsible for resolving the 'owner' field of the Owner type.
*/
public class OwnerResolver implements DataFetcher<CompletableFuture<CorpUser>> {
@Override
public CompletableFuture<CorpUser> get(DataFetchingEnvironment environment) {
final Owner parent = environment.getSource();
final DataLoader<String, com.linkedin.identity.CorpUser> dataLoader = environment.getDataLoader(CorpUserLoader.NAME);
return dataLoader.load(parent.getOwner().getUrn())
.thenApply(corpUser -> corpUser != null ? CorpUserMapper.map(corpUser) : null);
}
}

25
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/query/DatasetResolver.java Normal file
View File

@ -0,0 +1,25 @@
package com.linkedin.datahub.graphql.resolvers.query;
import com.linkedin.datahub.graphql.generated.Dataset;
import com.linkedin.datahub.graphql.loaders.DatasetLoader;
import com.linkedin.datahub.graphql.mappers.DatasetMapper;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import org.dataloader.DataLoader;
import java.util.concurrent.CompletableFuture;
import static com.linkedin.datahub.graphql.Constants.*;
/**
* Resolver responsible for resolving the 'dataset' field of the Query type
*/
public class DatasetResolver implements DataFetcher<CompletableFuture<Dataset>> {
@Override
public CompletableFuture<Dataset> get(DataFetchingEnvironment environment) {
final DataLoader<String, com.linkedin.dataset.Dataset> dataLoader = environment.getDataLoader(DatasetLoader.NAME);
return dataLoader.load(environment.getArgument(URN_FIELD_NAME))
.thenApply(dataset -> dataset != null ? DatasetMapper.map(dataset) : null);
}
}

4
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/scalar/LongScalarType.java Normal file
View File

@ -0,0 +1,4 @@
package com.linkedin.datahub.graphql.scalar;
public class LongScalarType {
}

368
datahub-graphql-core/src/main/resources/gms.graphql Normal file
View File

@ -0,0 +1,368 @@
# Extending the GQL type system to include Long type used for dates
scalar Long
schema {
query: Query
mutation: Mutation
}
enum EntityType {
"""
The Dataset Entity
"""
DATASET
"""
The CorpUser Entity
"""
USER
}
type Query {
"""
Fetch a Dataset by primary key
"""
dataset(urn: String!): Dataset
}
type Mutation { }
type AuditStamp {
"""
When the audited action took place
"""
time: Long!
"""
Who performed the audited action
"""
actor: String
}
type Dataset {
"""
The unique Dataset URN
"""
urn: String!
"""
Standardized platform urn where the dataset is defined
"""
platform: String!
"""
Dataset native name
"""
name: String!
"""
Fabric type where dataset belongs to or where it was generated
"""
origin: FabricType
"""
Human readable description for dataset
"""
description: String
"""
The native format for the data platform
"""
platformNativeType: PlatformNativeType
"""
Native Dataset Uri. Uri should not include any environment specific properties
"""
uri: String
"""
Tags used for searching dataset
"""
tags: [String!]!
"""
A list of platform-specific metadata tuples
"""
properties: [StringMapEntry!]
"""
Ownership metadata of the dataset
"""
ownership: Ownership
"""
An AuditStamp corresponding to the creation of this Dataset
"""
created: AuditStamp!
"""
An AuditStamp corresponding to the modification of this Dataset
"""
lastModified: AuditStamp!
"""
An optional AuditStamp corresponding to the deletion of this Dataset
"""
deleted: AuditStamp
}
enum FabricType {
"""
Designates development fabrics
"""
DEV
"""
Designates early-integration (staging) fabrics
"""
EI
"""
Designates production fabrics
"""
PROD
"""
Designates corporation fabrics
"""
CORP
}
enum PlatformNativeType {
"""
Table
"""
TABLE
"""
View
"""
VIEW
"""
Directory in file system
"""
DIRECTORY
"""
Stream
"""
STREAM
"""
Bucket in key value store
"""
BUCKET
}
type StringMapEntry {
key: String!
value: String
}
type Ownership {
"""
List of owners of the entity
"""
owners: [Owner!]
"""
Audit stamp containing who last modified the record and when
"""
lastModified: AuditStamp!
}
enum OwnershipSourceType {
"""
Auditing system or audit logs
"""
AUDIT
"""
Database, e.g. GRANTS table
"""
DATABASE
"""
File system, e.g. file/directory owner
"""
FILE_SYSTEM
"""
Issue tracking system, e.g. Jira
"""
ISSUE_TRACKING_SYSTEM
"""
Manually provided by a user
"""
MANUAL
"""
Other ownership-like service, e.g. Nuage, ACL service etc
"""
SERVICE
"""
SCM system, e.g. GIT, SVN
"""
SOURCE_CONTROL
"""
Other sources
"""
OTHER
}
type OwnershipSource {
"""
The type of the source
"""
type: OwnershipSourceType!
"""
A reference URL for the source
"""
url: String
}
enum OwnershipType {
"""
A person or group that is in charge of developing the code
"""
DEVELOPER
"""
A person or group that is owning the data
"""
DATAOWNER
"""
A person or a group that overseas the operation, e.g. a DBA or SRE.
"""
DELEGATE
"""
A person, group, or service that produces/generates the data
"""
PRODUCER
"""
A person, group, or service that consumes the data
"""
CONSUMER
"""
A person or a group that has direct business interest
"""
STAKEHOLDER
}
type Owner {
"""
Owner object - This should be extended to support CorpGroups as well
"""
owner: CorpUser!
"""
The type of the ownership
"""
type: OwnershipType
"""
Source information for the ownership
"""
source: OwnershipSource
}
type CorpUser {
"""
The unique user URN
"""
urn: String!
"""
Username of the corp user
"""
username: String!
"""
Readable info about the corp user
"""
info: CorpUserInfo
"""
Writable info about the corp user
"""
editableInfo: CorpUserEditableInfo
}
type CorpUserInfo {
"""
Whether the user is active
"""
active: Boolean!
"""
Display name of the user
"""
displayName: String
"""
Email address of the user
"""
email: String!
"""
Title of the user
"""
title: String
"""
Direct manager of the user
"""
manager: CorpUser
"""
department id the user belong to
"""
departmentId: Long
"""
department name this user belong to
"""
departmentName: String
"""
first name of the user
"""
firstName: String
"""
last name of the user
"""
lastName: String
"""
Common name of this user, format is firstName plus lastName
"""
fullName: String
"""
two uppercase letters country code
"""
countryCode: String
}
type CorpUserEditableInfo {
"""
About me section of the user
"""
aboutMe: String
"""
Teams that the user belongs to
"""
teams: [String!]
"""
Skills that the user possesses
"""
skills: [String!]
"""
A URL which points to a picture which user wants to set as a profile photo
"""
pictureLink: String
}

113
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/corpuser/TestManagerResolver.java Normal file
View File

@ -0,0 +1,113 @@
package com.linkedin.datahub.graphql.resolvers.corpuser;
import com.linkedin.common.url.Url;
import com.linkedin.common.urn.CorpuserUrn;
import com.linkedin.data.template.StringArray;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.loaders.CorpUserLoader;
import com.linkedin.identity.CorpUser;
import com.linkedin.identity.CorpUserEditableInfo;
import com.linkedin.identity.CorpUserInfo;
import graphql.schema.DataFetchingEnvironment;
import org.dataloader.DataLoader;
import org.testng.annotations.Test;
import java.net.URISyntaxException;
import java.util.concurrent.CompletableFuture;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.*;
public class TestManagerResolver {
private static final ManagerResolver RESOLVER = new ManagerResolver();
@Test
public void testResolverUrnNotFound() throws Exception {
DataFetchingEnvironment env = mock(DataFetchingEnvironment.class);
QueryContext context = mock(QueryContext.class);
when(context.isAuthenticated()).thenReturn(true);
DataLoader mockLoader = mock(DataLoader.class);
when(mockLoader.load("urn:li:corpuser:missingUser")).thenReturn(
CompletableFuture.completedFuture(null));
com.linkedin.datahub.graphql.generated.CorpUserInfo parentInfo = mock(com.linkedin.datahub.graphql.generated.CorpUserInfo.class);
com.linkedin.datahub.graphql.generated.CorpUser parentUser = mock(com.linkedin.datahub.graphql.generated.CorpUser.class);
when(parentUser.getUrn()).thenReturn("urn:li:corpuser:missingUser");
when(parentInfo.getManager()).thenReturn(parentUser);
when(env.getContext()).thenReturn(context);
when(env.getSource()).thenReturn(parentInfo);
when(env.getDataLoader(CorpUserLoader.NAME)).thenReturn(mockLoader);
assertEquals(RESOLVER.get(env).get(), null);
}
@Test
public void testResolverSuccess() throws Exception {
DataFetchingEnvironment env = mock(DataFetchingEnvironment.class);
QueryContext context = mock(QueryContext.class);
when(context.isAuthenticated()).thenReturn(true);
DataLoader mockLoader = mock(DataLoader.class);
when(mockLoader.load("urn:li:corpuser:testUser")).thenReturn(
CompletableFuture.completedFuture(testUser()));
com.linkedin.datahub.graphql.generated.CorpUserInfo parentInfo = mock(com.linkedin.datahub.graphql.generated.CorpUserInfo.class);
com.linkedin.datahub.graphql.generated.CorpUser parentUser = mock(com.linkedin.datahub.graphql.generated.CorpUser.class);
when(parentUser.getUrn()).thenReturn("urn:li:corpuser:testUser");
when(parentInfo.getManager()).thenReturn(parentUser);
when(env.getContext()).thenReturn(context);
when(env.getSource()).thenReturn(parentInfo);
when(env.getDataLoader(CorpUserLoader.NAME)).thenReturn(mockLoader);
CorpUser expectedUser = testUser();
com.linkedin.datahub.graphql.generated.CorpUser actualUser = RESOLVER.get(env).get();
assertEquals(actualUser.getUrn(), new CorpuserUrn(expectedUser.getUsername()).toString());
assertEquals(actualUser.getUsername(), expectedUser.getUsername());
assertEquals(actualUser.getEditableInfo().getAboutMe(), expectedUser.getEditableInfo().getAboutMe());
assertEquals(actualUser.getEditableInfo().getPictureLink(), expectedUser.getEditableInfo().getPictureLink().toString());
assertEquals(actualUser.getEditableInfo().getSkills(), expectedUser.getEditableInfo().getSkills());
assertEquals(actualUser.getEditableInfo().getTeams(), expectedUser.getEditableInfo().getTeams());
assertEquals(actualUser.getInfo().getActive(), expectedUser.getInfo().isActive().booleanValue());
assertEquals(actualUser.getInfo().getManager().getUrn(), expectedUser.getInfo().getManagerUrn().toString());
assertEquals(actualUser.getInfo().getCountryCode(), expectedUser.getInfo().getCountryCode());
assertEquals(actualUser.getInfo().getDepartmentId(), expectedUser.getInfo().getDepartmentId());
assertEquals(actualUser.getInfo().getEmail(), expectedUser.getInfo().getEmail());
assertEquals(actualUser.getInfo().getDisplayName(), expectedUser.getInfo().getDisplayName());
assertEquals(actualUser.getInfo().getFirstName(), expectedUser.getInfo().getFirstName());
assertEquals(actualUser.getInfo().getLastName(), expectedUser.getInfo().getLastName());
assertEquals(actualUser.getInfo().getFullName(), expectedUser.getInfo().getFullName());
assertEquals(actualUser.getInfo().getTitle(), expectedUser.getInfo().getTitle());
}
private CorpUser testUser() throws URISyntaxException {
CorpUser validCorpUser = new CorpUser();
validCorpUser.setUsername("testUser");
validCorpUser.setEditableInfo(new CorpUserEditableInfo()
.setAboutMe("Test About Me")
.setSkills(new StringArray("skill1", "skill2"))
.setTeams(new StringArray("engineering"))
.setPictureLink(new Url("www.test.com"))
);
validCorpUser.setInfo(new CorpUserInfo()
.setActive(true)
.setDisplayName("Test User")
.setEmail("test@datahub.com")
.setCountryCode("us")
.setDepartmentId(1L)
.setDepartmentName("Engineering")
.setFirstName("Test")
.setLastName("User")
.setManagerUrn(CorpuserUrn.createFromString("urn:li:corpuser:michaelmanager"))
.setTitle("Engineer")
);
return validCorpUser;
}
}

114
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ownership/TestOwnerResolver.java Normal file
View File

@ -0,0 +1,114 @@
package com.linkedin.datahub.graphql.resolvers.ownership;
import com.linkedin.common.url.Url;
import com.linkedin.common.urn.CorpuserUrn;
import com.linkedin.data.template.StringArray;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.Owner;
import com.linkedin.datahub.graphql.loaders.CorpUserLoader;
import com.linkedin.identity.CorpUser;
import com.linkedin.identity.CorpUserEditableInfo;
import com.linkedin.identity.CorpUserInfo;
import graphql.schema.DataFetchingEnvironment;
import org.dataloader.DataLoader;
import org.testng.annotations.Test;
import java.net.URISyntaxException;
import java.util.concurrent.CompletableFuture;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.*;
public class TestOwnerResolver {
private static final OwnerResolver RESOLVER = new OwnerResolver();
@Test
public void testResolverUrnNotFound() throws Exception {
DataFetchingEnvironment env = mock(DataFetchingEnvironment.class);
QueryContext context = mock(QueryContext.class);
when(context.isAuthenticated()).thenReturn(true);
DataLoader mockLoader = mock(DataLoader.class);
when(mockLoader.load("urn:li:corpuser:missingUser")).thenReturn(
CompletableFuture.completedFuture(null));
Owner parentOwner = mock(Owner.class);
com.linkedin.datahub.graphql.generated.CorpUser parentUser = mock(com.linkedin.datahub.graphql.generated.CorpUser.class);
when(parentUser.getUrn()).thenReturn("urn:li:corpuser:missingUser");
when(parentOwner.getOwner()).thenReturn(parentUser);
when(env.getContext()).thenReturn(context);
when(env.getSource()).thenReturn(parentOwner);
when(env.getDataLoader(CorpUserLoader.NAME)).thenReturn(mockLoader);
assertEquals(RESOLVER.get(env).get(), null);
}
@Test
public void testResolverSuccess() throws Exception {
DataFetchingEnvironment env = mock(DataFetchingEnvironment.class);
QueryContext context = mock(QueryContext.class);
when(context.isAuthenticated()).thenReturn(true);
DataLoader mockLoader = mock(DataLoader.class);
when(mockLoader.load("urn:li:corpuser:testUser")).thenReturn(
CompletableFuture.completedFuture(testUser()));
Owner parentOwner = mock(Owner.class);
com.linkedin.datahub.graphql.generated.CorpUser parentUser = mock(com.linkedin.datahub.graphql.generated.CorpUser.class);
when(parentUser.getUrn()).thenReturn("urn:li:corpuser:testUser");
when(parentOwner.getOwner()).thenReturn(parentUser);
when(env.getContext()).thenReturn(context);
when(env.getSource()).thenReturn(parentOwner);
when(env.getDataLoader(CorpUserLoader.NAME)).thenReturn(mockLoader);
CorpUser expectedUser = testUser();
com.linkedin.datahub.graphql.generated.CorpUser actualUser = RESOLVER.get(env).get();
assertEquals(actualUser.getUrn(), new CorpuserUrn(expectedUser.getUsername()).toString());
assertEquals(actualUser.getUsername(), expectedUser.getUsername());
assertEquals(actualUser.getEditableInfo().getAboutMe(), expectedUser.getEditableInfo().getAboutMe());
assertEquals(actualUser.getEditableInfo().getPictureLink(), expectedUser.getEditableInfo().getPictureLink().toString());
assertEquals(actualUser.getEditableInfo().getSkills(), expectedUser.getEditableInfo().getSkills());
assertEquals(actualUser.getEditableInfo().getTeams(), expectedUser.getEditableInfo().getTeams());
assertEquals(actualUser.getInfo().getActive(), expectedUser.getInfo().isActive().booleanValue());
assertEquals(actualUser.getInfo().getManager().getUrn(), expectedUser.getInfo().getManagerUrn().toString());
assertEquals(actualUser.getInfo().getCountryCode(), expectedUser.getInfo().getCountryCode());
assertEquals(actualUser.getInfo().getDepartmentId(), expectedUser.getInfo().getDepartmentId());
assertEquals(actualUser.getInfo().getEmail(), expectedUser.getInfo().getEmail());
assertEquals(actualUser.getInfo().getDisplayName(), expectedUser.getInfo().getDisplayName());
assertEquals(actualUser.getInfo().getFirstName(), expectedUser.getInfo().getFirstName());
assertEquals(actualUser.getInfo().getLastName(), expectedUser.getInfo().getLastName());
assertEquals(actualUser.getInfo().getFullName(), expectedUser.getInfo().getFullName());
assertEquals(actualUser.getInfo().getTitle(), expectedUser.getInfo().getTitle());
}
private CorpUser testUser() throws URISyntaxException {
CorpUser validCorpUser = new CorpUser();
validCorpUser.setUsername("testUser");
validCorpUser.setEditableInfo(new CorpUserEditableInfo()
.setAboutMe("Test About Me")
.setSkills(new StringArray("skill1", "skill2"))
.setTeams(new StringArray("engineering"))
.setPictureLink(new Url("www.test.com"))
);
validCorpUser.setInfo(new CorpUserInfo()
.setActive(true)
.setDisplayName("Test User")
.setEmail("test@datahub.com")
.setCountryCode("us")
.setDepartmentId(1L)
.setDepartmentName("Engineering")
.setFirstName("Test")
.setLastName("User")
.setManagerUrn(CorpuserUrn.createFromString("urn:li:corpuser:michaelmanager"))
.setTitle("Engineer")
);
return validCorpUser;
}
}

130
datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/TestDatasetResolver.java Normal file
View File

@ -0,0 +1,130 @@
package com.linkedin.datahub.graphql.resolvers.query;
import com.google.common.collect.ImmutableList;
import com.linkedin.common.AuditStamp;
import com.linkedin.common.FabricType;
import com.linkedin.common.Owner;
import com.linkedin.common.OwnerArray;
import com.linkedin.common.Ownership;
import com.linkedin.common.OwnershipSource;
import com.linkedin.common.OwnershipSourceType;
import com.linkedin.common.OwnershipType;
import com.linkedin.common.urn.DataPlatformUrn;
import com.linkedin.common.urn.DatasetUrn;
import com.linkedin.common.urn.Urn;
import com.linkedin.data.template.StringArray;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.loaders.DatasetLoader;
import com.linkedin.dataset.Dataset;
import com.linkedin.dataset.PlatformNativeType;
import com.linkedin.schema.SchemaMetadata;
import graphql.schema.DataFetchingEnvironment;
import org.dataloader.DataLoader;
import org.testng.annotations.Test;
import java.net.URISyntaxException;
import java.util.concurrent.CompletableFuture;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.*;
public class TestDatasetResolver {
private static final DatasetResolver RESOLVER = new DatasetResolver();
@Test
public void testResolverUrnNotFound() throws Exception {
DataFetchingEnvironment env = mock(DataFetchingEnvironment.class);
QueryContext context = mock(QueryContext.class);
when(context.isAuthenticated()).thenReturn(true);
DataLoader mockLoader = mock(DataLoader.class);
when(mockLoader.load("urn:li:dataset:missingDataset")).thenReturn(
CompletableFuture.completedFuture(null));
when(env.getContext()).thenReturn(context);
when(env.getArgument("urn")).thenReturn("urn:li:dataset:missingDataset");
when(env.getDataLoader(DatasetLoader.NAME)).thenReturn(mockLoader);
assertEquals(RESOLVER.get(env).get(), null);
}
@Test
public void testResolverSuccess() throws Exception {
DataFetchingEnvironment env = mock(DataFetchingEnvironment.class);
QueryContext context = mock(QueryContext.class);
when(context.isAuthenticated()).thenReturn(true);
DataLoader mockLoader = mock(DataLoader.class);
when(mockLoader.load("urn:li:dataset:testDataset")).thenReturn(
CompletableFuture.completedFuture(testDataset()));
when(env.getContext()).thenReturn(context);
when(env.getArgument("urn")).thenReturn("urn:li:dataset:testDataset");
when(env.getDataLoader(DatasetLoader.NAME)).thenReturn(mockLoader);
Dataset expectedDataset = testDataset();
com.linkedin.datahub.graphql.generated.Dataset actualDataset = RESOLVER.get(env).get();
assertEquals(actualDataset.getUrn(), expectedDataset.getUrn().toString());
assertEquals(actualDataset.getDescription(), expectedDataset.getDescription());
assertEquals(actualDataset.getName(), expectedDataset.getName());
assertEquals(actualDataset.getTags(), expectedDataset.getTags());
assertEquals(actualDataset.getPlatform(), expectedDataset.getPlatform().toString());
assertEquals(actualDataset.getUri(), null);
assertEquals(actualDataset.getOrigin(), com.linkedin.datahub.graphql.generated.FabricType.valueOf(expectedDataset.getOrigin().toString()));
assertEquals(actualDataset.getPlatformNativeType(),
com.linkedin.datahub.graphql.generated.PlatformNativeType.valueOf(expectedDataset.getPlatformNativeType().toString()));
assertEquals(actualDataset.getCreated().getActor(), expectedDataset.getSchemaMetadata().getCreated().getActor().toString());
assertEquals(actualDataset.getCreated().getTime(), expectedDataset.getSchemaMetadata().getCreated().getTime());
assertEquals(actualDataset.getLastModified().getActor(), expectedDataset.getSchemaMetadata().getLastModified().getActor().toString());
assertEquals(actualDataset.getLastModified().getTime(), expectedDataset.getSchemaMetadata().getLastModified().getTime());
assertEquals(actualDataset.getOwnership().getLastModified().getActor(), expectedDataset.getOwnership().getLastModified().getActor().toString());
assertEquals(actualDataset.getOwnership().getLastModified().getTime(), expectedDataset.getOwnership().getLastModified().getTime());
Owner expectedOwner = expectedDataset.getOwnership().getOwners().get(0);
com.linkedin.datahub.graphql.generated.Owner actualOwner = actualDataset.getOwnership().getOwners().get(0);
assertEquals(actualOwner.getType(),
com.linkedin.datahub.graphql.generated.OwnershipType.valueOf(expectedOwner.getType().toString()));
assertEquals(actualOwner.getSource().getType(),
com.linkedin.datahub.graphql.generated.OwnershipSourceType.valueOf(expectedOwner.getSource().getType().toString()));
assertEquals(actualOwner.getSource().getUrl(), expectedOwner.getSource().getUrl());
assertEquals(actualOwner.getOwner().getUrn(), expectedOwner.getOwner().toString());
}
private Dataset testDataset() throws URISyntaxException {
Dataset validDataset = new com.linkedin.dataset.Dataset()
.setUrn(DatasetUrn.createFromString("urn:li:dataset:(urn:li:dataPlatform:hive,TestDataset,PROD)"))
.setDescription("Test Dataset Description")
.setName("TestDataset")
.setOrigin(FabricType.PROD)
.setPlatform(DataPlatformUrn.createFromString("urn:li:dataPlatform:hive"))
.setPlatformNativeType(PlatformNativeType.TABLE)
.setTags(new StringArray("tag1", "tag2"))
.setSchemaMetadata(new SchemaMetadata()
.setCreated(new AuditStamp().setActor(Urn.createFromString("urn:li:corpuser:1")).setTime(0L))
.setLastModified(new AuditStamp().setActor(Urn.createFromString("urn:li:corpuser:2")).setTime(1L))
)
.setOwnership(new Ownership()
.setLastModified(new AuditStamp().setActor(Urn.createFromString("urn:li:corpuser:3")).setTime(2L))
.setOwners(new OwnerArray(ImmutableList.of(
new Owner()
.setOwner(Urn.createFromString("urn:li:corpuser:test"))
.setType(OwnershipType.DATAOWNER)
.setSource(new OwnershipSource()
.setType(OwnershipSourceType.FILE_SYSTEM)
.setUrl("www.datahub.test")
)
)))
);
return validDataset;
}
}

1
settings.gradle
View File

@ -1,5 +1,6 @@
include 'datahub-dao'
include 'datahub-frontend'
include 'datahub-graphql-core'
include 'datahub-web'
include 'gms:api'
include 'gms:client'