diff --git a/.gitignore b/.gitignore index 4272dc7c63..0ce69dc047 100644 --- a/.gitignore +++ b/.gitignore @@ -72,6 +72,7 @@ temp/** # frontend assets datahub-frontend/public/** +datahub-frontend/test/resources/public/** .remote* # Ignore runtime generated authenticatior/authorizer jar files diff --git a/build.gradle b/build.gradle index a75fbe6b57..22217f2149 100644 --- a/build.gradle +++ b/build.gradle @@ -10,6 +10,7 @@ buildscript { ext.testContainersVersion = '1.17.4' ext.jacksonVersion = '2.13.4' ext.jettyVersion = '9.4.46.v20220331' + ext.playVersion = '2.8.18' ext.log4jVersion = '2.19.0' ext.slf4jVersion = '1.7.32' ext.logbackClassic = '1.2.11' @@ -50,7 +51,7 @@ project.ext.spec = [ ] project.ext.externalDependency = [ - 'akkaHttp': 'com.typesafe.akka:akka-http-core_2.12:10.1.15', + 'akkaHttp': 'com.typesafe.akka:akka-http-core_2.12:10.2.10', 'antlr4Runtime': 'org.antlr:antlr4-runtime:4.7.2', 'antlr4': 'org.antlr:antlr4:4.7.2', 'assertJ': 'org.assertj:assertj-core:3.11.1', @@ -82,7 +83,7 @@ project.ext.externalDependency = [ 'graphqlJava': 'com.graphql-java:graphql-java:' + graphQLJavaVersion, 'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:' + graphQLJavaVersion, 'gson': 'com.google.code.gson:gson:2.8.9', - 'guice': 'com.google.inject:guice:4.2.2', + 'guice': 'com.google.inject:guice:4.2.3', 'guava': 'com.google.guava:guava:27.0.1-jre', 'h2': 'com.h2database:h2:2.1.214', 'hadoopClient': 'org.apache.hadoop:hadoop-client:3.2.1', @@ -125,6 +126,7 @@ project.ext.externalDependency = [ 'log4jCore': "org.apache.logging.log4j:log4j-core:$log4jVersion", 'log4jApi': "org.apache.logging.log4j:log4j-api:$log4jVersion", 'log4j12Api': "org.slf4j:log4j-over-slf4j:$slf4jVersion", + 'log4j2Api': "org.apache.logging.log4j:log4j-to-slf4j:$log4jVersion", 'lombok': 'org.projectlombok:lombok:1.18.12', 'mariadbConnector': 'org.mariadb.jdbc:mariadb-java-client:2.6.0', 'mavenArtifact': "org.apache.maven:maven-artifact:$mavenVersion", @@ -141,17 +143,18 @@ project.ext.externalDependency = [ 'opentracingJdbc':'io.opentracing.contrib:opentracing-jdbc:0.2.15', 'parquet': 'org.apache.parquet:parquet-avro:1.12.3', 'picocli': 'info.picocli:picocli:4.5.0', - 'playCache': 'com.typesafe.play:play-cache_2.12:2.7.6', - 'playEhcache': 'com.typesafe.play:play-ehcache_2.12:2.7.6', - 'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.0.8', - 'playDocs': 'com.typesafe.play:play-docs_2.12:2.7.6', - 'playGuice': 'com.typesafe.play:play-guice_2.12:2.7.6', - 'playJavaJdbc': 'com.typesafe.play:play-java-jdbc_2.12:2.7.6', - 'playAkkaHttpServer': 'com.typesafe.play:play-akka-http-server_2.12:2.7.6', - 'playServer': 'com.typesafe.play:play-server_2.12:2.7.6', - 'playTest': 'com.typesafe.play:play-test_2.12:2.7.6', - 'pac4j': 'org.pac4j:pac4j-oidc:3.6.0', - 'playPac4j': 'org.pac4j:play-pac4j_2.12:8.0.2', + 'playCache': "com.typesafe.play:play-cache_2.12:$playVersion", + 'playEhcache': "com.typesafe.play:play-ehcache_2.12:$playVersion", + 'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.1.10', + 'playDocs': "com.typesafe.play:play-docs_2.12:$playVersion", + 'playGuice': "com.typesafe.play:play-guice_2.12:$playVersion", + 'playJavaJdbc': "com.typesafe.play:play-java-jdbc_2.12:$playVersion", + 'playAkkaHttpServer': "com.typesafe.play:play-akka-http-server_2.12:$playVersion", + 'playServer': "com.typesafe.play:play-server_2.12:$playVersion", + 'playTest': "com.typesafe.play:play-test_2.12:$playVersion", + 'playFilters': "com.typesafe.play:filters-helpers_2.12:$playVersion", + 'pac4j': 'org.pac4j:pac4j-oidc:4.5.7', + 'playPac4j': 'org.pac4j:play-pac4j_2.12:9.0.2', 'postgresql': 'org.postgresql:postgresql:42.3.8', 'protobuf': 'com.google.protobuf:protobuf-java:3.19.6', 'rangerCommons': 'org.apache.ranger:ranger-plugins-common:2.3.0', @@ -203,7 +206,6 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) { exclude group: "io.netty", module: "netty" exclude group: "log4j", module: "log4j" exclude group: "org.springframework.boot", module: "spring-boot-starter-logging" - exclude group: "org.apache.logging.log4j", module: "log4j-to-slf4j" exclude group: "com.vaadin.external.google", module: "android-json" exclude group: "org.slf4j", module: "slf4j-reload4j" exclude group: "org.slf4j", module: "slf4j-log4j12" @@ -230,7 +232,7 @@ subprojects { dependencies { testCompile externalDependency.testng constraints { - implementation('io.netty:netty-all:4.1.68.Final') + implementation('io.netty:netty-all:4.1.85.Final') implementation('org.apache.commons:commons-compress:1.21') implementation('org.apache.velocity:velocity-engine-core:2.3') implementation('org.hibernate:hibernate-validator:6.0.20.Final') diff --git a/datahub-frontend/README.md b/datahub-frontend/README.md index 0bfb796562..b36e0b62f1 100644 --- a/datahub-frontend/README.md +++ b/datahub-frontend/README.md @@ -7,7 +7,7 @@ DataHub frontend is a [Play](https://www.playframework.com/) service written in between [DataHub GMS](../metadata-service) which is the backend service and [DataHub Web](../datahub-web-react/README.md). ## Pre-requisites -* You need to have [JDK8](https://www.oracle.com/java/technologies/jdk8-downloads.html) +* You need to have [JDK11](https://openjdk.org/projects/jdk/11/) installed on your machine to be able to build `DataHub Frontend`. * You need to have [Chrome](https://www.google.com/chrome/) web browser installed to be able to build because UI tests have a dependency on `Google Chrome`. diff --git a/datahub-frontend/app/auth/AuthModule.java b/datahub-frontend/app/auth/AuthModule.java index 92d495c3ed..9b1f89e20f 100644 --- a/datahub-frontend/app/auth/AuthModule.java +++ b/datahub-frontend/app/auth/AuthModule.java @@ -89,7 +89,7 @@ public class AuthModule extends AbstractModule { final String aesKeyHash = DigestUtils.sha1Hex(aesKeyBase.getBytes(StandardCharsets.UTF_8)); final String aesEncryptionKey = aesKeyHash.substring(0, 16); playCacheCookieStore = new PlayCookieSessionStore( - new ShiroAesDataEncrypter(aesEncryptionKey)); + new ShiroAesDataEncrypter(aesEncryptionKey.getBytes())); } catch (Exception e) { throw new RuntimeException("Failed to instantiate Pac4j cookie session store!", e); } diff --git a/datahub-frontend/app/auth/AuthUtils.java b/datahub-frontend/app/auth/AuthUtils.java index a32419ace1..9084311adb 100644 --- a/datahub-frontend/app/auth/AuthUtils.java +++ b/datahub-frontend/app/auth/AuthUtils.java @@ -63,8 +63,8 @@ public class AuthUtils { * * Returns true if the request is eligible to be forwarded to GMS, false otherwise. */ - public static boolean isEligibleForForwarding(Http.Context ctx) { - return hasValidSessionCookie(ctx) || hasAuthHeader(ctx); + public static boolean isEligibleForForwarding(Http.Request req) { + return hasValidSessionCookie(req) || hasAuthHeader(req); } /** @@ -75,17 +75,17 @@ public class AuthUtils { * Note that we depend on the presence of 2 cookies, one accessible to the browser and one not, * as well as their agreement to determine authentication status. */ - public static boolean hasValidSessionCookie(final Http.Context ctx) { - return ctx.session().containsKey(ACTOR) - && ctx.request().cookie(ACTOR) != null - && ctx.session().get(ACTOR).equals(ctx.request().cookie(ACTOR).value()); + public static boolean hasValidSessionCookie(final Http.Request req) { + return req.session().data().containsKey(ACTOR) + && req.cookie(ACTOR) != null + && req.session().data().get(ACTOR).equals(req.cookie(ACTOR).value()); } /** * Returns true if a request includes the Authorization header, false otherwise */ - public static boolean hasAuthHeader(final Http.Context ctx) { - return ctx.request().getHeaders().contains(Http.HeaderNames.AUTHORIZATION); + public static boolean hasAuthHeader(final Http.Request req) { + return req.getHeaders().contains(Http.HeaderNames.AUTHORIZATION); } /** diff --git a/datahub-frontend/app/auth/Authenticator.java b/datahub-frontend/app/auth/Authenticator.java index 0add06c297..ae847b318d 100644 --- a/datahub-frontend/app/auth/Authenticator.java +++ b/datahub-frontend/app/auth/Authenticator.java @@ -28,40 +28,21 @@ public class Authenticator extends Security.Authenticator { } @Override - public String getUsername(@Nonnull Http.Context ctx) { + public Optional getUsername(@Nonnull Http.Request req) { if (this.metadataServiceAuthEnabled) { // If Metadata Service auth is enabled, we only want to verify presence of the // "Authorization" header OR the presence of a frontend generated session cookie. // At this time, the actor is still considered to be unauthenicated. - return AuthUtils.isEligibleForForwarding(ctx) ? "urn:li:corpuser:UNKNOWN" : null; + return Optional.ofNullable(AuthUtils.isEligibleForForwarding(req) ? "urn:li:corpuser:UNKNOWN" : null); } else { // If Metadata Service auth is not enabled, verify the presence of a valid session cookie. - return AuthUtils.hasValidSessionCookie(ctx) ? ctx.session().get(ACTOR) : null; - } - } - - @Override - public Optional getUsername(@Nonnull Http.Request request) { - Http.Context ctx = Http.Context.current(); - if (this.metadataServiceAuthEnabled) { - // If Metadata Service auth is enabled, we only want to verify presence of the - // "Authorization" header OR the presence of a frontend generated session cookie. - // At this time, the actor is still considered to be unauthenicated. - return Optional.ofNullable(AuthUtils.isEligibleForForwarding(ctx) ? "urn:li:corpuser:UNKNOWN" : null); - } else { - // If Metadata Service auth is not enabled, verify the presence of a valid session cookie. - return Optional.ofNullable(AuthUtils.hasValidSessionCookie(ctx) ? ctx.session().get(ACTOR) : null); + return Optional.ofNullable(AuthUtils.hasValidSessionCookie(req) ? req.session().data().get(ACTOR) : null); } } @Override @Nonnull - public Result onUnauthorized(@Nullable Http.Context ctx) { - return unauthorized(); - } - - @Override - public Result onUnauthorized(Http.Request req) { + public Result onUnauthorized(@Nullable Http.Request req) { return unauthorized(); } } diff --git a/datahub-frontend/app/auth/sso/SsoProvider.java b/datahub-frontend/app/auth/sso/SsoProvider.java index 656d0ca2d7..f7454d599b 100644 --- a/datahub-frontend/app/auth/sso/SsoProvider.java +++ b/datahub-frontend/app/auth/sso/SsoProvider.java @@ -1,6 +1,7 @@ package auth.sso; import org.pac4j.core.client.Client; +import org.pac4j.core.credentials.Credentials; /** * A thin interface over a Pac4j {@link Client} object and its @@ -40,6 +41,6 @@ public interface SsoProvider { /** * Retrieves an initialized Pac4j {@link Client}. */ - Client client(); + Client client(); } diff --git a/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java b/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java index 723601d7f3..18b482de91 100644 --- a/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java +++ b/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java @@ -1,11 +1,13 @@ package auth.sso.oidc; import java.util.Map.Entry; +import java.util.Optional; import org.pac4j.core.authorization.generator.AuthorizationGenerator; import org.pac4j.core.context.WebContext; import org.pac4j.core.profile.AttributeLocation; import org.pac4j.core.profile.CommonProfile; +import org.pac4j.core.profile.UserProfile; import org.pac4j.core.profile.definition.ProfileDefinition; import org.pac4j.oidc.profile.OidcProfile; import org.slf4j.Logger; @@ -18,33 +20,38 @@ public class OidcAuthorizationGenerator implements AuthorizationGenerator { private static final Logger logger = LoggerFactory.getLogger(OidcAuthorizationGenerator.class); - private final ProfileDefinition profileDef; + private final ProfileDefinition profileDef; private final OidcConfigs oidcConfigs; - public OidcAuthorizationGenerator(final ProfileDefinition profileDef, final OidcConfigs oidcConfigs) { + public OidcAuthorizationGenerator(final ProfileDefinition profileDef, final OidcConfigs oidcConfigs) { this.profileDef = profileDef; this.oidcConfigs = oidcConfigs; } @Override - public CommonProfile generate(WebContext context, CommonProfile profile) { + public Optional generate(WebContext context, UserProfile profile) { if (oidcConfigs.getExtractJwtAccessTokenClaims().orElse(false)) { try { final JWT jwt = JWTParser.parse(((OidcProfile) profile).getAccessToken().getValue()); + + CommonProfile commonProfile = new CommonProfile(); for (final Entry entry : jwt.getJWTClaimsSet().getClaims().entrySet()) { final String claimName = entry.getKey(); + if (profile.getAttribute(claimName) == null) { - profileDef.convertAndAdd(profile, AttributeLocation.PROFILE_ATTRIBUTE, claimName, entry.getValue()); + profileDef.convertAndAdd(commonProfile, AttributeLocation.PROFILE_ATTRIBUTE, claimName, entry.getValue()); } } + + return Optional.of(commonProfile); } catch (Exception e) { logger.warn("Cannot parse access token claims", e); } } - return profile; + return Optional.ofNullable(profile); } } diff --git a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java index 0834d90b4f..002666e943 100644 --- a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java +++ b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java @@ -52,13 +52,16 @@ import org.pac4j.core.engine.DefaultCallbackLogic; import org.pac4j.core.http.adapter.HttpActionAdapter; import org.pac4j.core.profile.CommonProfile; import org.pac4j.core.profile.ProfileManager; +import org.pac4j.core.profile.UserProfile; import org.pac4j.play.PlayWebContext; import play.mvc.Result; import auth.sso.SsoManager; +import static auth.AuthUtils.ACCESS_TOKEN; +import static auth.AuthUtils.ACTOR; +import static auth.AuthUtils.createActorCookie; import static com.linkedin.metadata.Constants.*; -import static play.mvc.Results.*; -import static auth.AuthUtils.*; +import static play.mvc.Results.internalServerError; /** @@ -101,17 +104,17 @@ public class OidcCallbackLogic extends DefaultCallbackLogic profileManager) { + final ProfileManager profileManager) { log.debug("Beginning OIDC Callback Handling..."); if (profileManager.isAuthenticated()) { // If authenticated, the user should have a profile. - final CommonProfile profile = profileManager.get(true).get(); + final CommonProfile profile = (CommonProfile) profileManager.get(true).get(); log.debug(String.format("Found authenticated user with profile %s", profile.getAttributes().toString())); // Extract the User name required to log into DataHub. @@ -149,8 +152,8 @@ public class OidcCallbackLogic extends DefaultCallbackLogic { private static final String OIDC_CLIENT_NAME = "oidc"; private final OidcConfigs _oidcConfigs; - private final Client _oidcClient; // Used primarily for redirecting to IdP. + private final Client _oidcClient; // Used primarily for redirecting to IdP. public OidcProvider(final OidcConfigs configs) { _oidcConfigs = configs; @@ -35,7 +34,7 @@ public class OidcProvider implements SsoProvider { } @Override - public Client client() { + public Client client() { return _oidcClient; } @@ -49,7 +48,7 @@ public class OidcProvider implements SsoProvider { return SsoProtocol.OIDC; } - private Client createPac4jClient() { + private Client createPac4jClient() { final OidcConfiguration oidcConfiguration = new OidcConfiguration(); oidcConfiguration.setClientId(_oidcConfigs.getClientId()); oidcConfiguration.setSecret(_oidcConfigs.getClientSecret()); diff --git a/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java b/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java index f350f1b001..014632c17e 100644 --- a/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java +++ b/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java @@ -5,6 +5,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import play.mvc.Result; +import java.util.Optional; + import static play.mvc.Results.internalServerError; import static play.mvc.Results.unauthorized; @@ -26,7 +28,7 @@ public class OidcResponseErrorHandler { getError(context), getErrorDescription(context)); - if (getError(context).equals("access_denied")) { + if (getError(context).isPresent() && getError(context).get().equals("access_denied")) { return unauthorized(String.format("Access denied. " + "The OIDC service responded with 'Access denied'. " + "It seems that you don't have access to this application yet. Please apply for access. \n\n" @@ -38,18 +40,18 @@ public class OidcResponseErrorHandler { return internalServerError( String.format("Internal server error. The OIDC service responded with an error: '%s'.\n" - + "Error description: '%s'", getError(context), getErrorDescription(context))); + + "Error description: '%s'", getError(context).orElse(""), getErrorDescription(context).orElse(""))); } public static boolean isError(final PlayWebContext context) { - return getError(context) != null && !getError(context).isEmpty(); + return getError(context).isPresent() && !getError(context).get().isEmpty(); } - public static String getError(final PlayWebContext context) { + public static Optional getError(final PlayWebContext context) { return context.getRequestParameter(ERROR_FIELD_NAME); } - public static String getErrorDescription(final PlayWebContext context) { + public static Optional getErrorDescription(final PlayWebContext context) { return context.getRequestParameter(ERROR_DESCRIPTION_FIELD_NAME); } } diff --git a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java index d780f692f3..fcd9161513 100644 --- a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java +++ b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java @@ -49,14 +49,14 @@ public class CustomOidcAuthenticator implements Authenticator { protected OidcConfiguration configuration; - protected OidcClient client; + protected OidcClient client; - private ClientAuthentication clientAuthentication; + private final ClientAuthentication clientAuthentication; - public CustomOidcAuthenticator(final OidcConfiguration configuration, final OidcClient client) { - CommonHelper.assertNotNull("configuration", configuration); + public CustomOidcAuthenticator(final OidcClient client) { + CommonHelper.assertNotNull("configuration", client.getConfiguration()); CommonHelper.assertNotNull("client", client); - this.configuration = configuration; + this.configuration = client.getConfiguration(); this.client = client; // check authentication methods diff --git a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java index 5de463875d..67ec5d78ad 100644 --- a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java +++ b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java @@ -5,11 +5,10 @@ import org.pac4j.oidc.client.OidcClient; import org.pac4j.oidc.config.OidcConfiguration; import org.pac4j.oidc.credentials.extractor.OidcExtractor; import org.pac4j.oidc.logout.OidcLogoutActionBuilder; -import org.pac4j.oidc.profile.OidcProfile; import org.pac4j.oidc.profile.creator.OidcProfileCreator; -import org.pac4j.oidc.redirect.OidcRedirectActionBuilder; +import org.pac4j.oidc.redirect.OidcRedirectionActionBuilder; -public class CustomOidcClient extends OidcClient { +public class CustomOidcClient extends OidcClient { public CustomOidcClient(final OidcConfiguration configuration) { setConfiguration(configuration); @@ -19,10 +18,10 @@ public class CustomOidcClient extends OidcClient protected void clientInit() { CommonHelper.assertNotNull("configuration", getConfiguration()); getConfiguration().init(); - defaultRedirectActionBuilder(new OidcRedirectActionBuilder(getConfiguration(), this)); + defaultRedirectionActionBuilder(new OidcRedirectionActionBuilder(getConfiguration(), this)); defaultCredentialsExtractor(new OidcExtractor(getConfiguration(), this)); - defaultAuthenticator(new CustomOidcAuthenticator(getConfiguration(), this)); - defaultProfileCreator(new OidcProfileCreator<>(getConfiguration())); - defaultLogoutActionBuilder(new OidcLogoutActionBuilder<>(getConfiguration())); + defaultAuthenticator(new CustomOidcAuthenticator(this)); + defaultProfileCreator(new OidcProfileCreator<>(getConfiguration(), this)); + defaultLogoutActionBuilder(new OidcLogoutActionBuilder(getConfiguration())); } } diff --git a/datahub-frontend/app/controllers/Application.java b/datahub-frontend/app/controllers/Application.java index 01ccca6c75..59ad0d0ba1 100644 --- a/datahub-frontend/app/controllers/Application.java +++ b/datahub-frontend/app/controllers/Application.java @@ -15,7 +15,8 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; -import play.api.Play; + +import play.Environment; import play.http.HttpEntity; import play.libs.ws.InMemoryBodyWritable; import play.libs.ws.StandaloneWSClient; @@ -37,18 +38,20 @@ import play.shaded.ahc.org.asynchttpclient.DefaultAsyncHttpClientConfig; import utils.ConfigUtil; import java.time.Duration; -import static auth.AuthUtils.*; +import static auth.AuthUtils.ACTOR; +import static auth.AuthUtils.SESSION_COOKIE_GMS_TOKEN_NAME; public class Application extends Controller { - private final Config _config; private final StandaloneWSClient _ws; + private final Environment _environment; @Inject - public Application(@Nonnull Config config) { + public Application(Environment environment, @Nonnull Config config) { _config = config; _ws = createWsClient(); + _environment = environment; } /** @@ -60,9 +63,10 @@ public class Application extends Controller { */ @Nonnull private Result serveAsset(@Nullable String path) { - InputStream indexHtml = Play.current().classloader().getResourceAsStream("public/index.html"); - response().setHeader("Cache-Control", "no-cache"); - return ok(indexHtml).as("text/html"); + InputStream indexHtml = _environment.resourceAsStream("public/index.html"); + return ok(indexHtml) + .withHeader("Cache-Control", "no-cache") + .as("text/html"); } @Nonnull @@ -87,9 +91,9 @@ public class Application extends Controller { * TODO: Investigate using mutual SSL authentication to call Metadata Service. */ @Security.Authenticated(Authenticator.class) - public CompletableFuture proxy(String path) throws ExecutionException, InterruptedException { - final String authorizationHeaderValue = getAuthorizationHeaderValueToProxy(); - final String resolvedUri = mapPath(request().uri()); + public CompletableFuture proxy(String path, Http.Request request) throws ExecutionException, InterruptedException { + final String authorizationHeaderValue = getAuthorizationHeaderValueToProxy(request); + final String resolvedUri = mapPath(request.uri()); final String metadataServiceHost = ConfigUtil.getString( _config, @@ -108,14 +112,14 @@ public class Application extends Controller { // TODO: Fully support custom internal SSL. final String protocol = metadataServiceUseSsl ? "https" : "http"; - final Map> headers = request().getHeaders().toMap(); + final Map> headers = request.getHeaders().toMap(); if (headers.containsKey(Http.HeaderNames.HOST) && !headers.containsKey(Http.HeaderNames.X_FORWARDED_HOST)) { headers.put(Http.HeaderNames.X_FORWARDED_HOST, headers.get(Http.HeaderNames.HOST)); } return _ws.url(String.format("%s://%s:%s%s", protocol, metadataServiceHost, metadataServicePort, resolvedUri)) - .setMethod(request().method()) + .setMethod(request.method()) .setHeaders(headers .entrySet() .stream() @@ -129,8 +133,8 @@ public class Application extends Controller { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) ) .addHeader(Http.HeaderNames.AUTHORIZATION, authorizationHeaderValue) - .addHeader(AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER, getDataHubActorHeader()) - .setBody(new InMemoryBodyWritable(ByteString.fromByteBuffer(request().body().asBytes().asByteBuffer()), "application/json")) + .addHeader(AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER, getDataHubActorHeader(request)) + .setBody(new InMemoryBodyWritable(ByteString.fromByteBuffer(request.body().asBytes().asByteBuffer()), "application/json")) .setRequestTimeout(Duration.ofSeconds(120)) .execute() .thenApply(apiResponse -> { @@ -272,14 +276,14 @@ public class Application extends Controller { * * If neither are found, an empty string is returned. */ - private String getAuthorizationHeaderValueToProxy() { + private String getAuthorizationHeaderValueToProxy(Http.Request request) { // If the session cookie has an authorization token, use that. If there's an authorization header provided, simply // use that. String value = ""; - if (ctx().session().containsKey(SESSION_COOKIE_GMS_TOKEN_NAME)) { - value = "Bearer " + ctx().session().get(SESSION_COOKIE_GMS_TOKEN_NAME); - } else if (request().getHeaders().contains(Http.HeaderNames.AUTHORIZATION)) { - value = request().getHeaders().get(Http.HeaderNames.AUTHORIZATION).get(); + if (request.session().data().containsKey(SESSION_COOKIE_GMS_TOKEN_NAME)) { + value = "Bearer " + request.session().data().get(SESSION_COOKIE_GMS_TOKEN_NAME); + } else if (request.getHeaders().contains(Http.HeaderNames.AUTHORIZATION)) { + value = request.getHeaders().get(Http.HeaderNames.AUTHORIZATION).get(); } return value; } @@ -291,8 +295,8 @@ public class Application extends Controller { * If Metadata Service authentication is enabled, this value is not required because Actor context will most often come * from the authentication credentials provided in the Authorization header. */ - private String getDataHubActorHeader() { - String actor = ctx().session().get(ACTOR); + private String getDataHubActorHeader(Http.Request request) { + String actor = request.session().data().get(ACTOR); return actor == null ? "" : actor; } diff --git a/datahub-frontend/app/controllers/AuthenticationController.java b/datahub-frontend/app/controllers/AuthenticationController.java index 16d959cb68..2af9f5bfd8 100644 --- a/datahub-frontend/app/controllers/AuthenticationController.java +++ b/datahub-frontend/app/controllers/AuthenticationController.java @@ -10,8 +10,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.common.urn.Urn; import com.typesafe.config.Config; -import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -19,20 +19,34 @@ import javax.annotation.Nonnull; import javax.inject.Inject; import org.apache.commons.lang3.StringUtils; import org.pac4j.core.client.Client; -import org.pac4j.core.context.session.SessionStore; -import org.pac4j.core.exception.HttpAction; +import org.pac4j.core.exception.http.RedirectionAction; import org.pac4j.play.PlayWebContext; import org.pac4j.play.http.PlayHttpActionAdapter; +import org.pac4j.play.store.PlaySessionStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import play.libs.Json; import play.mvc.Controller; import play.mvc.Http; import play.mvc.Result; +import play.mvc.Results; import security.AuthenticationManager; -import static auth.AuthUtils.*; -import static org.pac4j.core.client.IndirectClient.*; +import static auth.AuthUtils.ACCESS_TOKEN; +import static auth.AuthUtils.ACTOR; +import static auth.AuthUtils.DEFAULT_ACTOR_URN; +import static auth.AuthUtils.DEFAULT_SESSION_TTL_HOURS; +import static auth.AuthUtils.EMAIL; +import static auth.AuthUtils.FULL_NAME; +import static auth.AuthUtils.INVITE_TOKEN; +import static auth.AuthUtils.LOGIN_ROUTE; +import static auth.AuthUtils.PASSWORD; +import static auth.AuthUtils.RESET_TOKEN; +import static auth.AuthUtils.SESSION_TTL_CONFIG_PATH; +import static auth.AuthUtils.TITLE; +import static auth.AuthUtils.USER_NAME; +import static auth.AuthUtils.createActorCookie; +import static org.pac4j.core.client.IndirectClient.ATTEMPTED_AUTHENTICATION_SUFFIX; // TODO add logging. @@ -42,6 +56,8 @@ public class AuthenticationController extends Controller { private static final String ERROR_MESSAGE_URI_PARAM = "error_msg"; private static final String SSO_DISABLED_ERROR_MESSAGE = "SSO is not configured"; + private static final String SSO_NO_REDIRECT_MESSAGE = "SSO is configured, however missing redirect from idp"; + private final Logger _logger = LoggerFactory.getLogger(AuthenticationController.class.getName()); private final Config _configs; private final JAASConfigs _jaasConfigs; @@ -51,7 +67,7 @@ public class AuthenticationController extends Controller { private org.pac4j.core.config.Config _ssoConfig; @Inject - private SessionStore _playSessionStore; + private PlaySessionStore _playSessionStore; @Inject private SsoManager _ssoManager; @@ -80,25 +96,27 @@ public class AuthenticationController extends Controller { final Optional maybeRedirectPath = Optional.ofNullable(request.getQueryString(AUTH_REDIRECT_URI_PARAM)); final String redirectPath = maybeRedirectPath.orElse("/"); - if (AuthUtils.hasValidSessionCookie(ctx())) { - return redirect(redirectPath); + if (AuthUtils.hasValidSessionCookie(request)) { + return Results.redirect(redirectPath); } // 1. If SSO is enabled, redirect to IdP if not authenticated. if (_ssoManager.isSsoEnabled()) { - return redirectToIdentityProvider(); + return redirectToIdentityProvider(request).orElse( + Results.redirect(LOGIN_ROUTE + String.format("?%s=%s", ERROR_MESSAGE_URI_PARAM, SSO_NO_REDIRECT_MESSAGE)) + ); } // 2. If either JAAS auth or Native auth is enabled, fallback to it if (_jaasConfigs.isJAASEnabled() || _nativeAuthenticationConfigs.isNativeAuthenticationEnabled()) { - return redirect( + return Results.redirect( LOGIN_ROUTE + String.format("?%s=%s", AUTH_REDIRECT_URI_PARAM, encodeRedirectUri(redirectPath))); } // 3. If no auth enabled, fallback to using default user account & redirect. // Generate GMS session token, TODO: final String accessToken = _authClient.generateSessionTokenForUser(DEFAULT_ACTOR_URN.getId()); - return redirect(redirectPath).withSession(createSessionMap(DEFAULT_ACTOR_URN.toString(), accessToken)) + return Results.redirect(redirectPath).withSession(createSessionMap(DEFAULT_ACTOR_URN.toString(), accessToken)) .withCookies(createActorCookie(DEFAULT_ACTOR_URN.toString(), _configs.hasPath(SESSION_TTL_CONFIG_PATH) ? _configs.getInt(SESSION_TTL_CONFIG_PATH) : DEFAULT_SESSION_TTL_HOURS)); @@ -108,11 +126,13 @@ public class AuthenticationController extends Controller { * Redirect to the identity provider for authentication. */ @Nonnull - public Result sso() { + public Result sso(Http.Request request) { if (_ssoManager.isSsoEnabled()) { - return redirectToIdentityProvider(); + return redirectToIdentityProvider(request).orElse( + Results.redirect(LOGIN_ROUTE + String.format("?%s=%s", ERROR_MESSAGE_URI_PARAM, SSO_NO_REDIRECT_MESSAGE)) + ); } - return redirect(LOGIN_ROUTE + String.format("?%s=%s", ERROR_MESSAGE_URI_PARAM, SSO_DISABLED_ERROR_MESSAGE)); + return Results.redirect(LOGIN_ROUTE + String.format("?%s=%s", ERROR_MESSAGE_URI_PARAM, SSO_DISABLED_ERROR_MESSAGE)); } /** @@ -131,7 +151,7 @@ public class AuthenticationController extends Controller { String message = "Neither JAAS nor native authentication is enabled on the server."; final ObjectNode error = Json.newObject(); error.put("message", message); - return badRequest(error); + return Results.badRequest(error); } final JsonNode json = request.body().asJson(); @@ -140,14 +160,14 @@ public class AuthenticationController extends Controller { if (StringUtils.isBlank(username)) { JsonNode invalidCredsJson = Json.newObject().put("message", "User name must not be empty."); - return badRequest(invalidCredsJson); + return Results.badRequest(invalidCredsJson); } JsonNode invalidCredsJson = Json.newObject().put("message", "Invalid Credentials"); boolean loginSucceeded = tryLogin(username, password); if (!loginSucceeded) { - return badRequest(invalidCredsJson); + return Results.badRequest(invalidCredsJson); } final Urn actorUrn = new CorpuserUrn(username); @@ -167,7 +187,7 @@ public class AuthenticationController extends Controller { String message = "Native authentication is not enabled on the server."; final ObjectNode error = Json.newObject(); error.put("message", message); - return badRequest(error); + return Results.badRequest(error); } final JsonNode json = request.body().asJson(); @@ -179,27 +199,27 @@ public class AuthenticationController extends Controller { if (StringUtils.isBlank(fullName)) { JsonNode invalidCredsJson = Json.newObject().put("message", "Full name must not be empty."); - return badRequest(invalidCredsJson); + return Results.badRequest(invalidCredsJson); } if (StringUtils.isBlank(email)) { JsonNode invalidCredsJson = Json.newObject().put("message", "Email must not be empty."); - return badRequest(invalidCredsJson); + return Results.badRequest(invalidCredsJson); } if (StringUtils.isBlank(password)) { JsonNode invalidCredsJson = Json.newObject().put("message", "Password must not be empty."); - return badRequest(invalidCredsJson); + return Results.badRequest(invalidCredsJson); } if (StringUtils.isBlank(title)) { JsonNode invalidCredsJson = Json.newObject().put("message", "Title must not be empty."); - return badRequest(invalidCredsJson); + return Results.badRequest(invalidCredsJson); } if (StringUtils.isBlank(inviteToken)) { JsonNode invalidCredsJson = Json.newObject().put("message", "Invite token must not be empty."); - return badRequest(invalidCredsJson); + return Results.badRequest(invalidCredsJson); } final Urn userUrn = new CorpuserUrn(email); @@ -231,17 +251,17 @@ public class AuthenticationController extends Controller { if (StringUtils.isBlank(email)) { JsonNode invalidCredsJson = Json.newObject().put("message", "Email must not be empty."); - return badRequest(invalidCredsJson); + return Results.badRequest(invalidCredsJson); } if (StringUtils.isBlank(password)) { JsonNode invalidCredsJson = Json.newObject().put("message", "Password must not be empty."); - return badRequest(invalidCredsJson); + return Results.badRequest(invalidCredsJson); } if (StringUtils.isBlank(resetToken)) { JsonNode invalidCredsJson = Json.newObject().put("message", "Reset token must not be empty."); - return badRequest(invalidCredsJson); + return Results.badRequest(invalidCredsJson); } final Urn userUrn = new CorpuserUrn(email); @@ -251,9 +271,9 @@ public class AuthenticationController extends Controller { return createSession(userUrnString, accessToken); } - private Result redirectToIdentityProvider() { - final PlayWebContext playWebContext = new PlayWebContext(ctx(), _playSessionStore); - final Client client = _ssoManager.getSsoProvider().client(); + private Optional redirectToIdentityProvider(Http.RequestHeader request) { + final PlayWebContext playWebContext = new PlayWebContext(request, _playSessionStore); + final Client client = _ssoManager.getSsoProvider().client(); // This is to prevent previous login attempts from being cached. // We replicate the logic here, which is buried in the Pac4j client. @@ -263,23 +283,19 @@ public class AuthenticationController extends Controller { } try { - final HttpAction action = client.redirect(playWebContext); - return new PlayHttpActionAdapter().adapt(action.getCode(), playWebContext); + final Optional action = client.getRedirectionAction(playWebContext); + return action.map(act -> new PlayHttpActionAdapter().adapt(act, playWebContext)); } catch (Exception e) { _logger.error("Caught exception while attempting to redirect to SSO identity provider! It's likely that SSO integration is mis-configured", e); - return redirect( + return Optional.of(Results.redirect( String.format("/login?error_msg=%s", URLEncoder.encode("Failed to redirect to Single Sign-On provider. Please contact your DataHub Administrator, " - + "or refer to server logs for more information."))); + + "or refer to server logs for more information.", StandardCharsets.UTF_8)))); } } private String encodeRedirectUri(final String redirectUri) { - try { - return URLEncoder.encode(redirectUri, "UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(String.format("Failed to encode redirect URI %s", redirectUri), e); - } + return URLEncoder.encode(redirectUri, StandardCharsets.UTF_8); } private boolean tryLogin(String username, String password) { @@ -310,7 +326,7 @@ public class AuthenticationController extends Controller { private Result createSession(String userUrnString, String accessToken) { int ttlInHours = _configs.hasPath(SESSION_TTL_CONFIG_PATH) ? _configs.getInt(SESSION_TTL_CONFIG_PATH) : DEFAULT_SESSION_TTL_HOURS; - return ok().withSession(createSessionMap(userUrnString, accessToken)) + return Results.ok().withSession(createSessionMap(userUrnString, accessToken)) .withCookies(createActorCookie(userUrnString, ttlInHours)); } diff --git a/datahub-frontend/app/controllers/CentralLogoutController.java b/datahub-frontend/app/controllers/CentralLogoutController.java index 38011cbc03..0c25fa9832 100644 --- a/datahub-frontend/app/controllers/CentralLogoutController.java +++ b/datahub-frontend/app/controllers/CentralLogoutController.java @@ -4,10 +4,12 @@ import com.typesafe.config.Config; import java.net.URLEncoder; import lombok.extern.slf4j.Slf4j; import org.pac4j.play.LogoutController; +import play.mvc.Http; import play.mvc.Result; +import play.mvc.Results; import javax.inject.Inject; -import java.util.concurrent.ExecutionException; +import java.nio.charset.StandardCharsets; /** * Responsible for handling logout logic with oidc providers @@ -31,18 +33,20 @@ public class CentralLogoutController extends LogoutController { /** * logout() method should not be called if oidc is not enabled */ - public Result executeLogout() throws ExecutionException, InterruptedException { + public Result executeLogout(Http.Request request) { if (_isOidcEnabled) { try { - return logout().toCompletableFuture().get().withNewSession(); + return Results.redirect(DEFAULT_BASE_URL_PATH) + .removingFromSession(request); } catch (Exception e) { log.error("Caught exception while attempting to perform SSO logout! It's likely that SSO integration is mis-configured.", e); return redirect( String.format("/login?error_msg=%s", URLEncoder.encode("Failed to sign out using Single Sign-On provider. Please contact your DataHub Administrator, " - + "or refer to server logs for more information."))); + + "or refer to server logs for more information.", StandardCharsets.UTF_8))); } } - return redirect(DEFAULT_BASE_URL_PATH).withNewSession(); + return Results.redirect(DEFAULT_BASE_URL_PATH) + .withNewSession(); } } \ No newline at end of file diff --git a/datahub-frontend/app/controllers/SsoCallbackController.java b/datahub-frontend/app/controllers/SsoCallbackController.java index 2bc1753798..a32cfe6245 100644 --- a/datahub-frontend/app/controllers/SsoCallbackController.java +++ b/datahub-frontend/app/controllers/SsoCallbackController.java @@ -4,6 +4,7 @@ import client.AuthServiceClient; import com.datahub.authentication.Authentication; import com.linkedin.entity.client.EntityClient; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import javax.annotation.Nonnull; @@ -14,10 +15,12 @@ import org.pac4j.core.engine.CallbackLogic; import org.pac4j.core.http.adapter.HttpActionAdapter; import org.pac4j.play.CallbackController; import org.pac4j.play.PlayWebContext; +import play.mvc.Http; import play.mvc.Result; import auth.sso.oidc.OidcCallbackLogic; import auth.sso.SsoManager; import auth.sso.SsoProvider; +import play.mvc.Results; /** @@ -44,21 +47,21 @@ public class SsoCallbackController extends CallbackController { setCallbackLogic(new SsoCallbackLogic(ssoManager, systemAuthentication, entityClient, authClient)); } - public CompletionStage handleCallback(String protocol) { + public CompletionStage handleCallback(String protocol, Http.Request request) { if (shouldHandleCallback(protocol)) { log.debug(String.format("Handling SSO callback. Protocol: %s", protocol)); - return callback().handle((res, e) -> { + return callback(request).handle((res, e) -> { if (e != null) { log.error("Caught exception while attempting to handle SSO callback! It's likely that SSO integration is mis-configured.", e); - return redirect( + return Results.redirect( String.format("/login?error_msg=%s", URLEncoder.encode("Failed to sign in using Single Sign-On provider. Please contact your DataHub Administrator, " - + "or refer to server logs for more information."))); + + "or refer to server logs for more information.", StandardCharsets.UTF_8))); } return res; }); } - return CompletableFuture.completedFuture(internalServerError( + return CompletableFuture.completedFuture(Results.internalServerError( String.format("Failed to perform SSO callback. SSO is not enabled for protocol: %s", protocol))); } diff --git a/datahub-frontend/app/controllers/TrackingController.java b/datahub-frontend/app/controllers/TrackingController.java index 980d2f4bcd..83bb7957b9 100644 --- a/datahub-frontend/app/controllers/TrackingController.java +++ b/datahub-frontend/app/controllers/TrackingController.java @@ -74,7 +74,7 @@ public class TrackingController extends Controller { } catch (Exception e) { return badRequest(); } - final String actor = ctx().session().get(ACTOR); + final String actor = request.session().data().get(ACTOR); try { _logger.debug(String.format("Emitting product analytics event. actor: %s, event: %s", actor, event)); final ProducerRecord record = new ProducerRecord<>( diff --git a/datahub-frontend/app/server/CustomAkkaHttpServer.scala b/datahub-frontend/app/server/CustomAkkaHttpServer.scala index 53824eccf6..9cd2529750 100644 --- a/datahub-frontend/app/server/CustomAkkaHttpServer.scala +++ b/datahub-frontend/app/server/CustomAkkaHttpServer.scala @@ -2,7 +2,6 @@ package server import play.api.Logger import play.core.server.AkkaHttpServer -import play.core.server.AkkaHttpServerProvider import play.core.server.ServerProvider import akka.http.scaladsl.settings.ParserSettings diff --git a/datahub-frontend/build.gradle b/datahub-frontend/build.gradle index c63e4ef9d0..b6c02413b5 100644 --- a/datahub-frontend/build.gradle +++ b/datahub-frontend/build.gradle @@ -1,7 +1,7 @@ plugins { id "io.github.kobylynskyi.graphql.codegen" version "4.1.1" } -apply plugin: 'java' +apply plugin: 'scala' apply from: './play.gradle' diff --git a/datahub-frontend/conf/application.conf b/datahub-frontend/conf/application.conf index de5a19879e..9c83c97c22 100644 --- a/datahub-frontend/conf/application.conf +++ b/datahub-frontend/conf/application.conf @@ -19,25 +19,39 @@ play.application.loader = play.inject.guice.GuiceApplicationLoader # Play http parser settings # # # # Increase default buffer size to handle large post request -play.http.parser.maxMemoryBuffer = ${DATAHUB_PLAY_MEM_BUFFER_SIZE} +play.http.parser.maxMemoryBuffer = 10MB +play.http.parser.maxMemoryBuffer = ${?DATAHUB_PLAY_MEM_BUFFER_SIZE} # TODO: Disable legacy URL encoding eventually play.modules.disabled += "play.api.mvc.CookiesModule" play.modules.enabled += "play.api.mvc.LegacyCookiesModule" play.modules.enabled += "auth.AuthModule" -# Legacy Configuration to avoid code changes, update to modern approaches eventually -play.allowHttpContext = true -play.allowGlobalApplication = true - # We override the Akka server provider to allow setting the max header count to a higher value # This is useful while using proxies like Envoy that result in the frontend server rejecting GMS # responses as there's more than the max of 64 allowed headers play.server.provider = server.CustomAkkaHttpServerProvider play.http.server.akka.max-header-count = 64 play.http.server.akka.max-header-count = ${?DATAHUB_AKKA_MAX_HEADER_COUNT} -play.server.akka.max-header-value-length = 8k -play.server.akka.max-header-value-length = ${?DATAHUB_AKKA_MAX_HEADER_VALUE_LENGTH} +play.server.akka.max-header-size = 8k +play.server.akka.max-header-size = ${?DATAHUB_AKKA_MAX_HEADER_VALUE_LENGTH} + +play.filters { + enabled = [ + play.filters.gzip.GzipFilter + ] + + gzip { + contentType { + # If non empty, then a response will only be compressed if its content type is in this list. + whiteList = [ "text/*", "application/javascript", "application/json" ] + + # The black list is only used if the white list is empty. + # Compress all responses except the ones whose content type is in this list. + blackList = [] + } + } +} # pac4j configuration # default to PlayCookieSessionStore to keep datahub-frontend's statelessness @@ -62,7 +76,8 @@ pac4j.sessionStore.provider= ${?PAC4J_SESSIONSTORE_PROVIDER} # You can disable evolutions for a specific datasource if necessary # play.evolutions.db.default.enabled=false -app.version = ${DATAHUB_APP_VERSION} +app.version = 0.0.0 +app.version = ${?DATAHUB_APP_VERSION} dataset.hdfs_browser.link = "" linkedin.internal = false @@ -174,7 +189,8 @@ analytics.enabled = ${?DATAHUB_ANALYTICS_ENABLED} # Kafka Producer Configuration analytics.kafka.bootstrap.server = ${KAFKA_BOOTSTRAP_SERVER} -analytics.tracking.topic = ${DATAHUB_TRACKING_TOPIC} +analytics.tracking.topic = DataHubUsageEvent_v1 +analytics.tracking.topic = ${?DATAHUB_TRACKING_TOPIC} # Kafka Producer SSL Configs. All must be provided to enable SSL. analytics.kafka.security.protocol = ${?KAFKA_PROPERTIES_SECURITY_PROTOCOL} @@ -194,8 +210,8 @@ analytics.kafka.sasl.kerberos.service.name = ${?KAFKA_PROPERTIES_SASL_KERBEROS_S analytics.kafka.sasl.login.callback.handler.class = ${?KAFKA_PROPERTIES_SASL_LOGIN_CALLBACK_HANDLER_CLASS} # Required Elastic Client Configuration -analytics.elastic.host = ${ELASTIC_CLIENT_HOST} -analytics.elastic.port = ${ELASTIC_CLIENT_PORT} +analytics.elastic.host = ${?ELASTIC_CLIENT_HOST} +analytics.elastic.port = ${?ELASTIC_CLIENT_PORT} # Optional Elastic Client Configurations analytics.elastic.threadCount = ${?ELASTIC_CLIENT_THREAD_COUNT} diff --git a/datahub-frontend/conf/routes b/datahub-frontend/conf/routes index 03ad776dfa..38e6f76902 100644 --- a/datahub-frontend/conf/routes +++ b/datahub-frontend/conf/routes @@ -7,37 +7,38 @@ GET / controllers.Application.index(path="index.html") GET /admin controllers.Application.healthcheck() +GET /health controllers.Application.healthcheck() GET /config controllers.Application.appConfig() # Routes used exclusively by the React application. # Authentication in React GET /authenticate controllers.AuthenticationController.authenticate(request: Request) -GET /sso controllers.AuthenticationController.sso() +GET /sso controllers.AuthenticationController.sso(request: Request) POST /logIn controllers.AuthenticationController.logIn(request: Request) POST /signUp controllers.AuthenticationController.signUp(request: Request) POST /resetNativeUserCredentials controllers.AuthenticationController.resetNativeUserCredentials(request: Request) -GET /callback/:protocol controllers.SsoCallbackController.handleCallback(protocol: String) -POST /callback/:protocol controllers.SsoCallbackController.handleCallback(protocol: String) -GET /logOut controllers.CentralLogoutController.executeLogout() +GET /callback/:protocol controllers.SsoCallbackController.handleCallback(protocol: String, request: Request) +POST /callback/:protocol controllers.SsoCallbackController.handleCallback(protocol: String, request: Request) +GET /logOut controllers.CentralLogoutController.executeLogout(request: Request) # Proxies API requests to the metadata service api -GET /api/*path controllers.Application.proxy(path) -POST /api/*path controllers.Application.proxy(path) -DELETE /api/*path controllers.Application.proxy(path) -PUT /api/*path controllers.Application.proxy(path) +GET /api/*path controllers.Application.proxy(path: String, request: Request) +POST /api/*path controllers.Application.proxy(path: String, request: Request) +DELETE /api/*path controllers.Application.proxy(path: String, request: Request) +PUT /api/*path controllers.Application.proxy(path: String, request: Request) # Proxies API requests to the metadata service api -GET /openapi/*path controllers.Application.proxy(path) -POST /openapi/*path controllers.Application.proxy(path) -DELETE /openapi/*path controllers.Application.proxy(path) -PUT /openapi/*path controllers.Application.proxy(path) +GET /openapi/*path controllers.Application.proxy(path: String, request: Request) +POST /openapi/*path controllers.Application.proxy(path: String, request: Request) +DELETE /openapi/*path controllers.Application.proxy(path: String, request: Request) +PUT /openapi/*path controllers.Application.proxy(path: String, request: Request) # Map static resources from the /public folder to the /assets URL path -GET /assets/*file controllers.Assets.at(path="/public", file) +GET /assets/*file controllers.Assets.at(path="/public", file) # Analytics route -POST /track controllers.TrackingController.track(request: Request) +POST /track controllers.TrackingController.track(request: Request) # Wildcard route accepts any routes and delegates to serveAsset which in turn serves the React Bundle -GET /*path controllers.Application.index(path) +GET /*path controllers.Application.index(path) diff --git a/datahub-frontend/play.gradle b/datahub-frontend/play.gradle index f6ecd57534..4d0a340d9c 100644 --- a/datahub-frontend/play.gradle +++ b/datahub-frontend/play.gradle @@ -10,22 +10,28 @@ tasks.withType(PlayRun) { configurations { assets + play } dependencies { assets project(path: ':datahub-web-react', configuration: 'assets') constraints { - play('org.springframework:spring-core:5.2.3.RELEASE') + play(externalDependency.springCore) + play(externalDependency.springBeans) + play(externalDependency.springContext) play(externalDependency.jacksonDataBind) - play('com.nimbusds:nimbus-jose-jwt:7.9') - play('com.typesafe.akka:akka-actor_2.12:2.5.16') - play('net.minidev:json-smart:2.4.1') - play('io.netty:netty-all:4.1.68.Final') + play('com.nimbusds:oauth2-oidc-sdk:8.36.2') + play('com.nimbusds:nimbus-jose-jwt:8.18') + play('com.typesafe.akka:akka-actor_2.12:2.6.20') + play('net.minidev:json-smart:2.4.8') + play('io.netty:netty-all:4.1.85.Final') } + compile project(":metadata-service:restli-client") compile project(":metadata-service:auth-api") + implementation externalDependency.jettyJaas implementation externalDependency.graphqlJava implementation externalDependency.antlr4Runtime @@ -39,7 +45,8 @@ dependencies { exclude group: "net.minidev", module: "json-smart" exclude group: "com.nimbusds", module: "nimbus-jose-jwt" } - implementation "com.nimbusds:nimbus-jose-jwt:7.9" + + implementation 'com.nimbusds:nimbus-jose-jwt:8.18' implementation externalDependency.jsonSmart implementation externalDependency.playPac4j implementation externalDependency.shiroCore @@ -48,12 +55,16 @@ dependencies { implementation externalDependency.playWs implementation externalDependency.playServer implementation externalDependency.playAkkaHttpServer + implementation externalDependency.playFilters implementation externalDependency.kafkaClients implementation externalDependency.awsMskIamAuth testImplementation externalDependency.mockito testImplementation externalDependency.playTest - testCompile externalDependency.testng + testImplementation 'no.nav.security:mock-oauth2-server:0.3.1' + testImplementation 'org.junit-pioneer:junit-pioneer:1.9.1' + testImplementation externalDependency.junitJupiterApi + testRuntime externalDependency.junitJupiterEngine implementation externalDependency.slf4jApi compileOnly externalDependency.lombok @@ -62,16 +73,19 @@ dependencies { exclude group: 'com.typesafe.akka', module: 'akka-http-core_2.12' } runtime externalDependency.playGuice + implementation externalDependency.log4j2Api + implementation externalDependency.logbackClassic annotationProcessor externalDependency.lombok } dist.dependsOn(':datahub-web-react:copyAssets') +test.dependsOn(':datahub-frontend:testResources') play { platform { - playVersion = '2.7.6' + playVersion = '2.8.18' scalaVersion = '2.12' javaVersion = JavaVersion.VERSION_11 } @@ -82,7 +96,7 @@ play { model { components { play { - platform play: '2.7.6', scala: '2.12', java: '11' + platform play: '2.8.18', scala: '2.12', java: '11' injectedRoutesGenerator = true binaries.all { @@ -99,6 +113,26 @@ model { } } +test { + useJUnitPlatform() +} + +sourceSets { + test { + resources { + srcDirs = ['test/resources'] + } + } +} + +// minimal files for low-level tests +task testResources { + copy { + from '../datahub-web-react/public' + into 'test/resources/public' + } +} + task unzipAssets(type: Copy, dependsOn: [configurations.assets, ':datahub-web-react:yarnBuild']) { into "${buildDir}/assets" from { @@ -121,4 +155,5 @@ clean { delete 'public/logo.png' delete 'public/index.html' delete 'public/favicon.ico' + delete 'test/resources/public' } diff --git a/datahub-frontend/test/app/ApplicationTest.java b/datahub-frontend/test/app/ApplicationTest.java index 58af440b32..58a602532f 100644 --- a/datahub-frontend/test/app/ApplicationTest.java +++ b/datahub-frontend/test/app/ApplicationTest.java @@ -1,11 +1,147 @@ package app; -import org.junit.Test; +import controllers.routes; +import no.nav.security.mock.oauth2.MockOAuth2Server; +import no.nav.security.mock.oauth2.token.DefaultOAuth2TokenCallback; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junitpioneer.jupiter.SetEnvironmentVariable; +import org.openqa.selenium.Cookie; +import play.Application; +import play.Environment; +import play.Mode; +import play.inject.guice.GuiceApplicationBuilder; +import play.mvc.Http; +import play.mvc.Result; +import play.test.Helpers; +import play.test.TestBrowser; +import play.test.WithBrowser; -public class ApplicationTest { +import java.io.IOException; +import java.net.InetAddress; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static play.mvc.Http.Status.NOT_FOUND; +import static play.mvc.Http.Status.OK; +import static play.test.Helpers.fakeRequest; +import static play.test.Helpers.route; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@SetEnvironmentVariable(key = "DATAHUB_SECRET", value = "test") +@SetEnvironmentVariable(key = "KAFKA_BOOTSTRAP_SERVER", value = "") +@SetEnvironmentVariable(key = "DATAHUB_ANALYTICS_ENABLED", value = "false") +@SetEnvironmentVariable(key = "AUTH_OIDC_ENABLED", value = "true") +@SetEnvironmentVariable(key = "AUTH_OIDC_JIT_PROVISIONING_ENABLED", value = "false") +@SetEnvironmentVariable(key = "AUTH_OIDC_CLIENT_ID", value = "testclient") +@SetEnvironmentVariable(key = "AUTH_OIDC_CLIENT_SECRET", value = "testsecret") +public class ApplicationTest extends WithBrowser { + private static final String ISSUER_ID = "testIssuer"; + + @Override + protected Application provideApplication() { + return new GuiceApplicationBuilder() + .configure("metadataService.port", String.valueOf(gmsServerPort())) + .configure("auth.baseUrl", "http://localhost:" + providePort()) + .configure("auth.oidc.discoveryUri", "http://localhost:" + oauthServerPort() + + "/testIssuer/.well-known/openid-configuration") + .in(new Environment(Mode.TEST)).build(); + } + + public int oauthServerPort() { + return providePort() + 1; + } + + public int gmsServerPort() { + return providePort() + 2; + } + + @Override + protected TestBrowser provideBrowser(int port) { + return Helpers.testBrowser(providePort()); + } + + private MockOAuth2Server _oauthServer; + private MockWebServer _gmsServer; + + private String _wellKnownUrl; + + @BeforeAll + public void init() throws IOException, InterruptedException { + _gmsServer = new MockWebServer(); + _gmsServer.enqueue(new MockResponse().setBody("{\"value\":\"urn:li:corpuser:testUser@myCompany.com\"}")); + _gmsServer.enqueue(new MockResponse().setBody("{\"accessToken\":\"faketoken_YCpYIrjQH4sD3_rAc3VPPFg4\"}")); + _gmsServer.start(gmsServerPort()); + + _oauthServer = new MockOAuth2Server(); + _oauthServer.enqueueCallback( + new DefaultOAuth2TokenCallback(ISSUER_ID, "testUser", List.of(), Map.of( + "email", "testUser@myCompany.com", + "groups", "myGroup" + ), 600) + ); + _oauthServer.start(InetAddress.getByName("localhost"), oauthServerPort()); + + // Discovery url to authorization server metadata + _wellKnownUrl = _oauthServer.wellKnownUrl(ISSUER_ID).toString(); + + startServer(); + createBrowser(); + Thread.sleep(5000); + } + + @AfterAll + public void shutdown() throws IOException { + if (_gmsServer != null) { + _gmsServer.shutdown(); + } + if (_oauthServer != null) { + _oauthServer.shutdown(); + } + stopServer(); + } @Test - public void renderTemplate() { + public void testHealth() { + Http.RequestBuilder request = fakeRequest(routes.Application.healthcheck()); + + Result result = route(app, request); + assertEquals(OK, result.status()); } + + @Test + public void testIndex() { + Http.RequestBuilder request = fakeRequest(routes.Application.index("")); + + Result result = route(app, request); + assertEquals(OK, result.status()); + } + + @Test + public void testIndexNotFound() { + Http.RequestBuilder request = fakeRequest(routes.Application.index("/other")); + Result result = route(app, request); + assertEquals(NOT_FOUND, result.status()); + } + + @Test + public void testOpenIdConfig() { + assertEquals("http://localhost:" + oauthServerPort() + + "/testIssuer/.well-known/openid-configuration", _wellKnownUrl); + } + + @Test + public void testHappyPathOidc() throws InterruptedException { + browser.goTo("/authenticate"); + assertEquals("", browser.url()); + Cookie actorCookie = browser.getCookie("actor"); + assertEquals("urn:li:corpuser:testUser@myCompany.com", actorCookie.getValue()); + } + } diff --git a/datahub-frontend/test/security/DummyLoginModuleTest.java b/datahub-frontend/test/security/DummyLoginModuleTest.java index da5316691c..6727513d88 100644 --- a/datahub-frontend/test/security/DummyLoginModuleTest.java +++ b/datahub-frontend/test/security/DummyLoginModuleTest.java @@ -1,12 +1,13 @@ package security; import com.sun.security.auth.callback.TextCallbackHandler; +import org.junit.jupiter.api.Test; + import java.util.HashMap; import javax.security.auth.Subject; import javax.security.auth.login.LoginException; -import org.testng.annotations.Test; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.*; public class DummyLoginModuleTest { @@ -17,10 +18,10 @@ public class DummyLoginModuleTest { lmodule.initialize(new Subject(), new TextCallbackHandler(), null, new HashMap<>()); try { - assertTrue("Failed to login", lmodule.login()); - assertTrue("Failed to logout", lmodule.logout()); - assertTrue("Failed to commit", lmodule.commit()); - assertTrue("Failed to abort", lmodule.abort()); + assertTrue(lmodule.login(), "Failed to login"); + assertTrue(lmodule.logout(), "Failed to logout"); + assertTrue(lmodule.commit(), "Failed to commit"); + assertTrue(lmodule.abort(), "Failed to abort"); } catch (LoginException e) { fail(e.toString()); } diff --git a/datahub-frontend/test/security/OidcConfigurationTest.java b/datahub-frontend/test/security/OidcConfigurationTest.java index d11d9daaed..ed16014b58 100644 --- a/datahub-frontend/test/security/OidcConfigurationTest.java +++ b/datahub-frontend/test/security/OidcConfigurationTest.java @@ -19,11 +19,12 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; import org.pac4j.oidc.client.OidcClient; -import org.testng.annotations.Test; import static auth.sso.oidc.OidcConfigs.*; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertEquals; public class OidcConfigurationTest { diff --git a/datahub-frontend/test/utils/SearchUtilTest.java b/datahub-frontend/test/utils/SearchUtilTest.java index a111c82342..428566ae3f 100644 --- a/datahub-frontend/test/utils/SearchUtilTest.java +++ b/datahub-frontend/test/utils/SearchUtilTest.java @@ -1,8 +1,8 @@ package utils; -import org.testng.annotations.Test; +import org.junit.jupiter.api.Test; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertEquals; public class SearchUtilTest { @Test diff --git a/smoke-test/cypress-dev.sh b/smoke-test/cypress-dev.sh new file mode 100755 index 0000000000..3e266bd3ec --- /dev/null +++ b/smoke-test/cypress-dev.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -euxo pipefail + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR" + +python3 -m venv venv +source venv/bin/activate +pip install --upgrade pip wheel setuptools +pip install -r requirements.txt + +mkdir -p ~/.datahub/plugins/frontend/auth/ +echo "test_user:test_pass" > ~/.datahub/plugins/frontend/auth/user.props + +echo "DATAHUB_VERSION = ${DATAHUB_VERSION:=acryl-datahub 0.0.0.dev0}" +DATAHUB_TELEMETRY_ENABLED=false \ +DOCKER_COMPOSE_BASE="file://$( dirname "$DIR" )" \ +datahub docker quickstart --build-locally --standalone_consumers --dump-logs-on-failure + +python -c 'from tests.cypress.integration_test import ingest_data; ingest_data()' + +cd tests/cypress +npm install + +export CYPRESS_ADMIN_USERNAME=${ADMIN_USERNAME:-test_user} +export CYPRESS_ADMIN_PASSWORD=${ADMIN_PASSWORD:-test_pass} + +npx cypress open \ No newline at end of file diff --git a/smoke-test/tests/cypress/README.txt b/smoke-test/tests/cypress/README.txt index 3fbbd8492c..031a1dd889 100644 --- a/smoke-test/tests/cypress/README.txt +++ b/smoke-test/tests/cypress/README.txt @@ -1,3 +1,8 @@ +# Quick Run Tests with UI + +cd smoke-test/ +./cypress-dev.sh + # Running Cypress Tests Locally 1. Make sure the packages are installed. It uses some node modules to run locally. Run `yarn install` from this directory. If you don't have `yarn`, download it. @@ -9,3 +14,5 @@ 4. Set the port that you want to run your cypress tests against in ./cypress.json. The default is 9002- if you are developing on react locally, you probably want 3000. Do not commit this change to github. 5. Now, start the local cypress server: `npx cypress open`. + + diff --git a/smoke-test/tests/cypress/integration_test.py b/smoke-test/tests/cypress/integration_test.py index 13ee156798..c1f229f758 100644 --- a/smoke-test/tests/cypress/integration_test.py +++ b/smoke-test/tests/cypress/integration_test.py @@ -99,8 +99,7 @@ for id_list in ONBOARDING_ID_LISTS: ONBOARDING_IDS.extend(id_list) -@pytest.fixture(scope="module", autouse=True) -def ingest_cleanup_data(): +def ingest_data(): print("creating onboarding data file") create_datahub_step_state_aspects( get_admin_username(), @@ -113,6 +112,11 @@ def ingest_cleanup_data(): ingest_file_via_rest(f"{CYPRESS_TEST_DATA_DIR}/{TEST_DBT_DATA_FILENAME}") ingest_file_via_rest(f"{CYPRESS_TEST_DATA_DIR}/{TEST_SCHEMA_BLAME_DATA_FILENAME}") ingest_file_via_rest(f"{CYPRESS_TEST_DATA_DIR}/{TEST_ONBOARDING_DATA_FILENAME}") + + +@pytest.fixture(scope="module", autouse=True) +def ingest_cleanup_data(): + ingest_data() yield print("removing test data") delete_urns_from_file(f"{CYPRESS_TEST_DATA_DIR}/{TEST_DATA_FILENAME}")