yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-23 08:38:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added an end point in wherehows-frontend to display all the libs from container directory (#639)

Browse Source
This commit is contained in:
Shridhar Sattur 2017-08-07 13:55:05 -07:00 committed by Mars Lan
parent 8ec3113a33
commit aa28eba78a
5 changed files with 175 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
wherehows-backend/app/controllers/Application.java
View File

@ -1,24 +1,30 @@
/** /**
* Copyright 2015 LinkedIn Corp. All rights reserved. * Copyright 2015 LinkedIn Corp. All rights reserved.
* * <p>
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
* * <p>
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* * <p>
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/ */
package controllers; package controllers;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.lang.reflect.Constructor; import java.lang.reflect.Constructor;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import javax.persistence.EntityManagerFactory; import javax.persistence.EntityManagerFactory;
import javax.persistence.Persistence; import javax.persistence.Persistence;
import org.hibernate.hikaricp.internal.HikariCPConnectionProvider; import org.hibernate.hikaricp.internal.HikariCPConnectionProvider;
import play.Logger;
import play.Play; import play.Play;
import play.mvc.Controller; import play.mvc.Controller;
import play.mvc.Result; import play.mvc.Result;
@ -27,53 +33,83 @@ import wherehows.dao.DaoFactory;
public class Application extends Controller { public class Application extends Controller {
private static final String DB_WHEREHOWS_URL = Play.application().configuration().getString("db.wherehows.url"); private static final String WHZ_APP_ENV = System.getenv("WHZ_APP_HOME");
private static final String DB_WHEREHOWS_USERNAME = private static final String DB_WHEREHOWS_URL = Play.application().configuration().getString("db.wherehows.url");
Play.application().configuration().getString("db.wherehows.username"); private static final String DB_WHEREHOWS_USERNAME =
private static final String DB_WHEREHOWS_PASSWORD = Play.application().configuration().getString("db.wherehows.username");
Play.application().configuration().getString("db.wherehows.password"); private static final String DB_WHEREHOWS_PASSWORD =
Play.application().configuration().getString("db.wherehows.password");
public static final EntityManagerFactory entityManagerFactory; public static final EntityManagerFactory entityManagerFactory;
static { static {
Map<String, String> properties = new HashMap<>(); Map<String, String> properties = new HashMap<>();
properties.put("hibernate.connection.provider_class", HikariCPConnectionProvider.class.getName()); properties.put("hibernate.connection.provider_class", HikariCPConnectionProvider.class.getName());
properties.put("hibernate.hikari.dataSourceClassName", "com.mysql.jdbc.jdbc2.optional.MysqlDataSource"); properties.put("hibernate.hikari.dataSourceClassName", "com.mysql.jdbc.jdbc2.optional.MysqlDataSource");
properties.put("hibernate.hikari.dataSource.url", DB_WHEREHOWS_URL); properties.put("hibernate.hikari.dataSource.url", DB_WHEREHOWS_URL);
properties.put("hibernate.hikari.dataSource.user", DB_WHEREHOWS_USERNAME); properties.put("hibernate.hikari.dataSource.user", DB_WHEREHOWS_USERNAME);
properties.put("hibernate.hikari.dataSource.password", DB_WHEREHOWS_PASSWORD); properties.put("hibernate.hikari.dataSource.password", DB_WHEREHOWS_PASSWORD);
properties.put("hibernate.hikari.dataSource.cachePrepStmts", "true"); properties.put("hibernate.hikari.dataSource.cachePrepStmts", "true");
properties.put("hibernate.hikari.dataSource.prepStmtCacheSize", "250"); properties.put("hibernate.hikari.dataSource.prepStmtCacheSize", "250");
properties.put("hibernate.hikari.dataSource.prepStmtCacheSqlLimit", "2048"); properties.put("hibernate.hikari.dataSource.prepStmtCacheSqlLimit", "2048");
properties.put("hibernate.hikari.minimumIdle", "5"); properties.put("hibernate.hikari.minimumIdle", "5");
properties.put("hibernate.hikari.maximumPoolSize", "10"); properties.put("hibernate.hikari.maximumPoolSize", "10");
properties.put("hibernate.hikari.idleTimeout", "30000"); properties.put("hibernate.hikari.idleTimeout", "30000");
properties.put("hibernate.show_sql", "false"); properties.put("hibernate.show_sql", "false");
properties.put("hibernate.dialect", "MySQL5"); properties.put("hibernate.dialect", "MySQL5");
properties.put("hibernate.jdbc.batch_size", "100"); properties.put("hibernate.jdbc.batch_size", "100");
properties.put("hibernate.order_inserts", "true"); properties.put("hibernate.order_inserts", "true");
properties.put("hibernate.order_updates", "true"); properties.put("hibernate.order_updates", "true");
entityManagerFactory = Persistence.createEntityManagerFactory("default", properties); entityManagerFactory = Persistence.createEntityManagerFactory("default", properties);
}
public static final DaoFactory daoFactory = createDaoFactory();
private static DaoFactory createDaoFactory() {
try {
String className = Play.application().configuration().getString("dao.entityManagerFactory.class", DaoFactory.class.getCanonicalName());
Class factoryClass = Class.forName(className);
Constructor<? extends DaoFactory> ctor = factoryClass.getConstructor(EntityManagerFactory.class);
return ctor.newInstance(entityManagerFactory);
} catch (Exception e) {
throw new RuntimeException(e);
} }
}
public static Result index() { public static final DaoFactory daoFactory = createDaoFactory();
return ok("TEST");
}
public static Result healthcheck() { private static DaoFactory createDaoFactory() {
return ok("GOOD"); try {
} String className = Play.application().configuration().getString("dao.entityManagerFactory.class", DaoFactory.class.getCanonicalName());
Class factoryClass = Class.forName(className);
Constructor<? extends DaoFactory> ctor = factoryClass.getConstructor(EntityManagerFactory.class);
return ctor.newInstance(entityManagerFactory);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public static Result index() {
return ok("TEST");
}
public static Result healthcheck() {
return ok("GOOD");
}
public static Result printDeps() {
String libPath = WHZ_APP_ENV + "/lib";
String commitFile = WHZ_APP_ENV + "/commit";
String libraries = "";
String commit = "";
if (WHZ_APP_ENV == null) {
return ok("WHZ_APP_HOME environmental variable not defined");
}
try {
BufferedReader br = new BufferedReader(new FileReader(commitFile));
commit = br.readLine();
} catch (IOException ioe) {
Logger.error("Error while reading commit file. Error message: " +
ioe.getMessage());
}
//get all the files from a directory
File directory = new File(libPath);
for (File file : directory.listFiles()) {
if (file.isFile()) {
libraries += file.getName() + "\n";
}
}
return ok("commit: " + commit + "\n" + libraries);
}
} }

3
wherehows-backend/conf/routes
View File

@ -8,6 +8,9 @@ GET / controllers.Application.index()
# Health check endpoint # Health check endpoint
GET /admin controllers.Application.healthcheck() GET /admin controllers.Application.healthcheck()
#container libraries
GET /deps controllers.Application.printDeps()
# Map static resources from the /public folder to the /assets URL path # Map static resources from the /public folder to the /assets URL path
GET /assets/*file controllers.Assets.at(path="/public", file) GET /assets/*file controllers.Assets.at(path="/public", file)

18
wherehows-dao/build.gradle
View File

@ -1,13 +1,13 @@
apply plugin: 'java' apply plugin: 'java'
dependencies { dependencies {
compile externalDependency.commons_lang3 compile externalDependency.commons_lang3
compile externalDependency.slf4j_api compile externalDependency.slf4j_api
compile externalDependency.spring_jdbc compile externalDependency.spring_jdbc
compile externalDependency.jackson_databind compile externalDependency.jackson_databind
compile externalDependency.jackson_core compile externalDependency.jackson_core
compile externalDependency.jackson_annotations compile externalDependency.jackson_annotations
compile externalDependency.lombok compile externalDependency.lombok
compile externalDependency.hibernate_core compile externalDependency.hibernate_core
compile externalDependency.hibernate_hikaricp compile externalDependency.hibernate_hikaricp
} }

151
wherehows-frontend/app/controllers/Application.java
View File

@ -1,12 +1,12 @@
/** /**
* Copyright 2015 LinkedIn Corp. All rights reserved. * Copyright 2015 LinkedIn Corp. All rights reserved.
* * <p>
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
* * <p>
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* * <p>
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -18,8 +18,15 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import dao.FlowsDAO; import dao.FlowsDAO;
import dao.MetricsDAO; import dao.MetricsDAO;
import dao.UserDAO; import dao.UserDAO;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.lang.reflect.Constructor; import java.lang.reflect.Constructor;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import play.Logger; import play.Logger;
import play.Play; import play.Play;
@ -35,9 +42,10 @@ import wherehows.dao.DaoFactory;
import static play.data.Form.*; import static play.data.Form.*;
public class Application extends Controller public class Application extends Controller {
{
private static String TREE_NAME_SUBFIX = ".tree.name"; private static String TREE_NAME_SUBFIX = ".tree.name";
private static final String WHZ_APP_ENV = System.getenv("WHZ_APP_HOME");
private static final String APP_VERSION = Play.application().configuration().getString("app.version"); private static final String APP_VERSION = Play.application().configuration().getString("app.version");
private static final String PIWIK_SITE_ID = Play.application().configuration().getString("tracking.piwik.siteid"); private static final String PIWIK_SITE_ID = Play.application().configuration().getString("tracking.piwik.siteid");
private static final String PIWIK_URL = Play.application().configuration().getString("tracking.piwik.url"); private static final String PIWIK_URL = Play.application().configuration().getString("tracking.piwik.url");
@ -48,7 +56,7 @@ public class Application extends Controller
private static DaoFactory createDaoFactory() { private static DaoFactory createDaoFactory() {
try { try {
String className = String className =
Play.application().configuration().getString("dao.factory.class", DaoFactory.class.getCanonicalName()); Play.application().configuration().getString("dao.factory.class", DaoFactory.class.getCanonicalName());
Class factoryClass = Class.forName(className); Class factoryClass = Class.forName(className);
Constructor<? extends DaoFactory> ctor = factoryClass.getConstructor(); Constructor<? extends DaoFactory> ctor = factoryClass.getConstructor();
return ctor.newInstance(); return ctor.newInstance();
@ -59,6 +67,7 @@ public class Application extends Controller
/** /**
* Serves the build output index.html for any given path * Serves the build output index.html for any given path
*
* @param path takes a path string, which essentially is ignored * @param path takes a path string, which essentially is ignored
* routing is managed client side * routing is managed client side
* @return {Result} build output index.html resource * @return {Result} build output index.html resource
@ -70,13 +79,43 @@ public class Application extends Controller
return ok(indexHtml).as("text/html"); return ok(indexHtml).as("text/html");
} }
public static Result healthcheck() public static Result healthcheck() {
{
return ok("GOOD"); return ok("GOOD");
} }
public static Result printDeps() {
String libPath = WHZ_APP_ENV + "/lib";
String commitFile = WHZ_APP_ENV + "/commit";
String libraries = "";
String commit = "";
if (WHZ_APP_ENV == null) {
return ok("WHZ_APP_HOME environmental variable not defined");
}
try {
BufferedReader br = new BufferedReader(new FileReader(commitFile));
commit = br.readLine();
} catch (IOException ioe) {
Logger.error("Error while reading commit file. Error message: " +
ioe.getMessage());
}
//get all the files from a directory
File directory = new File(libPath);
for (File file : directory.listFiles()) {
if (file.isFile()) {
libraries += file.getName() + "\n";
}
}
return ok("commit: " + commit + "\n" + libraries);
}
/** /**
* index Action proxies to serveAsset * index Action proxies to serveAsset
*
* @param path takes a path string which is either index.html or the path segment after / * @param path takes a path string which is either index.html or the path segment after /
* @return {Result} response from serveAsset method * @return {Result} response from serveAsset method
*/ */
@ -127,22 +166,18 @@ public class Application extends Controller
} }
@Security.Authenticated(Secured.class) @Security.Authenticated(Secured.class)
public static Result lineage() public static Result lineage() {
{
String username = session("user"); String username = session("user");
if (username == null) if (username == null) {
{
username = ""; username = "";
} }
return serveAsset(""); return serveAsset("");
} }
@Security.Authenticated(Secured.class) @Security.Authenticated(Secured.class)
public static Result datasetLineage(int id) public static Result datasetLineage(int id) {
{
String username = session("user"); String username = session("user");
if (username == null) if (username == null) {
{
username = ""; username = "";
} }
@ -150,11 +185,9 @@ public class Application extends Controller
} }
@Security.Authenticated(Secured.class) @Security.Authenticated(Secured.class)
public static Result metricLineage(int id) public static Result metricLineage(int id) {
{
String username = session("user"); String username = session("user");
if (username == null) if (username == null) {
{
username = ""; username = "";
} }
@ -162,16 +195,13 @@ public class Application extends Controller
} }
@Security.Authenticated(Secured.class) @Security.Authenticated(Secured.class)
public static Result flowLineage(String application, String project, String flow) public static Result flowLineage(String application, String project, String flow) {
{
String username = session("user"); String username = session("user");
if (username == null) if (username == null) {
{
username = ""; username = "";
} }
String type = "azkaban"; String type = "azkaban";
if (StringUtils.isNotBlank(application) && (application.toLowerCase().indexOf("appworx") != -1)) if (StringUtils.isNotBlank(application) && (application.toLowerCase().indexOf("appworx") != -1)) {
{
type = "appworx"; type = "appworx";
} }
@ -180,11 +210,9 @@ public class Application extends Controller
} }
@Security.Authenticated(Secured.class) @Security.Authenticated(Secured.class)
public static Result schemaHistory() public static Result schemaHistory() {
{
String username = session("user"); String username = session("user");
if (username == null) if (username == null) {
{
username = ""; username = "";
} }
@ -192,11 +220,9 @@ public class Application extends Controller
} }
@Security.Authenticated(Secured.class) @Security.Authenticated(Secured.class)
public static Result scriptFinder() public static Result scriptFinder() {
{
String username = session("user"); String username = session("user");
if (username == null) if (username == null) {
{
username = ""; username = "";
} }
@ -204,29 +230,24 @@ public class Application extends Controller
} }
@Security.Authenticated(Secured.class) @Security.Authenticated(Secured.class)
public static Result idpc() public static Result idpc() {
{
String username = session("user"); String username = session("user");
if (username == null) if (username == null) {
{
username = ""; username = "";
} }
return serveAsset(""); return serveAsset("");
} }
@Security.Authenticated(Secured.class) @Security.Authenticated(Secured.class)
public static Result dashboard() public static Result dashboard() {
{
String username = session("user"); String username = session("user");
if (username == null) if (username == null) {
{
username = ""; username = "";
} }
return serveAsset(""); return serveAsset("");
} }
public static Result login() public static Result login() {
{
//You cann generate the Csrf token such as String csrfToken = SecurityPlugin.getInstance().getCsrfToken(); //You cann generate the Csrf token such as String csrfToken = SecurityPlugin.getInstance().getCsrfToken();
String csrfToken = ""; String csrfToken = "";
return serveAsset(""); return serveAsset("");
@ -277,8 +298,7 @@ public class Application extends Controller
return ok(response); return ok(response);
} }
public static Result signUp() public static Result signUp() {
{
DynamicForm loginForm = form().bindFromRequest(); DynamicForm loginForm = form().bindFromRequest();
String username = loginForm.get("inputName"); String username = loginForm.get("inputName");
String firstName = loginForm.get("inputFirstName"); String firstName = loginForm.get("inputFirstName");
@ -286,62 +306,47 @@ public class Application extends Controller
String email = loginForm.get("inputEmail"); String email = loginForm.get("inputEmail");
String password = loginForm.get("inputPassword"); String password = loginForm.get("inputPassword");
String errorMessage = ""; String errorMessage = "";
try try {
{
errorMessage = UserDAO.signUp(username, firstName, lastName, email, password); errorMessage = UserDAO.signUp(username, firstName, lastName, email, password);
if (StringUtils.isNotBlank(errorMessage)) if (StringUtils.isNotBlank(errorMessage)) {
{
flash("error", errorMessage); flash("error", errorMessage);
} } else {
else
{
flash("success", "Congratulations! Your account has been created. Please login."); flash("success", "Congratulations! Your account has been created. Please login.");
} }
} } catch (Exception e) {
catch (Exception e)
{
flash("error", e.getMessage()); flash("error", e.getMessage());
} }
return serveAsset(""); return serveAsset("");
} }
public static Result logout() public static Result logout() {
{
session().clear(); session().clear();
return ok(); return ok();
} }
public static Result loadTree(String key) public static Result loadTree(String key) {
{ if (StringUtils.isNotBlank(key) && key.equalsIgnoreCase("flows")) {
if (StringUtils.isNotBlank(key) && key.equalsIgnoreCase("flows"))
{
return ok(FlowsDAO.getFlowApplicationNodes()); return ok(FlowsDAO.getFlowApplicationNodes());
} } else if (StringUtils.isNotBlank(key) && key.equalsIgnoreCase("metrics")) {
else if (StringUtils.isNotBlank(key) && key.equalsIgnoreCase("metrics"))
{
return ok(MetricsDAO.getMetricDashboardNodes()); return ok(MetricsDAO.getMetricDashboardNodes());
} }
return ok(Tree.loadTreeJsonNode(key + TREE_NAME_SUBFIX)); return ok(Tree.loadTreeJsonNode(key + TREE_NAME_SUBFIX));
} }
public static Result loadFlowProjects(String app) public static Result loadFlowProjects(String app) {
{
return ok(FlowsDAO.getFlowProjectNodes(app)); return ok(FlowsDAO.getFlowProjectNodes(app));
} }
public static Result loadFlowNodes(String app, String project) public static Result loadFlowNodes(String app, String project) {
{
return ok(FlowsDAO.getFlowNodes(app, project)); return ok(FlowsDAO.getFlowNodes(app, project));
} }
public static Result loadMetricGroups(String dashboard) public static Result loadMetricGroups(String dashboard) {
{
return ok(MetricsDAO.getMetricGroupNodes(dashboard)); return ok(MetricsDAO.getMetricGroupNodes(dashboard));
} }
public static Result loadMetricNodes(String dashboard, String group) public static Result loadMetricNodes(String dashboard, String group) {
{
return ok(MetricsDAO.getMetricNodes(dashboard, group)); return ok(MetricsDAO.getMetricNodes(dashboard, group));
} }

2
wherehows-frontend/conf/routes
View File

@ -8,6 +8,8 @@ GET /
GET /admin controllers.Application.healthcheck() GET /admin controllers.Application.healthcheck()
GET /deps controllers.Application.printDeps()
GET /config controllers.Application.appConfig() GET /config controllers.Application.appConfig()
GET /login controllers.Application.login() GET /login controllers.Application.login()