yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-13 11:06:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Configurize lineage look back time

Browse Source
This commit is contained in:
SunZhaonan 2016-03-14 15:03:08 -07:00
parent c4671d2579
commit 6db4a64eb2
3 changed files with 36 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
web/app/controllers/api/v1/Lineage.java
View File

@ -17,6 +17,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import dao.DatasetsDAO; import dao.DatasetsDAO;
import dao.FlowsDAO; import dao.FlowsDAO;
import dao.LineageDAO; import dao.LineageDAO;
import play.Play;
import play.libs.Json; import play.libs.Json;
import play.mvc.Controller; import play.mvc.Controller;
import play.mvc.Result; import play.mvc.Result;
@ -25,6 +26,7 @@ import org.apache.commons.lang3.StringUtils;
public class Lineage extends Controller public class Lineage extends Controller
{ {
private static final String LINEAGE_LOOK_BACK_TIME_KEY = "lineage.look.back.time";
public static Result getDatasetLineageGraphData(int id) public static Result getDatasetLineageGraphData(int id)
{ {
@ -89,8 +91,25 @@ public class Lineage extends Controller
if (downLevel < 1) if (downLevel < 1)
downLevel = 1; downLevel = 1;
int lookBackTimeDefault = Integer.valueOf(Play.application().configuration().getString(LINEAGE_LOOK_BACK_TIME_KEY, "30"));
int lookBackTime = lookBackTimeDefault;
String lookBackTimeStr = request().getQueryString("period");
if (!StringUtils.isBlank(lookBackTimeStr))
{
try
{
lookBackTime = Integer.parseInt(lookBackTimeStr);
}
catch(NumberFormatException e)
{
Logger.error("Lineage Controller getDatasetLineageGraphData wrong period parameter. Error message: "
+ e.getMessage());
lookBackTime = lookBackTimeDefault;
}
}
result.put("status", "ok"); result.put("status", "ok");
result.set("data", Json.toJson(LineageDAO.getObjectAdjacnet(dataset.urn, upLevel, downLevel))); result.set("data", Json.toJson(LineageDAO.getObjectAdjacnet(dataset.urn, upLevel, downLevel, lookBackTime)));
return ok(result); return ok(result);
} }

22
web/app/dao/LineageDAO.java
View File

@ -94,7 +94,7 @@ public class LineageDAO extends AbstractMySQLOpenSourceDAO
"FROM_UNIXTIME(job_finished_unixtime) > CURRENT_DATE - INTERVAL 60 DAY"; "FROM_UNIXTIME(job_finished_unixtime) > CURRENT_DATE - INTERVAL 60 DAY";
public static JsonNode getObjectAdjacnet(String urn, int upLevel, int downLevel) public static JsonNode getObjectAdjacnet(String urn, int upLevel, int downLevel, int lookBackTime)
{ {
ObjectNode resultNode = Json.newObject(); ObjectNode resultNode = Json.newObject();
LineagePathInfo pathInfo = utils.Lineage.convertFromURN(urn); LineagePathInfo pathInfo = utils.Lineage.convertFromURN(urn);
@ -127,7 +127,8 @@ public class LineageDAO extends AbstractMySQLOpenSourceDAO
nodes, nodes,
edges, edges,
addedDataNodes, addedDataNodes,
addedJobNodes); addedJobNodes,
lookBackTime);
if (nodes.size() > 0) if (nodes.size() > 0)
{ {
message = "Found lineage on azkaban"; message = "Found lineage on azkaban";
@ -153,7 +154,8 @@ public class LineageDAO extends AbstractMySQLOpenSourceDAO
List<LineageNode> nodes, List<LineageNode> nodes,
List<LineageEdge> edges, List<LineageEdge> edges,
Map<String, Integer> addedDataNodes, Map<String, Integer> addedDataNodes,
Map<Long, Integer> addedJobNodes) Map<Long, Integer> addedJobNodes,
int lookBackTime)
{ {
if (upLevel < 1 && downLevel < 1) if (upLevel < 1 && downLevel < 1)
{ {
@ -164,7 +166,7 @@ public class LineageDAO extends AbstractMySQLOpenSourceDAO
rows = getJdbcTemplate().queryForList( rows = getJdbcTemplate().queryForList(
GET_JOB, GET_JOB,
pathInfo.filePath, pathInfo.filePath,
30); lookBackTime);
if (rows != null) if (rows != null)
{ {
for (Map row : rows) for (Map row : rows)
@ -293,7 +295,8 @@ public class LineageDAO extends AbstractMySQLOpenSourceDAO
nodes, nodes,
edges, edges,
addedDataNodes, addedDataNodes,
addedJobNodes); addedJobNodes,
lookBackTime);
} }
nodeIndex = nodes.size(); nodeIndex = nodes.size();
} }
@ -345,7 +348,8 @@ public class LineageDAO extends AbstractMySQLOpenSourceDAO
nodes, nodes,
edges, edges,
addedDataNodes, addedDataNodes,
addedJobNodes); addedJobNodes,
lookBackTime);
} }
nodeIndex = nodes.size(); nodeIndex = nodes.size();
} }
@ -366,7 +370,8 @@ public class LineageDAO extends AbstractMySQLOpenSourceDAO
List<LineageNode> nodes, List<LineageNode> nodes,
List<LineageEdge> edges, List<LineageEdge> edges,
Map<String, Integer> addedDataNodes, Map<String, Integer> addedDataNodes,
Map<Long, Integer> addedJobNodes) Map<Long, Integer> addedJobNodes,
int lookBackTime)
{ {
if (upLevel < 1 && downLevel < 1) if (upLevel < 1 && downLevel < 1)
{ {
@ -397,7 +402,8 @@ public class LineageDAO extends AbstractMySQLOpenSourceDAO
nodes, nodes,
edges, edges,
addedDataNodes, addedDataNodes,
addedJobNodes); addedJobNodes,
lookBackTime);
break; break;
} }
} }

2
web/conf/application.conf
View File

@ -79,3 +79,5 @@ authentication.ldap.context_factory_class = "com.sun.jndi.ldap.LdapCtxFactory"
authentication.principal.domain = "$YOUR_LDAP_DOMAIN" authentication.principal.domain = "$YOUR_LDAP_DOMAIN"
dataset.hdfs_browser.link = "https://localhost:8888/filebrowser/#" dataset.hdfs_browser.link = "https://localhost:8888/filebrowser/#"
lineage.look.back.time = 60