yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-12 10:35:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Configurize lineage look back time

Browse Source
This commit is contained in:
SunZhaonan 2016-03-14 15:03:08 -07:00
parent c4671d2579
commit 6db4a64eb2
3 changed files with 36 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
web/app/controllers/api/v1/Lineage.java
View File

@ -17,6 +17,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import dao.DatasetsDAO;
import dao.FlowsDAO;
import dao.LineageDAO;
import play.Play;
import play.libs.Json;
import play.mvc.Controller;
import play.mvc.Result;
@ -25,6 +26,7 @@ import org.apache.commons.lang3.StringUtils;
public class Lineage extends Controller
{
private static final String LINEAGE_LOOK_BACK_TIME_KEY = "lineage.look.back.time";
public static Result getDatasetLineageGraphData(int id)
{
@ -89,8 +91,25 @@ public class Lineage extends Controller
if (downLevel < 1)
downLevel = 1;
int lookBackTimeDefault = Integer.valueOf(Play.application().configuration().getString(LINEAGE_LOOK_BACK_TIME_KEY, "30"));
int lookBackTime = lookBackTimeDefault;
String lookBackTimeStr = request().getQueryString("period");
if (!StringUtils.isBlank(lookBackTimeStr))
{
try
{
lookBackTime = Integer.parseInt(lookBackTimeStr);
}
catch(NumberFormatException e)
{
Logger.error("Lineage Controller getDatasetLineageGraphData wrong period parameter. Error message: "
+ e.getMessage());
lookBackTime = lookBackTimeDefault;
}
}
result.put("status", "ok");
result.set("data", Json.toJson(LineageDAO.getObjectAdjacnet(dataset.urn, upLevel, downLevel)));
result.set("data", Json.toJson(LineageDAO.getObjectAdjacnet(dataset.urn, upLevel, downLevel, lookBackTime)));
return ok(result);
}

22
web/app/dao/LineageDAO.java
View File

@ -94,7 +94,7 @@ public class LineageDAO extends AbstractMySQLOpenSourceDAO
"FROM_UNIXTIME(job_finished_unixtime) > CURRENT_DATE - INTERVAL 60 DAY";
public static JsonNode getObjectAdjacnet(String urn, int upLevel, int downLevel)
public static JsonNode getObjectAdjacnet(String urn, int upLevel, int downLevel, int lookBackTime)
{
ObjectNode resultNode = Json.newObject();
LineagePathInfo pathInfo = utils.Lineage.convertFromURN(urn);
@ -127,7 +127,8 @@ public class LineageDAO extends AbstractMySQLOpenSourceDAO
nodes,
edges,
addedDataNodes,
addedJobNodes);
addedJobNodes,
lookBackTime);
if (nodes.size() > 0)
{
message = "Found lineage on azkaban";
@ -153,7 +154,8 @@ public class LineageDAO extends AbstractMySQLOpenSourceDAO
List<LineageNode> nodes,
List<LineageEdge> edges,
Map<String, Integer> addedDataNodes,
Map<Long, Integer> addedJobNodes)
Map<Long, Integer> addedJobNodes,
int lookBackTime)
{
if (upLevel < 1 && downLevel < 1)
{
@ -164,7 +166,7 @@ public class LineageDAO extends AbstractMySQLOpenSourceDAO
rows = getJdbcTemplate().queryForList(
GET_JOB,
pathInfo.filePath,
30);
lookBackTime);
if (rows != null)
{
for (Map row : rows)
@ -293,7 +295,8 @@ public class LineageDAO extends AbstractMySQLOpenSourceDAO
nodes,
edges,
addedDataNodes,
addedJobNodes);
addedJobNodes,
lookBackTime);
}
nodeIndex = nodes.size();
}
@ -345,7 +348,8 @@ public class LineageDAO extends AbstractMySQLOpenSourceDAO
nodes,
edges,
addedDataNodes,
addedJobNodes);
addedJobNodes,
lookBackTime);
}
nodeIndex = nodes.size();
}
@ -366,7 +370,8 @@ public class LineageDAO extends AbstractMySQLOpenSourceDAO
List<LineageNode> nodes,
List<LineageEdge> edges,
Map<String, Integer> addedDataNodes,
Map<Long, Integer> addedJobNodes)
Map<Long, Integer> addedJobNodes,
int lookBackTime)
{
if (upLevel < 1 && downLevel < 1)
{
@ -397,7 +402,8 @@ public class LineageDAO extends AbstractMySQLOpenSourceDAO
nodes,
edges,
addedDataNodes,
addedJobNodes);
addedJobNodes,
lookBackTime);
break;
}
}

2
web/conf/application.conf
View File

@ -79,3 +79,5 @@ authentication.ldap.context_factory_class = "com.sun.jndi.ldap.LdapCtxFactory"
authentication.principal.domain = "$YOUR_LDAP_DOMAIN"
dataset.hdfs_browser.link = "https://localhost:8888/filebrowser/#"
lineage.look.back.time = 60