yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-04 12:51:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

implement the dataset accessiblities section

Browse Source
This commit is contained in:
jbai 2016-08-09 11:34:46 -07:00
parent 5305124d8c
commit 45c528c9d9
10 changed files with 304 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
web/app/controllers/api/v1/Dataset.java
View File

@ -804,4 +804,20 @@ public class Dataset extends Controller
return ok(result);
}
public static Result getDatasetPartitions(Long datasetId)
{
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.put("partitions", Json.toJson(DatasetsDAO.getDatasetPartitionGains(datasetId)));
return ok(result);
}
public static Result getDatasetAccess(Long datasetId)
{
ObjectNode result = Json.newObject();
result.put("status", "ok");
result.put("access", Json.toJson(DatasetsDAO.getDatasetAccessibilty(datasetId)));
return ok(result);
}
}

112
web/app/dao/DatasetsDAO.java
View File

@ -351,6 +351,19 @@ public class DatasetsDAO extends AbstractMySQLOpenSourceDAO
"dict_dataset_instance i JOIN cfg_database c ON i.db_id = c.db_id " +
"WHERE i.dataset_id = ?";
private final static String GET_DATASET_ACCESS_PARTITION_GAIN = "SELECT DISTINCT partition_grain " +
"FROM log_dataset_instance_load_status WHERE dataset_id = ? order by 1";
private final static String GET_DATASET_ACCESS_PARTITION_INSTANCES = "SELECT DISTINCT d.db_code " +
"FROM log_dataset_instance_load_status l " +
"JOIN cfg_database d on l.db_id = d.db_id WHERE dataset_id = ? and partition_grain = ? ORDER BY 1";
private final static String GET_DATASET_ACCESS = "SELECT l.db_id, d.db_code, l.dataset_type, l.partition_expr, " +
"l.data_time_expr, l.data_time_epoch, l.record_count, l.size_in_byte, l.log_time_epoch, " +
"from_unixtime(l.log_time_epoch) as log_time_str FROM log_dataset_instance_load_status l " +
"JOIN cfg_database d on l.db_id = d.db_id WHERE dataset_id = ? and partition_grain = ? " +
"ORDER by l.data_time_expr DESC";
public static List<String> getDatasetOwnerTypes()
{
return getJdbcTemplate().queryForList(GET_DATASET_OWNER_TYPES, String.class);
@ -1985,4 +1998,103 @@ public class DatasetsDAO extends AbstractMySQLOpenSourceDAO
}
return datasetInstances;
}
public static List<String> getDatasetPartitionGains(Long id) {
return getJdbcTemplate().queryForList(GET_DATASET_ACCESS_PARTITION_GAIN, String.class, id);
}
public static List<String> getDatasetPartitionInstance(Long id, String partition) {
return getJdbcTemplate().queryForList(GET_DATASET_ACCESS_PARTITION_INSTANCES, String.class, id, partition);
}
public static List<DatasetPartition> getDatasetAccessibilty(Long id) {
ObjectNode resultNode = Json.newObject();
List<String> partitions = getDatasetPartitionGains(id);
List<DatasetPartition> datasetPartitions = new ArrayList<DatasetPartition>();
if (partitions != null && partitions.size() > 0)
{
for(String partition:partitions)
{
List<Map<String, Object>> rows = null;
Map<String, DatasetAccessibility> addedAccessibilities= new HashMap<String, DatasetAccessibility>();
rows = getJdbcTemplate().queryForList(
GET_DATASET_ACCESS,
id,
partition);
List<DatasetAccessibility> datasetAccessibilities = new ArrayList<DatasetAccessibility>();
List<String> instances = new ArrayList<String>();
instances = getDatasetPartitionInstance(id, partition);
if (rows != null)
{
for (Map row : rows) {
DatasetAccessibility datasetAccessibility = new DatasetAccessibility();
datasetAccessibility.datasetId = id;
datasetAccessibility.itemList = new ArrayList<DatasetAccessItem>();
datasetAccessibility.dbId = (Integer) row.get("db_id");
datasetAccessibility.dbName = (String) row.get("db_code");
datasetAccessibility.datasetType = (String) row.get("dataset_type");
datasetAccessibility.partitionExpr = (String) row.get("partition_expr");
datasetAccessibility.partitionGain = partition;
datasetAccessibility.dataTimeExpr = (String) row.get("data_time_expr");
datasetAccessibility.dataTimeEpoch = (Integer) row.get("data_time_epoch");
datasetAccessibility.recordCount = (Long) row.get("record_count");
if (datasetAccessibility.recordCount == null)
{
datasetAccessibility.recordCount = 0L;
}
datasetAccessibility.sizeInByte = (Long) row.get("size_in_byte");
datasetAccessibility.logTimeEpoch = (Integer) row.get("log_time_epoch");
datasetAccessibility.logTimeEpochStr = row.get("log_time_str").toString();
DatasetAccessibility exist = addedAccessibilities.get(datasetAccessibility.dataTimeExpr);
if (exist == null)
{
for(int i = 0; i < instances.size(); i++)
{
DatasetAccessItem datasetAccessItem = new DatasetAccessItem();
if(instances.get(i).equalsIgnoreCase(datasetAccessibility.dbName))
{
datasetAccessItem.recordCountStr = Long.toString(datasetAccessibility.recordCount);
datasetAccessItem.logTimeEpochStr = datasetAccessibility.logTimeEpochStr;
datasetAccessItem.isPlaceHolder = false;
}
else
{
datasetAccessItem.recordCountStr = "";
datasetAccessItem.logTimeEpochStr = "";
datasetAccessItem.isPlaceHolder = true;
}
datasetAccessibility.itemList.add(datasetAccessItem);
}
addedAccessibilities.put(datasetAccessibility.dataTimeExpr, datasetAccessibility);
datasetAccessibilities.add(datasetAccessibility);
}
else
{
for(int i = 0; i < instances.size(); i++)
{
if(instances.get(i).equalsIgnoreCase(datasetAccessibility.dbName))
{
DatasetAccessItem datasetAccessItem = new DatasetAccessItem();
datasetAccessItem.logTimeEpochStr = datasetAccessibility.logTimeEpochStr;
datasetAccessItem.recordCountStr = Long.toString(datasetAccessibility.recordCount);
datasetAccessItem.isPlaceHolder = false;
exist.itemList.set(i, datasetAccessItem);
}
}
}
}
}
DatasetPartition datasetPartition = new DatasetPartition();
datasetPartition.datasetId = id;
datasetPartition.accessibilityList = datasetAccessibilities;
datasetPartition.instanceList = instances;
datasetPartition.partition = partition;
datasetPartitions.add(datasetPartition);
}
}
return datasetPartitions;
}
}

24
web/app/models/DatasetAccessItem.java Normal file
View File

@ -0,0 +1,24 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package models;
import java.util.List;
public class DatasetAccessItem {
public String recordCountStr;
public String sizeInByteStr;
public String logTimeEpochStr;
public Boolean isPlaceHolder;
}

33
web/app/models/DatasetAccessibility.java Normal file
View File

@ -0,0 +1,33 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package models;
import java.util.List;
public class DatasetAccessibility {
public Long datasetId;
public Integer dbId;
public String dbName;
public String datasetType;
public String partitionGain;
public String partitionExpr;
public String dataTimeExpr;
public Integer dataTimeEpoch;
public Long recordCount;
public Long sizeInByte;
public Integer logTimeEpoch;
public String logTimeEpochStr;
public List<DatasetAccessItem> itemList;
}

24
web/app/models/DatasetPartition.java Normal file
View File

@ -0,0 +1,24 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package models;
import java.util.List;
public class DatasetPartition {
public Long datasetId;
public List<DatasetAccessibility> accessibilityList;
public List<String> instanceList;
public String partition;
}

65
web/app/views/index.scala.html
View File

@ -537,6 +537,43 @@
{{/if}}
</script>
<script type="text/x-handlebars" id="components/dataset-access">
{{#if hasAccess}}
{{#each accessibilities as |accessibility|}}
<h4>Partition By: {{accessibility.partition}}</h4>
<table id="references-table" class="columntreegrid tree table table-bordered dataset-detail-table">
<thead>
<tr class="results-header">
<th>Time</th>
{{#each accessibility.instanceList as |instance|}}
<th>{{instance}}</th>
{{/each}}
</tr>
</thead>
<tbody>
{{#each accessibility.accessibilityList as |access|}}
<tr>
<td>{{access.dataTimeExpr}}</td>
{{#each access.itemList as |item|}}
<td>
{{#unless item.isPlaceHolder}}
<div>
<p>Log time: {{item.logTimeEpochStr}}</p>
<p>Record count: {{item.recordCountStr}}</p>
</div>
{{/unless}}
</td>
{{/each}}
</tr>
{{/each}}
</tbody>
</table>
{{/each}}
{{else}}
<p>Accessibility is not available</p>
{{/if}}
</script>
<script type="text/x-handlebars" id="components/metric-detail">
<div id="metric" class="container-fluid">
<div class="row-fluid">
@ -932,6 +969,13 @@
Relations
</a>
</li>
<li id="access">
<a data-toggle="tab"
title="Accessibilities"
href="#accesstab">
Accessibilities
</a>
</li>
</ul>
<div class="tab-content">
{{#unless isPinot}}
@ -966,6 +1010,10 @@
{{#dataset-relations hasDepends=hasDepends depends=depends hasReferences=hasReferences references=references}}
{{/dataset-relations}}
</div>
<div id="accesstab" class="tab-pane">
{{#dataset-access hasAccess=hasAccess accessibilities=accessibilities}}
{{/dataset-access}}
</div>
</div>
{{else}}
<div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
@ -1094,6 +1142,23 @@
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab" id="accessHeading">
<h4 class="panel-title">
<a class="collapsed" data-toggle="collapse" data-parent="#accordion"
href="#accessview" aria-expanded="false" aria-controls="accessData">
Accessiblities
</a>
</h4>
</div>
<div id="accessview" class="panel-collapse collapse" role="tabpanel" aria-labelledby="accessHeading">
<div class="panel-body">
{{#dataset-access hasAccess=hasAccess accessibilities=accessibilities}}
{{/dataset-access}}
</div>
</div>
</div>
</div>
{{/if}}
</script>

4
web/conf/routes
View File

@ -71,6 +71,10 @@ GET /api/v1/datasets/:id/depends controllers.api.v1.Dataset.getDepend
GET /api/v1/datasets/:id/references controllers.api.v1.Dataset.getReferenceViews(id:Long)
GET /api/v1/datasets/:id/partitions controllers.api.v1.Dataset.getDatasetPartitions(id:Long)
GET /api/v1/datasets/:id/access controllers.api.v1.Dataset.getDatasetAccess(id:Long)
GET /api/v1/datasets/:id/properties controllers.api.v1.Dataset.getDatasetPropertiesByID(id:Int)
GET /api/v1/datasets/:id/sample controllers.api.v1.Dataset.getDatasetSampleDataByID(id:Int)

3
web/public/javascripts/components/components.js
View File

@ -130,6 +130,9 @@ App.DatasetSchemaComponent = Ember.Component.extend({
App.DatasetSampleComponent = Ember.Component.extend({
});
App.DatasetAccessComponent = Ember.Component.extend({
});
App.DatasetImpactComponent = Ember.Component.extend({
});

17
web/public/javascripts/routers/datasets.js
View File

@ -516,6 +516,23 @@ App.DatasetRoute = Ember.Route.extend({
}
});
var datasetPartitionsUrl = 'api/v1/datasets/' + id + "/access";
var datasetAccessibilities = [];
$.get(datasetPartitionsUrl, function(data) {
if (data && data.status == "ok")
{
if (data.access && (data.access.length > 0))
{
controller.set("hasAccess", true);
controller.set("accessibilities", data.access);
}
else
{
controller.set("hasAccess", false);
}
}
});
var datasetReferencesUrl = 'api/v1/datasets/' + id + "/references";
$.get(datasetReferencesUrl, function(data) {
if (data && data.status == "ok")

6
web/test/ControllersTest.java
View File

@ -142,6 +142,12 @@ public class ControllersTest {
JsonNode schemaTextNode = Json.parse(contentAsString(result));
assertThat(schemaTextNode.isContainerNode());
assertThat(schemaTextNode.get("status").asText()).isEqualTo("ok");
result = controllers.api.v1.Dataset.getDatasetAccess(datasetId)
assertThat(status(result)).isEqualTo(OK);
JsonNode accessNode = Json.parse(contentAsString(result));
assertThat(accessNode.isContainerNode());
assertThat(accessNode.get("status").asText()).isEqualTo("ok");
}
/*