yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-17 11:58:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add dataset deprecation frontend API (#812)

Browse Source
This commit is contained in:
Yi (Alan) Wang 2017-10-20 11:16:09 -07:00 committed by GitHub
parent 4c3aaf5305
commit cfb29deb68
15 changed files with 424 additions and 221 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
wherehows-dao/src/main/java/wherehows/dao/table/BaseDao.java
View File

@ -239,7 +239,7 @@ public class BaseDao {
}
/**
* Execute an update or delete query String.
* Execute an update or delete HQL query String, not native query string.
* @param queryStr String
* @param params Parameters
*/
@ -248,6 +248,7 @@ public class BaseDao {
EntityManager entityManager = null;
try {
entityManager = entityManagerFactory.createEntityManager();
entityManager.getTransaction().begin();
Query query = entityManager.createQuery(queryStr);
for (Map.Entry<String, Object> param : params.entrySet()) {
@ -255,6 +256,7 @@ public class BaseDao {
}
query.executeUpdate();
entityManager.getTransaction().commit();
} finally {
if (entityManager != null) {
entityManager.close();

9
wherehows-dao/src/main/java/wherehows/dao/table/DatasetsDao.java
View File

@ -88,6 +88,15 @@ public class DatasetsDao {
return -1;
}
/**
* Update dataset owners, set removed owners as deleted, and update existing owner information.
* @param jdbcTemplate JdbcTemplate
* @param user String
* @param datasetId int
* @param datasetUrn String
* @param owners List<DatasetOwner>
* @throws Exception
*/
public void updateDatasetOwners(JdbcTemplate jdbcTemplate, String user, int datasetId, String datasetUrn,
List<DatasetOwner> owners) throws Exception {
// first mark existing owners as deleted, new owners will be updated later

21
wherehows-dao/src/main/java/wherehows/dao/table/DictDatasetDao.java
View File

@ -42,6 +42,9 @@ public class DictDatasetDao extends BaseDao {
super(factory);
}
private static final String SET_DATASET_DEPRECATION =
"UPDATE DictDataset SET isDeprecated = :deprecated WHERE id = :datasetId";
public DictDataset findByUrn(@Nonnull String urn) {
return findBy(DictDataset.class, "urn", urn);
}
@ -167,4 +170,22 @@ public class DictDatasetDao extends BaseDao {
ds.setSourceModifiedTime(sourceTime);
}
}
/**
* Set deprecation status of a dataset.
* @param datasetId int
* @param datasetUrn String
* @param isDeprecated boolean
* @param deprecationNote String
* @param user String
* @throws Exception
*/
public void setDatasetDeprecation(int datasetId, @Nonnull String datasetUrn, boolean isDeprecated,
@Nullable String deprecationNote, @Nonnull String user) throws Exception {
Map<String, Object> params = new HashMap<>();
params.put("datasetId", datasetId);
params.put("deprecated", isDeprecated);
executeUpdate(SET_DATASET_DEPRECATION, params);
}
}

2
wherehows-dao/src/main/java/wherehows/dao/table/FieldDetailDao.java
View File

@ -31,7 +31,7 @@ import static wherehows.util.UrnUtil.*;
public class FieldDetailDao extends BaseDao {
private static final String DELETE_BY_DATASET_ID = "DELETE FROM dict_field_detail WHERE dataset_id = :datasetId";
private static final String DELETE_BY_DATASET_ID = "DELETE FROM DictFieldDetail WHERE datasetId = :datasetId";
public FieldDetailDao(@Nonnull EntityManagerFactory factory) {
super(factory);

13
wherehows-dao/src/main/java/wherehows/dao/view/BaseViewDao.java
View File

@ -15,6 +15,7 @@ package wherehows.dao.view;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import javax.persistence.EntityManager;
import javax.persistence.EntityManagerFactory;
import javax.persistence.Query;
@ -29,7 +30,7 @@ public class BaseViewDao {
private final EntityManagerFactory _emFactory;
public BaseViewDao(EntityManagerFactory factory) {
public BaseViewDao(@Nonnull EntityManagerFactory factory) {
this._emFactory = factory;
}
@ -43,7 +44,8 @@ public class BaseViewDao {
*/
@SneakyThrows
@SuppressWarnings("unchecked")
protected <T> List<T> getEntityListBy(String sqlQuery, Class<T> classType, Map<String, Object> params) {
protected <T> List<T> getEntityListBy(@Nonnull String sqlQuery, @Nonnull Class<T> classType,
@Nonnull Map<String, Object> params) {
EntityManager entityManager = null;
try {
entityManager = _emFactory.createEntityManager();
@ -71,7 +73,8 @@ public class BaseViewDao {
*/
@SneakyThrows
@SuppressWarnings("unchecked")
protected <T> T getEntityBy(String sqlQuery, Class<T> classType, Map<String, Object> params) {
protected <T> T getEntityBy(@Nonnull String sqlQuery, @Nonnull Class<T> classType,
@Nonnull Map<String, Object> params) {
EntityManager entityManager = null;
try {
entityManager = _emFactory.createEntityManager();
@ -97,7 +100,7 @@ public class BaseViewDao {
*/
@SneakyThrows
@SuppressWarnings("unchecked")
protected List<Object[]> getObjectArrayListBy(String sqlQuery, Map<String, Object> params) {
protected List<Object[]> getObjectArrayListBy(@Nonnull String sqlQuery, @Nonnull Map<String, Object> params) {
EntityManager entityManager = null;
try {
entityManager = _emFactory.createEntityManager();
@ -123,7 +126,7 @@ public class BaseViewDao {
*/
@SneakyThrows
@SuppressWarnings("unchecked")
protected List<Object> getObjectListBy(String sqlQuery, Map<String, Object> params) {
protected List<Object> getObjectListBy(@Nonnull String sqlQuery, @Nonnull Map<String, Object> params) {
EntityManager entityManager = null;
try {
entityManager = _emFactory.createEntityManager();

50
wherehows-dao/src/main/java/wherehows/dao/view/DatasetViewDao.java
View File

@ -16,20 +16,26 @@ package wherehows.dao.view;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import javax.persistence.EntityManagerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.extern.slf4j.Slf4j;
import wherehows.dao.table.DictDatasetDao;
import wherehows.models.table.DictDataset;
import wherehows.models.view.DatasetColumn;
import wherehows.models.view.DatasetView;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static wherehows.util.UrnUtil.*;
@Slf4j
public class DatasetViewDao extends BaseViewDao {
private static final Logger log = LoggerFactory.getLogger(DatasetViewDao.class);
private final DictDatasetDao _dictDatasetDao;
public DatasetViewDao(EntityManagerFactory factory) {
public DatasetViewDao(@Nonnull EntityManagerFactory factory) {
super(factory);
_dictDatasetDao = new DictDatasetDao(factory);
}
private static final String GET_DATASET_COLUMNS_BY_DATASET_ID =
@ -52,13 +58,45 @@ public class DatasetViewDao extends BaseViewDao {
+ "(ddfc.field_id = dfd.field_id AND ddfc.is_default = true) LEFT JOIN comments c ON "
+ "c.id = ddfc.comment_id WHERE dfd.dataset_id = :datasetId AND dfd.field_id = :columnId ORDER BY dfd.sort_id";
/**
* Get dataset view from dict dataset.
* @param datasetId int
* @param datasetUrn String
* @return DatasetView
*/
public DatasetView getDatasetView(int datasetId, @Nonnull String datasetUrn) {
return fillDatasetViewFromDictDataset(_dictDatasetDao.findById(datasetId));
}
/**
* Convert DictDataset to DatasetView
* @param ds DictDataset
* @return DatasetView
*/
public DatasetView fillDatasetViewFromDictDataset(DictDataset ds) {
String[] urnParts = splitWhUrn(ds.getUrn());
DatasetView view = new DatasetView();
view.setPlatform(urnParts[0]);
view.setNativeName(urnParts[1]);
view.setUri(ds.getUrn());
view.setNativeType(ds.getStorageType());
view.setProperties(ds.getProperties());
view.setRemoved(ds.getIsActive() != null && !ds.getIsActive());
view.setDeprecated(ds.getIsDeprecated());
view.setCreatedTime(1000L * ds.getCreatedTime());
view.setModifiedTime(1000L * ds.getModifiedTime());
return view;
}
/**
* Get dataset columns by dataset id
* @param datasetId int
* @param datasetUrn String
* @return List of DatasetColumn
*/
public List<DatasetColumn> getDatasetColumnsByID(int datasetId, String datasetUrn) {
public List<DatasetColumn> getDatasetColumnsByID(int datasetId, @Nonnull String datasetUrn) {
Map<String, Object> params = new HashMap<>();
params.put("datasetId", datasetId);
@ -83,7 +121,7 @@ public class DatasetViewDao extends BaseViewDao {
return columns;
}
private void fillInColumnEntity(List<DatasetColumn> columns) {
private void fillInColumnEntity(@Nonnull List<DatasetColumn> columns) {
for (DatasetColumn column : columns) {
column.setFullFieldPath(isNotBlank(column.getParentPath()) ? column.getParentPath() + "." + column.getFieldName()
: column.getFieldName());

1
wherehows-dao/src/main/java/wherehows/models/table/Dataset.java
View File

@ -36,5 +36,4 @@ public class Dataset {
public boolean hasSchemaHistory;
public JsonNode properties;
public List<User> owners;
}

50
wherehows-dao/src/main/java/wherehows/models/view/DatasetView.java Normal file
View File

@ -0,0 +1,50 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package wherehows.models.view;
import java.util.List;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
public class DatasetView {
private String platform;
private String nativeName;
private String fabric;
private String uri;
private String description;
private String nativeType;
private String properties;
private List<String> tags;
private Boolean removed;
private Boolean deprecated;
private String deprecationNote;
private Long createdTime;
private Long modifiedTime;
}

59
wherehows-dao/src/test/java/wherehows/dao/DatasetViewDaoTest.java Normal file
View File

@ -0,0 +1,59 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package wherehows.dao;
import org.testng.annotations.Test;
import wherehows.dao.view.DatasetViewDao;
import wherehows.models.table.DictDataset;
import wherehows.models.view.DatasetView;
import static org.testng.Assert.*;
public class DatasetViewDaoTest {
@Test
public void testFillDatasetViewFromDictDataset() {
DatasetViewDao datasetViewDao = new DatasetViewDao(null);
String urn = "hdfs:///foo/bar";
int createTime = 1;
int modifiedTime = 2;
String type = "Avro";
String properties = "{properties}";
boolean active = false;
boolean deprecated = true;
DictDataset ds = new DictDataset();
ds.setUrn(urn);
ds.setStorageType(type);
ds.setCreatedTime(createTime);
ds.setModifiedTime(modifiedTime);
ds.setProperties(properties);
ds.setIsActive(active);
ds.setIsDeprecated(deprecated);
DatasetView view = datasetViewDao.fillDatasetViewFromDictDataset(ds);
assertEquals(view.getPlatform(), "hdfs");
assertEquals(view.getNativeName(), "/foo/bar");
assertEquals(view.getNativeType(), type);
assertEquals(view.getUri(), urn);
assertTrue(view.getRemoved());
assertTrue(view.getDeprecated());
assertEquals(view.getProperties(), properties);
assertEquals(view.getCreatedTime(), new Long(createTime * 1000));
assertEquals(view.getModifiedTime(), new Long(modifiedTime * 1000));
}
}

65
wherehows-frontend/app/controllers/api/v1/Dataset.java
View File

@ -32,9 +32,9 @@ import play.libs.F.Promise;
import play.libs.Json;
import play.mvc.Controller;
import play.mvc.Result;
import wherehows.dao.table.DatasetClassificationDao;
import wherehows.dao.table.DatasetComplianceDao;
import wherehows.dao.table.DatasetsDao;
import wherehows.dao.table.DictDatasetDao;
import wherehows.dao.view.DatasetViewDao;
import wherehows.dao.view.OwnerViewDao;
import wherehows.models.view.DatasetCompliance;
@ -42,6 +42,7 @@ import wherehows.models.table.DatasetDependency;
import wherehows.models.table.ImpactDataset;
import wherehows.models.view.DatasetColumn;
import wherehows.models.view.DatasetOwner;
import wherehows.models.view.DatasetView;
import wherehows.models.view.DsComplianceSuggestion;
@ -53,8 +54,7 @@ public class Dataset extends Controller {
private static final DatasetsDao DATASETS_DAO = Application.DAO_FACTORY.getDatasetsDao();
private static final DatasetClassificationDao CLASSIFICATION_DAO =
Application.DAO_FACTORY.getDatasetClassificationDao();
private static final DictDatasetDao DICT_DATASET_DAO = Application.DAO_FACTORY.getDictDatasetDao();
private static final DatasetViewDao DATASET_VIEW_DAO = Application.DAO_FACTORY.getDatasetViewDao();
@ -129,6 +129,63 @@ public class Dataset extends Controller {
return ok(result);
}
public static Promise<Result> getDatasetViewById(int datasetId) {
DatasetView view = null;
try {
String urn = getDatasetUrnByIdOrCache(datasetId);
view = DATASET_VIEW_DAO.getDatasetView(datasetId, urn);
} catch (Exception e) {
Logger.warn("Failed to get dataset view", e);
JsonNode result = Json.newObject()
.put("status", "failed")
.put("error", "true")
.put("msg", "Fetch data Error: " + e.getMessage());
return Promise.promise(() -> ok(result));
}
if (view == null) {
JsonNode result = Json.newObject().put("status", "failed").put("msg", "Not found");
return Promise.promise(() -> ok(result));
}
JsonNode result = Json.newObject().put("status", "ok").set("dataset", Json.toJson(view));
return Promise.promise(() -> ok(result));
}
public static Promise<Result> updateDatasetDeprecation(int datasetId) {
String username = session("user");
if (StringUtils.isBlank(username)) {
JsonNode result = Json.newObject().put("status", "failed").put("error", "true").put("msg", "Unauthorized User.");
return Promise.promise(() -> ok(result));
}
try {
JsonNode record = request().body().asJson();
boolean deprecated = record.get("deprecated").asBoolean();
String deprecationNote = record.get("deprecationNote").asText();
String urn = getDatasetUrnByIdOrCache(datasetId);
DICT_DATASET_DAO.setDatasetDeprecation(datasetId, urn, deprecated, deprecationNote, username);
} catch (Exception e) {
JsonNode result = Json.newObject()
.put("status", "failed")
.put("error", "true")
.put("msg", "Update Compliance Info Error: " + e.getMessage());
Logger.warn("Update dataset deprecation info fail", e);
return Promise.promise(() -> ok(result));
}
return Promise.promise(() -> ok(Json.newObject().put("status", "ok")));
}
private static Integer getDatasetIdByUrnOrCache(String urn) {
String cacheKey = URN_CACHE_KEY + urn;
@ -165,7 +222,7 @@ public class Dataset extends Controller {
urn = DATASETS_DAO.validateUrn(JDBC_TEMPLATE, datasetId);
if (urn != null) {
Cache.set(cacheKey, urn);
Cache.set(cacheKey, urn, DATASET_ID_CACHE_PERIOD);
}
return urn;
}

146
wherehows-frontend/app/dao/DatasetRowMapper.java
View File

@ -13,98 +13,78 @@
*/
package dao;
import wherehows.models.table.Dataset;
import wherehows.models.table.User;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Time;
import org.apache.commons.lang3.StringUtils;
import org.springframework.jdbc.core.RowMapper;
import play.Play;
import java.sql.Time;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
public class DatasetRowMapper implements RowMapper<Dataset>
{
private static final String HDFS_BROWSER_URL =
Play.application().configuration().getString(DatasetsDAO.HDFS_BROWSER_URL_KEY);
public static final String DATASET_ID_COLUMN = "id";
public static final String DATASET_NAME_COLUMN = "name";
public static final String DATASET_URN_COLUMN = "urn";
public static final String DATASET_SOURCE_COLUMN = "source";
public static final String DATASET_CREATED_TIME_COLUMN = "created";
private static final String DATASET_MODIFIED_TIME_COLUMN = "modified";
private static final String DATASET_SOURCE_MODIFIED_TIME_COLUMN = "source_modified_time";
private static final String DATASET_PROPERTIES_COLUMN = "properties";
private static final String DATASET_OWNER_ID_COLUMN = "owner_id";
private static final String DATASET_OWNER_NAME_COLUMN = "owner_name";
private static final String DATASET_OWNER_EMAIL_COLUMN = "owner_email";
private static final String SCHEMA_HISTORY_ID_COLUMN = "schema_history_id";
public static final String DATASET_SCHEMA_COLUMN = "schema";
public static final String HDFS_PREFIX = "hdfs";
public static final int HDFS_URN_PREFIX_LEN = 7; //for hdfs prefix is hdfs:///, but we need the last slash
import wherehows.models.table.Dataset;
@Override
public Dataset mapRow(ResultSet rs, int rowNum) throws SQLException
{
int id = rs.getInt(DATASET_ID_COLUMN);
String name = rs.getString(DATASET_NAME_COLUMN);
String urn = rs.getString(DATASET_URN_COLUMN);
String source = rs.getString(DATASET_SOURCE_COLUMN);
String strOwner = rs.getString(DATASET_OWNER_ID_COLUMN);
String strOwnerName = rs.getString(DATASET_OWNER_NAME_COLUMN);
String strOwnerEmail = rs.getString(DATASET_OWNER_EMAIL_COLUMN);
String schema = rs.getString(DATASET_SCHEMA_COLUMN);
Time created = rs.getTime(DATASET_CREATED_TIME_COLUMN);
Time modified = rs.getTime(DATASET_MODIFIED_TIME_COLUMN);
Integer schemaHistoryId = rs.getInt(SCHEMA_HISTORY_ID_COLUMN);
Long sourceModifiedTime = rs.getLong(DATASET_SOURCE_MODIFIED_TIME_COLUMN);
public class DatasetRowMapper implements RowMapper<Dataset> {
Dataset dataset = new Dataset();
dataset.id = id;
dataset.name = name;
dataset.urn = urn;
dataset.schema = schema;
String[] owners = StringUtils.isNotBlank(strOwner) ? strOwner.split(",") : null;
String[] ownerNames = StringUtils.isNotBlank(strOwnerName) ? strOwnerName.split(",") : null;
String[] ownerEmail = StringUtils.isNotBlank(strOwnerEmail) ? strOwnerEmail.split(",") : null;
public static final String HDFS_BROWSER_URL =
Play.application().configuration().getString(DatasetsDAO.HDFS_BROWSER_URL_KEY);
dataset.owners = new ArrayList<>();
if (owners != null && ownerNames != null && ownerEmail != null && owners.length == ownerNames.length
&& owners.length == ownerEmail.length)
{
for (int i = 0; i < owners.length; i++)
{
User user = new User();
user.setUserName(owners[i]);
user.setName(ownerNames[i]);
user.setEmail(ownerEmail[i]);
dataset.owners.add(user);
}
}
public static final String DATASET_ID_COLUMN = "id";
public static final String DATASET_NAME_COLUMN = "name";
public static final String DATASET_URN_COLUMN = "urn";
public static final String DATASET_SCHEMA_COLUMN = "schema";
public static final String DATASET_SOURCE_COLUMN = "source";
public static final String DATASET_PROPERTIES_COLUMN = "properties";
public static final String DATASET_CREATED_TIME_COLUMN = "created";
public static final String DATASET_MODIFIED_TIME_COLUMN = "modified";
public static final String DATASET_SOURCE_MODIFIED_TIME_COLUMN = "source_modified_time";
public static final String SCHEMA_HISTORY_ID_COLUMN = "schema_history_id";
public static final String DATASET_OWNER_ID_COLUMN = "owner_id";
public static final String DATASET_OWNER_NAME_COLUMN = "owner_name";
public static final String DATASET_OWNER_EMAIL_COLUMN = "owner_email";
public static final String FAVORITE_DATASET_ID_COLUMN = "dataset_id";
public static final String DATASET_WATCH_ID_COLUMN = "watch_id";
if (StringUtils.isNotBlank(dataset.urn) && dataset.urn.substring(0, 4).equalsIgnoreCase(HDFS_PREFIX))
{
dataset.hdfsBrowserLink = HDFS_BROWSER_URL + dataset.urn.substring(HDFS_URN_PREFIX_LEN);
}
public static final String HDFS_PREFIX = "hdfs";
public static final int HDFS_URN_PREFIX_LEN = 7; //for hdfs prefix is hdfs:///, but we need the last slash
dataset.source = source;
if (modified != null && sourceModifiedTime != null && sourceModifiedTime > 0)
{
dataset.modified = new java.util.Date(modified.getTime());
dataset.formatedModified = dataset.modified.toString();
}
if (created != null)
{
dataset.created = new java.util.Date(created.getTime());
} else if (modified != null) {
dataset.created = new java.util.Date(modified.getTime());
}
@Override
public Dataset mapRow(ResultSet rs, int rowNum) throws SQLException {
Dataset dataset = new Dataset();
dataset.id = rs.getInt(DATASET_ID_COLUMN);
dataset.name = rs.getString(DATASET_NAME_COLUMN);
dataset.urn = rs.getString(DATASET_URN_COLUMN);
dataset.schema = rs.getString(DATASET_SCHEMA_COLUMN);
dataset.source = rs.getString(DATASET_SOURCE_COLUMN);
dataset.hasSchemaHistory = schemaHistoryId != null && schemaHistoryId > 0;
return dataset;
if (StringUtils.isNotBlank(dataset.urn) && dataset.urn.substring(0, 4).equalsIgnoreCase(HDFS_PREFIX)) {
dataset.hdfsBrowserLink = HDFS_BROWSER_URL + dataset.urn.substring(HDFS_URN_PREFIX_LEN);
}
String strOwner = rs.getString(DATASET_OWNER_ID_COLUMN);
String strOwnerName = rs.getString(DATASET_OWNER_NAME_COLUMN);
String strOwnerEmail = rs.getString(DATASET_OWNER_EMAIL_COLUMN);
String[] owners = StringUtils.isNotBlank(strOwner) ? strOwner.split(",") : null;
String[] ownerNames = StringUtils.isNotBlank(strOwnerName) ? strOwnerName.split(",") : null;
String[] ownerEmail = StringUtils.isNotBlank(strOwnerEmail) ? strOwnerEmail.split(",") : null;
dataset.owners = utils.Dataset.fillDatasetOwnerList(owners, ownerNames, ownerEmail);
Time created = rs.getTime(DATASET_CREATED_TIME_COLUMN);
Time modified = rs.getTime(DATASET_MODIFIED_TIME_COLUMN);
Long sourceModifiedTime = rs.getLong(DATASET_SOURCE_MODIFIED_TIME_COLUMN);
Integer schemaHistoryId = rs.getInt(SCHEMA_HISTORY_ID_COLUMN);
if (modified != null && sourceModifiedTime != null && sourceModifiedTime > 0) {
dataset.modified = new java.util.Date(modified.getTime());
dataset.formatedModified = dataset.modified.toString();
}
if (created != null) {
dataset.created = new java.util.Date(created.getTime());
} else if (modified != null) {
dataset.created = new java.util.Date(modified.getTime());
}
dataset.hasSchemaHistory = schemaHistoryId != null && schemaHistoryId > 0;
return dataset;
}
}

160
wherehows-frontend/app/dao/DatasetWithUserRowMapper.java
View File

@ -13,113 +13,71 @@
*/
package dao;
import wherehows.models.table.Dataset;
import wherehows.models.table.User;
import org.apache.commons.lang3.StringUtils;
import org.springframework.jdbc.core.RowMapper;
import play.Play;
import java.sql.Time;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.sql.Time;
import org.apache.commons.lang3.StringUtils;
import org.springframework.jdbc.core.RowMapper;
import wherehows.models.table.Dataset;
public class DatasetWithUserRowMapper implements RowMapper<Dataset>
{
private static final String HDFS_BROWSER_URL =
Play.application().configuration().getString(DatasetsDAO.HDFS_BROWSER_URL_KEY);
public static final String DATASET_ID_COLUMN = "id";
public static final String DATASET_NAME_COLUMN = "name";
public static final String DATASET_URN_COLUMN = "urn";
public static final String DATASET_SCHEMA_COLUMN = "schema";
public static final String DATASET_SOURCE_COLUMN = "source";
public static final String DATASET_CREATED_TIME_COLUMN = "created";
public static final String DATASET_MODIFIED_TIME_COLUMN = "modified";
public static final String DATASET_SOURCE_MODIFIED_TIME_COLUMN = "source_modified_time";
public static final String FAVORITE_DATASET_ID_COLUMN = "dataset_id";
public static final String DATASET_WATCH_ID_COLUMN = "watch_id";
public static final String DATASET_PROPERTIES_COLUMN = "properties";
public static final String SCHEMA_HISTORY_ID_COLUMN = "schema_history_id";
public static final String DATASET_OWNER_ID_COLUMN = "owner_id";
public static final String DATASET_OWNER_NAME_COLUMN = "owner_name";
public static final String DATASET_OWNER_EMAIL_COLUMN = "owner_email";
public static final String HDFS_PREFIX = "hdfs";
public class DatasetWithUserRowMapper implements RowMapper<Dataset> {
@Override
public Dataset mapRow(ResultSet rs, int rowNum) throws SQLException
{
int id = rs.getInt(DATASET_ID_COLUMN);
String name = rs.getString(DATASET_NAME_COLUMN);
String urn = rs.getString(DATASET_URN_COLUMN);
String source = rs.getString(DATASET_SOURCE_COLUMN);
String schema = rs.getString(DATASET_SCHEMA_COLUMN);
Time created = rs.getTime(DATASET_CREATED_TIME_COLUMN);
Time modified = rs.getTime(DATASET_MODIFIED_TIME_COLUMN);
Integer favoriteId = rs.getInt(FAVORITE_DATASET_ID_COLUMN);
Integer schemaHistoryId = rs.getInt(SCHEMA_HISTORY_ID_COLUMN);
Long watchId = rs.getLong(DATASET_WATCH_ID_COLUMN);
Long sourceModifiedTime = rs.getLong(DATASET_SOURCE_MODIFIED_TIME_COLUMN);
String strOwner = rs.getString(DATASET_OWNER_ID_COLUMN);
String strOwnerName = rs.getString(DATASET_OWNER_NAME_COLUMN);
String strOwnerEmail = rs.getString(DATASET_OWNER_EMAIL_COLUMN);
@Override
public Dataset mapRow(ResultSet rs, int rowNum) throws SQLException {
Dataset dataset = new Dataset();
dataset.id = id;
dataset.name = name;
dataset.urn = urn;
dataset.schema = schema;
String[] owners = StringUtils.isNotBlank(strOwner) ? strOwner.split(",") : null;
String[] ownerNames = StringUtils.isNotBlank(strOwnerName) ? strOwnerName.split(",") : null;
String[] ownerEmail = StringUtils.isNotBlank(strOwnerEmail) ? strOwnerEmail.split(",") : null;
Dataset dataset = new Dataset();
dataset.id = rs.getInt(DatasetRowMapper.DATASET_ID_COLUMN);
dataset.name = rs.getString(DatasetRowMapper.DATASET_NAME_COLUMN);
dataset.urn = rs.getString(DatasetRowMapper.DATASET_URN_COLUMN);
dataset.schema = rs.getString(DatasetRowMapper.DATASET_SCHEMA_COLUMN);
dataset.source = rs.getString(DatasetRowMapper.DATASET_SOURCE_COLUMN);
dataset.owners = new ArrayList<>();
if (owners != null && ownerNames != null && ownerEmail != null && owners.length == ownerNames.length
&& owners.length == ownerEmail.length)
{
for (int i = 0; i < owners.length; i++)
{
User user = new User();
user.setUserName(owners[i]);
user.setName(ownerNames[i]);
user.setEmail(ownerEmail[i]);
dataset.owners.add(user);
}
}
if (StringUtils.isNotBlank(dataset.urn) && dataset.urn.substring(0, 4).equalsIgnoreCase(HDFS_PREFIX))
{
dataset.hdfsBrowserLink = HDFS_BROWSER_URL + dataset.urn.substring(DatasetRowMapper.HDFS_URN_PREFIX_LEN);
}
dataset.source = source;
if (modified != null && sourceModifiedTime != null && sourceModifiedTime > 0)
{
dataset.modified = new java.util.Date(modified.getTime());
dataset.formatedModified = dataset.modified.toString();
}
if (created != null)
{
dataset.created = new java.util.Date(created.getTime());
} else if (modified != null) {
dataset.created = new java.util.Date(modified.getTime());
}
dataset.isFavorite = favoriteId != null && favoriteId > 0;
if (watchId != null && watchId > 0)
{
dataset.watchId = watchId;
dataset.isWatched = true;
}
else
{
dataset.watchId = 0L;
dataset.isWatched = false;
}
dataset.hasSchemaHistory = schemaHistoryId != null && schemaHistoryId > 0;
return dataset;
if (StringUtils.isNotBlank(dataset.urn) && dataset.urn.substring(0, 4)
.equalsIgnoreCase(DatasetRowMapper.HDFS_PREFIX)) {
dataset.hdfsBrowserLink =
DatasetRowMapper.HDFS_BROWSER_URL + dataset.urn.substring(DatasetRowMapper.HDFS_URN_PREFIX_LEN);
}
String strOwner = rs.getString(DatasetRowMapper.DATASET_OWNER_ID_COLUMN);
String strOwnerName = rs.getString(DatasetRowMapper.DATASET_OWNER_NAME_COLUMN);
String strOwnerEmail = rs.getString(DatasetRowMapper.DATASET_OWNER_EMAIL_COLUMN);
String[] owners = StringUtils.isNotBlank(strOwner) ? strOwner.split(",") : null;
String[] ownerNames = StringUtils.isNotBlank(strOwnerName) ? strOwnerName.split(",") : null;
String[] ownerEmail = StringUtils.isNotBlank(strOwnerEmail) ? strOwnerEmail.split(",") : null;
dataset.owners = utils.Dataset.fillDatasetOwnerList(owners, ownerNames, ownerEmail);
Time created = rs.getTime(DatasetRowMapper.DATASET_CREATED_TIME_COLUMN);
Time modified = rs.getTime(DatasetRowMapper.DATASET_MODIFIED_TIME_COLUMN);
Long sourceModifiedTime = rs.getLong(DatasetRowMapper.DATASET_SOURCE_MODIFIED_TIME_COLUMN);
Integer schemaHistoryId = rs.getInt(DatasetRowMapper.SCHEMA_HISTORY_ID_COLUMN);
if (modified != null && sourceModifiedTime != null && sourceModifiedTime > 0) {
dataset.modified = new java.util.Date(modified.getTime());
dataset.formatedModified = dataset.modified.toString();
}
if (created != null) {
dataset.created = new java.util.Date(created.getTime());
} else if (modified != null) {
dataset.created = new java.util.Date(modified.getTime());
}
dataset.hasSchemaHistory = schemaHistoryId != null && schemaHistoryId > 0;
Long watchId = rs.getLong(DatasetRowMapper.DATASET_WATCH_ID_COLUMN);
Integer favoriteId = rs.getInt(DatasetRowMapper.FAVORITE_DATASET_ID_COLUMN);
dataset.isFavorite = favoriteId != null && favoriteId > 0;
if (watchId != null && watchId > 0) {
dataset.watchId = watchId;
dataset.isWatched = true;
} else {
dataset.watchId = 0L;
dataset.isWatched = false;
}
return dataset;
}
}

36
wherehows-frontend/app/dao/DatasetsDAO.java
View File

@ -439,17 +439,17 @@ public class DatasetsDAO extends AbstractMySQLOpenSourceDAO
for (Map row : rows) {
Dataset ds = new Dataset();
Timestamp modified = (Timestamp)row.get(DatasetWithUserRowMapper.DATASET_MODIFIED_TIME_COLUMN);
ds.id = (Long)row.get(DatasetWithUserRowMapper.DATASET_ID_COLUMN);
ds.name = (String)row.get(DatasetWithUserRowMapper.DATASET_NAME_COLUMN);
ds.source = (String)row.get(DatasetWithUserRowMapper.DATASET_SOURCE_COLUMN);
ds.urn = (String)row.get(DatasetWithUserRowMapper.DATASET_URN_COLUMN);
ds.schema = (String)row.get(DatasetWithUserRowMapper.DATASET_SCHEMA_COLUMN);
String strOwner = (String)row.get(DatasetWithUserRowMapper.DATASET_OWNER_ID_COLUMN);
String strOwnerName = (String)row.get(DatasetWithUserRowMapper.DATASET_OWNER_NAME_COLUMN);
Timestamp modified = (Timestamp)row.get(DatasetRowMapper.DATASET_MODIFIED_TIME_COLUMN);
ds.id = (Long)row.get(DatasetRowMapper.DATASET_ID_COLUMN);
ds.name = (String)row.get(DatasetRowMapper.DATASET_NAME_COLUMN);
ds.source = (String)row.get(DatasetRowMapper.DATASET_SOURCE_COLUMN);
ds.urn = (String)row.get(DatasetRowMapper.DATASET_URN_COLUMN);
ds.schema = (String)row.get(DatasetRowMapper.DATASET_SCHEMA_COLUMN);
String strOwner = (String)row.get(DatasetRowMapper.DATASET_OWNER_ID_COLUMN);
String strOwnerName = (String)row.get(DatasetRowMapper.DATASET_OWNER_NAME_COLUMN);
Long sourceModifiedTime =
(Long)row.get(DatasetWithUserRowMapper.DATASET_SOURCE_MODIFIED_TIME_COLUMN);
String properties = (String)row.get(DatasetWithUserRowMapper.DATASET_PROPERTIES_COLUMN);
(Long)row.get(DatasetRowMapper.DATASET_SOURCE_MODIFIED_TIME_COLUMN);
String properties = (String)row.get(DatasetRowMapper.DATASET_PROPERTIES_COLUMN);
try
{
if (StringUtils.isNotBlank(properties))
@ -509,8 +509,8 @@ public class DatasetsDAO extends AbstractMySQLOpenSourceDAO
}
}
Integer favoriteId = (Integer)row.get(DatasetWithUserRowMapper.FAVORITE_DATASET_ID_COLUMN);
Long watchId = (Long)row.get(DatasetWithUserRowMapper.DATASET_WATCH_ID_COLUMN);
Integer favoriteId = (Integer)row.get(DatasetRowMapper.FAVORITE_DATASET_ID_COLUMN);
Long watchId = (Long)row.get(DatasetRowMapper.DATASET_WATCH_ID_COLUMN);
Long schemaHistoryRecordCount = 0L;
try
@ -622,7 +622,7 @@ public class DatasetsDAO extends AbstractMySQLOpenSourceDAO
int sortId = 0;
for (Map row : rows)
{
String ownerId = (String)row.get(DatasetWithUserRowMapper.DATASET_OWNER_ID_COLUMN);
String ownerId = (String)row.get(DatasetRowMapper.DATASET_OWNER_ID_COLUMN);
String namespace = (String)row.get("namespace");
int ret = getJdbcTemplate().update(UPDATE_DATASET_OWNER_SORT_ID, ++sortId, id, ownerId, namespace);
if (ret <= 0)
@ -656,7 +656,7 @@ public class DatasetsDAO extends AbstractMySQLOpenSourceDAO
List<User> owners = new ArrayList<User>();
for (Map row : rows)
{
String ownerId = (String)row.get(DatasetWithUserRowMapper.DATASET_OWNER_ID_COLUMN);
String ownerId = (String)row.get(DatasetRowMapper.DATASET_OWNER_ID_COLUMN);
String dislayName = (String)row.get("display_name");
if (StringUtils.isBlank(dislayName))
{
@ -694,7 +694,7 @@ public class DatasetsDAO extends AbstractMySQLOpenSourceDAO
int sortId = 0;
for (Map row : rows)
{
String ownerId = (String)row.get(DatasetWithUserRowMapper.DATASET_OWNER_ID_COLUMN);
String ownerId = (String)row.get(DatasetRowMapper.DATASET_OWNER_ID_COLUMN);
String dislayName = (String)row.get("display_name");
String namespace = (String)row.get("namespace");
if (StringUtils.isBlank(dislayName))
@ -1792,9 +1792,9 @@ public class DatasetsDAO extends AbstractMySQLOpenSourceDAO
for (Map row : rows) {
DatasetListViewNode node = new DatasetListViewNode();
node.datasetId = (Long) row.get(DatasetWithUserRowMapper.DATASET_ID_COLUMN);
node.nodeName = (String) row.get(DatasetWithUserRowMapper.DATASET_NAME_COLUMN);
String nodeUrn = (String) row.get(DatasetWithUserRowMapper.DATASET_URN_COLUMN);
node.datasetId = (Long) row.get(DatasetRowMapper.DATASET_ID_COLUMN);
node.nodeName = (String) row.get(DatasetRowMapper.DATASET_NAME_COLUMN);
String nodeUrn = (String) row.get(DatasetRowMapper.DATASET_URN_COLUMN);
if (node.datasetId != null && node.datasetId > 0)
{
node.nodeUrl = "#/datasets/" + node.datasetId;

25
wherehows-frontend/app/utils/Dataset.java
View File

@ -17,15 +17,22 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.io.BufferedReader;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import play.Logger;
import play.cache.Cache;
import wherehows.models.table.User;
public class Dataset {
private Dataset() {
}
public static final String URNIDMAPKey = "impactUrnIDMap";
private static Cache currentCache = null;
@ -90,4 +97,20 @@ public class Dataset {
}
return impacts;
}
}
public static List<User> fillDatasetOwnerList(String[] owners, String[] ownerNames, String[] ownerEmails) {
List<User> users = new ArrayList<>();
if (owners != null && ownerNames != null && ownerEmails != null && owners.length == ownerNames.length
&& owners.length == ownerEmails.length) {
for (int i = 0; i < owners.length; i++) {
User user = new User();
user.setUserName(owners[i]);
user.setName(ownerNames[i]);
user.setEmail(ownerEmails[i]);
users.add(user);
}
}
return users;
}
}

4
wherehows-frontend/conf/routes
View File

@ -88,6 +88,10 @@ HEAD /api/v1/datasets/urntoid/:urn
GET /api/v1/datasets/:id controllers.api.v1.Dataset.getDatasetByID(id:Int)
GET /api/v1/datasets/:id/view controllers.api.v1.Dataset.getDatasetViewById(id:Int)
PUT /api/v1/datasets/:id/deprecate controllers.api.v1.Dataset.updateDatasetDeprecation(id:Int)
GET /api/v1/datasets/:id/owners controllers.api.v1.Dataset.getDatasetOwnersByID(id:Int)
POST /api/v1/datasets/:id/owners controllers.api.v1.Dataset.updateDatasetOwners(id:Int)