From cfb29deb68778aad5472578069c4f19ad0d8f85f Mon Sep 17 00:00:00 2001 From: "Yi (Alan) Wang" Date: Fri, 20 Oct 2017 11:16:09 -0700 Subject: [PATCH] Add dataset deprecation frontend API (#812) --- .../java/wherehows/dao/table/BaseDao.java | 4 +- .../java/wherehows/dao/table/DatasetsDao.java | 9 + .../wherehows/dao/table/DictDatasetDao.java | 21 +++ .../wherehows/dao/table/FieldDetailDao.java | 2 +- .../java/wherehows/dao/view/BaseViewDao.java | 13 +- .../wherehows/dao/view/DatasetViewDao.java | 50 +++++- .../java/wherehows/models/table/Dataset.java | 1 - .../wherehows/models/view/DatasetView.java | 50 ++++++ .../wherehows/dao/DatasetViewDaoTest.java | 59 +++++++ .../app/controllers/api/v1/Dataset.java | 65 ++++++- .../app/dao/DatasetRowMapper.java | 146 +++++++--------- .../app/dao/DatasetWithUserRowMapper.java | 160 +++++++----------- wherehows-frontend/app/dao/DatasetsDAO.java | 36 ++-- wherehows-frontend/app/utils/Dataset.java | 25 ++- wherehows-frontend/conf/routes | 4 + 15 files changed, 424 insertions(+), 221 deletions(-) create mode 100644 wherehows-dao/src/main/java/wherehows/models/view/DatasetView.java create mode 100644 wherehows-dao/src/test/java/wherehows/dao/DatasetViewDaoTest.java diff --git a/wherehows-dao/src/main/java/wherehows/dao/table/BaseDao.java b/wherehows-dao/src/main/java/wherehows/dao/table/BaseDao.java index b502a350af..ba4ef957b2 100644 --- a/wherehows-dao/src/main/java/wherehows/dao/table/BaseDao.java +++ b/wherehows-dao/src/main/java/wherehows/dao/table/BaseDao.java @@ -239,7 +239,7 @@ public class BaseDao { } /** - * Execute an update or delete query String. + * Execute an update or delete HQL query String, not native query string. * @param queryStr String * @param params Parameters */ @@ -248,6 +248,7 @@ public class BaseDao { EntityManager entityManager = null; try { entityManager = entityManagerFactory.createEntityManager(); + entityManager.getTransaction().begin(); Query query = entityManager.createQuery(queryStr); for (Map.Entry param : params.entrySet()) { @@ -255,6 +256,7 @@ public class BaseDao { } query.executeUpdate(); + entityManager.getTransaction().commit(); } finally { if (entityManager != null) { entityManager.close(); diff --git a/wherehows-dao/src/main/java/wherehows/dao/table/DatasetsDao.java b/wherehows-dao/src/main/java/wherehows/dao/table/DatasetsDao.java index f60052b444..f0fc7a6588 100644 --- a/wherehows-dao/src/main/java/wherehows/dao/table/DatasetsDao.java +++ b/wherehows-dao/src/main/java/wherehows/dao/table/DatasetsDao.java @@ -88,6 +88,15 @@ public class DatasetsDao { return -1; } + /** + * Update dataset owners, set removed owners as deleted, and update existing owner information. + * @param jdbcTemplate JdbcTemplate + * @param user String + * @param datasetId int + * @param datasetUrn String + * @param owners List + * @throws Exception + */ public void updateDatasetOwners(JdbcTemplate jdbcTemplate, String user, int datasetId, String datasetUrn, List owners) throws Exception { // first mark existing owners as deleted, new owners will be updated later diff --git a/wherehows-dao/src/main/java/wherehows/dao/table/DictDatasetDao.java b/wherehows-dao/src/main/java/wherehows/dao/table/DictDatasetDao.java index 46f93da5d3..7760e15e23 100644 --- a/wherehows-dao/src/main/java/wherehows/dao/table/DictDatasetDao.java +++ b/wherehows-dao/src/main/java/wherehows/dao/table/DictDatasetDao.java @@ -42,6 +42,9 @@ public class DictDatasetDao extends BaseDao { super(factory); } + private static final String SET_DATASET_DEPRECATION = + "UPDATE DictDataset SET isDeprecated = :deprecated WHERE id = :datasetId"; + public DictDataset findByUrn(@Nonnull String urn) { return findBy(DictDataset.class, "urn", urn); } @@ -167,4 +170,22 @@ public class DictDatasetDao extends BaseDao { ds.setSourceModifiedTime(sourceTime); } } + + /** + * Set deprecation status of a dataset. + * @param datasetId int + * @param datasetUrn String + * @param isDeprecated boolean + * @param deprecationNote String + * @param user String + * @throws Exception + */ + public void setDatasetDeprecation(int datasetId, @Nonnull String datasetUrn, boolean isDeprecated, + @Nullable String deprecationNote, @Nonnull String user) throws Exception { + Map params = new HashMap<>(); + params.put("datasetId", datasetId); + params.put("deprecated", isDeprecated); + + executeUpdate(SET_DATASET_DEPRECATION, params); + } } diff --git a/wherehows-dao/src/main/java/wherehows/dao/table/FieldDetailDao.java b/wherehows-dao/src/main/java/wherehows/dao/table/FieldDetailDao.java index 0c1a2a2e2b..08a16d8a65 100644 --- a/wherehows-dao/src/main/java/wherehows/dao/table/FieldDetailDao.java +++ b/wherehows-dao/src/main/java/wherehows/dao/table/FieldDetailDao.java @@ -31,7 +31,7 @@ import static wherehows.util.UrnUtil.*; public class FieldDetailDao extends BaseDao { - private static final String DELETE_BY_DATASET_ID = "DELETE FROM dict_field_detail WHERE dataset_id = :datasetId"; + private static final String DELETE_BY_DATASET_ID = "DELETE FROM DictFieldDetail WHERE datasetId = :datasetId"; public FieldDetailDao(@Nonnull EntityManagerFactory factory) { super(factory); diff --git a/wherehows-dao/src/main/java/wherehows/dao/view/BaseViewDao.java b/wherehows-dao/src/main/java/wherehows/dao/view/BaseViewDao.java index b919661c8a..fadd805a10 100644 --- a/wherehows-dao/src/main/java/wherehows/dao/view/BaseViewDao.java +++ b/wherehows-dao/src/main/java/wherehows/dao/view/BaseViewDao.java @@ -15,6 +15,7 @@ package wherehows.dao.view; import java.util.List; import java.util.Map; +import javax.annotation.Nonnull; import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; import javax.persistence.Query; @@ -29,7 +30,7 @@ public class BaseViewDao { private final EntityManagerFactory _emFactory; - public BaseViewDao(EntityManagerFactory factory) { + public BaseViewDao(@Nonnull EntityManagerFactory factory) { this._emFactory = factory; } @@ -43,7 +44,8 @@ public class BaseViewDao { */ @SneakyThrows @SuppressWarnings("unchecked") - protected List getEntityListBy(String sqlQuery, Class classType, Map params) { + protected List getEntityListBy(@Nonnull String sqlQuery, @Nonnull Class classType, + @Nonnull Map params) { EntityManager entityManager = null; try { entityManager = _emFactory.createEntityManager(); @@ -71,7 +73,8 @@ public class BaseViewDao { */ @SneakyThrows @SuppressWarnings("unchecked") - protected T getEntityBy(String sqlQuery, Class classType, Map params) { + protected T getEntityBy(@Nonnull String sqlQuery, @Nonnull Class classType, + @Nonnull Map params) { EntityManager entityManager = null; try { entityManager = _emFactory.createEntityManager(); @@ -97,7 +100,7 @@ public class BaseViewDao { */ @SneakyThrows @SuppressWarnings("unchecked") - protected List getObjectArrayListBy(String sqlQuery, Map params) { + protected List getObjectArrayListBy(@Nonnull String sqlQuery, @Nonnull Map params) { EntityManager entityManager = null; try { entityManager = _emFactory.createEntityManager(); @@ -123,7 +126,7 @@ public class BaseViewDao { */ @SneakyThrows @SuppressWarnings("unchecked") - protected List getObjectListBy(String sqlQuery, Map params) { + protected List getObjectListBy(@Nonnull String sqlQuery, @Nonnull Map params) { EntityManager entityManager = null; try { entityManager = _emFactory.createEntityManager(); diff --git a/wherehows-dao/src/main/java/wherehows/dao/view/DatasetViewDao.java b/wherehows-dao/src/main/java/wherehows/dao/view/DatasetViewDao.java index 2b915b69fb..28c9098f4d 100644 --- a/wherehows-dao/src/main/java/wherehows/dao/view/DatasetViewDao.java +++ b/wherehows-dao/src/main/java/wherehows/dao/view/DatasetViewDao.java @@ -16,20 +16,26 @@ package wherehows.dao.view; import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.annotation.Nonnull; import javax.persistence.EntityManagerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; +import wherehows.dao.table.DictDatasetDao; +import wherehows.models.table.DictDataset; import wherehows.models.view.DatasetColumn; +import wherehows.models.view.DatasetView; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static wherehows.util.UrnUtil.*; +@Slf4j public class DatasetViewDao extends BaseViewDao { - private static final Logger log = LoggerFactory.getLogger(DatasetViewDao.class); + private final DictDatasetDao _dictDatasetDao; - public DatasetViewDao(EntityManagerFactory factory) { + public DatasetViewDao(@Nonnull EntityManagerFactory factory) { super(factory); + _dictDatasetDao = new DictDatasetDao(factory); } private static final String GET_DATASET_COLUMNS_BY_DATASET_ID = @@ -52,13 +58,45 @@ public class DatasetViewDao extends BaseViewDao { + "(ddfc.field_id = dfd.field_id AND ddfc.is_default = true) LEFT JOIN comments c ON " + "c.id = ddfc.comment_id WHERE dfd.dataset_id = :datasetId AND dfd.field_id = :columnId ORDER BY dfd.sort_id"; + /** + * Get dataset view from dict dataset. + * @param datasetId int + * @param datasetUrn String + * @return DatasetView + */ + public DatasetView getDatasetView(int datasetId, @Nonnull String datasetUrn) { + return fillDatasetViewFromDictDataset(_dictDatasetDao.findById(datasetId)); + } + + /** + * Convert DictDataset to DatasetView + * @param ds DictDataset + * @return DatasetView + */ + public DatasetView fillDatasetViewFromDictDataset(DictDataset ds) { + String[] urnParts = splitWhUrn(ds.getUrn()); + + DatasetView view = new DatasetView(); + view.setPlatform(urnParts[0]); + view.setNativeName(urnParts[1]); + view.setUri(ds.getUrn()); + view.setNativeType(ds.getStorageType()); + view.setProperties(ds.getProperties()); + view.setRemoved(ds.getIsActive() != null && !ds.getIsActive()); + view.setDeprecated(ds.getIsDeprecated()); + view.setCreatedTime(1000L * ds.getCreatedTime()); + view.setModifiedTime(1000L * ds.getModifiedTime()); + + return view; + } + /** * Get dataset columns by dataset id * @param datasetId int * @param datasetUrn String * @return List of DatasetColumn */ - public List getDatasetColumnsByID(int datasetId, String datasetUrn) { + public List getDatasetColumnsByID(int datasetId, @Nonnull String datasetUrn) { Map params = new HashMap<>(); params.put("datasetId", datasetId); @@ -83,7 +121,7 @@ public class DatasetViewDao extends BaseViewDao { return columns; } - private void fillInColumnEntity(List columns) { + private void fillInColumnEntity(@Nonnull List columns) { for (DatasetColumn column : columns) { column.setFullFieldPath(isNotBlank(column.getParentPath()) ? column.getParentPath() + "." + column.getFieldName() : column.getFieldName()); diff --git a/wherehows-dao/src/main/java/wherehows/models/table/Dataset.java b/wherehows-dao/src/main/java/wherehows/models/table/Dataset.java index ec3621102b..df40e99f58 100644 --- a/wherehows-dao/src/main/java/wherehows/models/table/Dataset.java +++ b/wherehows-dao/src/main/java/wherehows/models/table/Dataset.java @@ -36,5 +36,4 @@ public class Dataset { public boolean hasSchemaHistory; public JsonNode properties; public List owners; - } diff --git a/wherehows-dao/src/main/java/wherehows/models/view/DatasetView.java b/wherehows-dao/src/main/java/wherehows/models/view/DatasetView.java new file mode 100644 index 0000000000..90459dffb7 --- /dev/null +++ b/wherehows-dao/src/main/java/wherehows/models/view/DatasetView.java @@ -0,0 +1,50 @@ +/** + * Copyright 2015 LinkedIn Corp. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ +package wherehows.models.view; + +import java.util.List; +import lombok.Data; +import lombok.NoArgsConstructor; + + +@Data +@NoArgsConstructor +public class DatasetView { + + private String platform; + + private String nativeName; + + private String fabric; + + private String uri; + + private String description; + + private String nativeType; + + private String properties; + + private List tags; + + private Boolean removed; + + private Boolean deprecated; + + private String deprecationNote; + + private Long createdTime; + + private Long modifiedTime; +} diff --git a/wherehows-dao/src/test/java/wherehows/dao/DatasetViewDaoTest.java b/wherehows-dao/src/test/java/wherehows/dao/DatasetViewDaoTest.java new file mode 100644 index 0000000000..4664f582fb --- /dev/null +++ b/wherehows-dao/src/test/java/wherehows/dao/DatasetViewDaoTest.java @@ -0,0 +1,59 @@ +/** + * Copyright 2015 LinkedIn Corp. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ +package wherehows.dao; + +import org.testng.annotations.Test; +import wherehows.dao.view.DatasetViewDao; +import wherehows.models.table.DictDataset; +import wherehows.models.view.DatasetView; + +import static org.testng.Assert.*; + + +public class DatasetViewDaoTest { + + @Test + public void testFillDatasetViewFromDictDataset() { + DatasetViewDao datasetViewDao = new DatasetViewDao(null); + + String urn = "hdfs:///foo/bar"; + int createTime = 1; + int modifiedTime = 2; + String type = "Avro"; + String properties = "{properties}"; + boolean active = false; + boolean deprecated = true; + + DictDataset ds = new DictDataset(); + ds.setUrn(urn); + ds.setStorageType(type); + ds.setCreatedTime(createTime); + ds.setModifiedTime(modifiedTime); + ds.setProperties(properties); + ds.setIsActive(active); + ds.setIsDeprecated(deprecated); + + DatasetView view = datasetViewDao.fillDatasetViewFromDictDataset(ds); + + assertEquals(view.getPlatform(), "hdfs"); + assertEquals(view.getNativeName(), "/foo/bar"); + assertEquals(view.getNativeType(), type); + assertEquals(view.getUri(), urn); + assertTrue(view.getRemoved()); + assertTrue(view.getDeprecated()); + assertEquals(view.getProperties(), properties); + assertEquals(view.getCreatedTime(), new Long(createTime * 1000)); + assertEquals(view.getModifiedTime(), new Long(modifiedTime * 1000)); + } +} diff --git a/wherehows-frontend/app/controllers/api/v1/Dataset.java b/wherehows-frontend/app/controllers/api/v1/Dataset.java index 0a3f2bc86d..3ffddf8bfb 100644 --- a/wherehows-frontend/app/controllers/api/v1/Dataset.java +++ b/wherehows-frontend/app/controllers/api/v1/Dataset.java @@ -32,9 +32,9 @@ import play.libs.F.Promise; import play.libs.Json; import play.mvc.Controller; import play.mvc.Result; -import wherehows.dao.table.DatasetClassificationDao; import wherehows.dao.table.DatasetComplianceDao; import wherehows.dao.table.DatasetsDao; +import wherehows.dao.table.DictDatasetDao; import wherehows.dao.view.DatasetViewDao; import wherehows.dao.view.OwnerViewDao; import wherehows.models.view.DatasetCompliance; @@ -42,6 +42,7 @@ import wherehows.models.table.DatasetDependency; import wherehows.models.table.ImpactDataset; import wherehows.models.view.DatasetColumn; import wherehows.models.view.DatasetOwner; +import wherehows.models.view.DatasetView; import wherehows.models.view.DsComplianceSuggestion; @@ -53,8 +54,7 @@ public class Dataset extends Controller { private static final DatasetsDao DATASETS_DAO = Application.DAO_FACTORY.getDatasetsDao(); - private static final DatasetClassificationDao CLASSIFICATION_DAO = - Application.DAO_FACTORY.getDatasetClassificationDao(); + private static final DictDatasetDao DICT_DATASET_DAO = Application.DAO_FACTORY.getDictDatasetDao(); private static final DatasetViewDao DATASET_VIEW_DAO = Application.DAO_FACTORY.getDatasetViewDao(); @@ -129,6 +129,63 @@ public class Dataset extends Controller { return ok(result); } + public static Promise getDatasetViewById(int datasetId) { + DatasetView view = null; + try { + String urn = getDatasetUrnByIdOrCache(datasetId); + + view = DATASET_VIEW_DAO.getDatasetView(datasetId, urn); + } catch (Exception e) { + Logger.warn("Failed to get dataset view", e); + JsonNode result = Json.newObject() + .put("status", "failed") + .put("error", "true") + .put("msg", "Fetch data Error: " + e.getMessage()); + + return Promise.promise(() -> ok(result)); + } + + if (view == null) { + JsonNode result = Json.newObject().put("status", "failed").put("msg", "Not found"); + return Promise.promise(() -> ok(result)); + } + + JsonNode result = Json.newObject().put("status", "ok").set("dataset", Json.toJson(view)); + return Promise.promise(() -> ok(result)); + } + + public static Promise updateDatasetDeprecation(int datasetId) { + String username = session("user"); + if (StringUtils.isBlank(username)) { + JsonNode result = Json.newObject().put("status", "failed").put("error", "true").put("msg", "Unauthorized User."); + + return Promise.promise(() -> ok(result)); + } + + try { + JsonNode record = request().body().asJson(); + + boolean deprecated = record.get("deprecated").asBoolean(); + + String deprecationNote = record.get("deprecationNote").asText(); + + String urn = getDatasetUrnByIdOrCache(datasetId); + + DICT_DATASET_DAO.setDatasetDeprecation(datasetId, urn, deprecated, deprecationNote, username); + } catch (Exception e) { + JsonNode result = Json.newObject() + .put("status", "failed") + .put("error", "true") + .put("msg", "Update Compliance Info Error: " + e.getMessage()); + + Logger.warn("Update dataset deprecation info fail", e); + + return Promise.promise(() -> ok(result)); + } + + return Promise.promise(() -> ok(Json.newObject().put("status", "ok"))); + } + private static Integer getDatasetIdByUrnOrCache(String urn) { String cacheKey = URN_CACHE_KEY + urn; @@ -165,7 +222,7 @@ public class Dataset extends Controller { urn = DATASETS_DAO.validateUrn(JDBC_TEMPLATE, datasetId); if (urn != null) { - Cache.set(cacheKey, urn); + Cache.set(cacheKey, urn, DATASET_ID_CACHE_PERIOD); } return urn; } diff --git a/wherehows-frontend/app/dao/DatasetRowMapper.java b/wherehows-frontend/app/dao/DatasetRowMapper.java index 3360f24dbc..a554873c8d 100644 --- a/wherehows-frontend/app/dao/DatasetRowMapper.java +++ b/wherehows-frontend/app/dao/DatasetRowMapper.java @@ -13,98 +13,78 @@ */ package dao; -import wherehows.models.table.Dataset; -import wherehows.models.table.User; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Time; import org.apache.commons.lang3.StringUtils; import org.springframework.jdbc.core.RowMapper; import play.Play; - -import java.sql.Time; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; - -public class DatasetRowMapper implements RowMapper -{ - private static final String HDFS_BROWSER_URL = - Play.application().configuration().getString(DatasetsDAO.HDFS_BROWSER_URL_KEY); - - public static final String DATASET_ID_COLUMN = "id"; - public static final String DATASET_NAME_COLUMN = "name"; - public static final String DATASET_URN_COLUMN = "urn"; - public static final String DATASET_SOURCE_COLUMN = "source"; - public static final String DATASET_CREATED_TIME_COLUMN = "created"; - private static final String DATASET_MODIFIED_TIME_COLUMN = "modified"; - private static final String DATASET_SOURCE_MODIFIED_TIME_COLUMN = "source_modified_time"; - private static final String DATASET_PROPERTIES_COLUMN = "properties"; - private static final String DATASET_OWNER_ID_COLUMN = "owner_id"; - private static final String DATASET_OWNER_NAME_COLUMN = "owner_name"; - private static final String DATASET_OWNER_EMAIL_COLUMN = "owner_email"; - private static final String SCHEMA_HISTORY_ID_COLUMN = "schema_history_id"; - public static final String DATASET_SCHEMA_COLUMN = "schema"; - public static final String HDFS_PREFIX = "hdfs"; - public static final int HDFS_URN_PREFIX_LEN = 7; //for hdfs prefix is hdfs:///, but we need the last slash +import wherehows.models.table.Dataset; - @Override - public Dataset mapRow(ResultSet rs, int rowNum) throws SQLException - { - int id = rs.getInt(DATASET_ID_COLUMN); - String name = rs.getString(DATASET_NAME_COLUMN); - String urn = rs.getString(DATASET_URN_COLUMN); - String source = rs.getString(DATASET_SOURCE_COLUMN); - String strOwner = rs.getString(DATASET_OWNER_ID_COLUMN); - String strOwnerName = rs.getString(DATASET_OWNER_NAME_COLUMN); - String strOwnerEmail = rs.getString(DATASET_OWNER_EMAIL_COLUMN); - String schema = rs.getString(DATASET_SCHEMA_COLUMN); - Time created = rs.getTime(DATASET_CREATED_TIME_COLUMN); - Time modified = rs.getTime(DATASET_MODIFIED_TIME_COLUMN); - Integer schemaHistoryId = rs.getInt(SCHEMA_HISTORY_ID_COLUMN); - Long sourceModifiedTime = rs.getLong(DATASET_SOURCE_MODIFIED_TIME_COLUMN); +public class DatasetRowMapper implements RowMapper { - Dataset dataset = new Dataset(); - dataset.id = id; - dataset.name = name; - dataset.urn = urn; - dataset.schema = schema; - String[] owners = StringUtils.isNotBlank(strOwner) ? strOwner.split(",") : null; - String[] ownerNames = StringUtils.isNotBlank(strOwnerName) ? strOwnerName.split(",") : null; - String[] ownerEmail = StringUtils.isNotBlank(strOwnerEmail) ? strOwnerEmail.split(",") : null; + public static final String HDFS_BROWSER_URL = + Play.application().configuration().getString(DatasetsDAO.HDFS_BROWSER_URL_KEY); - dataset.owners = new ArrayList<>(); - if (owners != null && ownerNames != null && ownerEmail != null && owners.length == ownerNames.length - && owners.length == ownerEmail.length) - { - for (int i = 0; i < owners.length; i++) - { - User user = new User(); - user.setUserName(owners[i]); - user.setName(ownerNames[i]); - user.setEmail(ownerEmail[i]); - dataset.owners.add(user); - } - } + public static final String DATASET_ID_COLUMN = "id"; + public static final String DATASET_NAME_COLUMN = "name"; + public static final String DATASET_URN_COLUMN = "urn"; + public static final String DATASET_SCHEMA_COLUMN = "schema"; + public static final String DATASET_SOURCE_COLUMN = "source"; + public static final String DATASET_PROPERTIES_COLUMN = "properties"; + public static final String DATASET_CREATED_TIME_COLUMN = "created"; + public static final String DATASET_MODIFIED_TIME_COLUMN = "modified"; + public static final String DATASET_SOURCE_MODIFIED_TIME_COLUMN = "source_modified_time"; + public static final String SCHEMA_HISTORY_ID_COLUMN = "schema_history_id"; + public static final String DATASET_OWNER_ID_COLUMN = "owner_id"; + public static final String DATASET_OWNER_NAME_COLUMN = "owner_name"; + public static final String DATASET_OWNER_EMAIL_COLUMN = "owner_email"; + public static final String FAVORITE_DATASET_ID_COLUMN = "dataset_id"; + public static final String DATASET_WATCH_ID_COLUMN = "watch_id"; - if (StringUtils.isNotBlank(dataset.urn) && dataset.urn.substring(0, 4).equalsIgnoreCase(HDFS_PREFIX)) - { - dataset.hdfsBrowserLink = HDFS_BROWSER_URL + dataset.urn.substring(HDFS_URN_PREFIX_LEN); - } + public static final String HDFS_PREFIX = "hdfs"; + public static final int HDFS_URN_PREFIX_LEN = 7; //for hdfs prefix is hdfs:///, but we need the last slash - dataset.source = source; - if (modified != null && sourceModifiedTime != null && sourceModifiedTime > 0) - { - dataset.modified = new java.util.Date(modified.getTime()); - dataset.formatedModified = dataset.modified.toString(); - } - if (created != null) - { - dataset.created = new java.util.Date(created.getTime()); - } else if (modified != null) { - dataset.created = new java.util.Date(modified.getTime()); - } + @Override + public Dataset mapRow(ResultSet rs, int rowNum) throws SQLException { + Dataset dataset = new Dataset(); + dataset.id = rs.getInt(DATASET_ID_COLUMN); + dataset.name = rs.getString(DATASET_NAME_COLUMN); + dataset.urn = rs.getString(DATASET_URN_COLUMN); + dataset.schema = rs.getString(DATASET_SCHEMA_COLUMN); + dataset.source = rs.getString(DATASET_SOURCE_COLUMN); - dataset.hasSchemaHistory = schemaHistoryId != null && schemaHistoryId > 0; - - return dataset; + if (StringUtils.isNotBlank(dataset.urn) && dataset.urn.substring(0, 4).equalsIgnoreCase(HDFS_PREFIX)) { + dataset.hdfsBrowserLink = HDFS_BROWSER_URL + dataset.urn.substring(HDFS_URN_PREFIX_LEN); } + + String strOwner = rs.getString(DATASET_OWNER_ID_COLUMN); + String strOwnerName = rs.getString(DATASET_OWNER_NAME_COLUMN); + String strOwnerEmail = rs.getString(DATASET_OWNER_EMAIL_COLUMN); + String[] owners = StringUtils.isNotBlank(strOwner) ? strOwner.split(",") : null; + String[] ownerNames = StringUtils.isNotBlank(strOwnerName) ? strOwnerName.split(",") : null; + String[] ownerEmail = StringUtils.isNotBlank(strOwnerEmail) ? strOwnerEmail.split(",") : null; + + dataset.owners = utils.Dataset.fillDatasetOwnerList(owners, ownerNames, ownerEmail); + + Time created = rs.getTime(DATASET_CREATED_TIME_COLUMN); + Time modified = rs.getTime(DATASET_MODIFIED_TIME_COLUMN); + Long sourceModifiedTime = rs.getLong(DATASET_SOURCE_MODIFIED_TIME_COLUMN); + Integer schemaHistoryId = rs.getInt(SCHEMA_HISTORY_ID_COLUMN); + + if (modified != null && sourceModifiedTime != null && sourceModifiedTime > 0) { + dataset.modified = new java.util.Date(modified.getTime()); + dataset.formatedModified = dataset.modified.toString(); + } + if (created != null) { + dataset.created = new java.util.Date(created.getTime()); + } else if (modified != null) { + dataset.created = new java.util.Date(modified.getTime()); + } + + dataset.hasSchemaHistory = schemaHistoryId != null && schemaHistoryId > 0; + + return dataset; + } } diff --git a/wherehows-frontend/app/dao/DatasetWithUserRowMapper.java b/wherehows-frontend/app/dao/DatasetWithUserRowMapper.java index 32e90e959c..9825800375 100644 --- a/wherehows-frontend/app/dao/DatasetWithUserRowMapper.java +++ b/wherehows-frontend/app/dao/DatasetWithUserRowMapper.java @@ -13,113 +13,71 @@ */ package dao; -import wherehows.models.table.Dataset; -import wherehows.models.table.User; -import org.apache.commons.lang3.StringUtils; -import org.springframework.jdbc.core.RowMapper; -import play.Play; - -import java.sql.Time; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; +import java.sql.Time; +import org.apache.commons.lang3.StringUtils; +import org.springframework.jdbc.core.RowMapper; +import wherehows.models.table.Dataset; -public class DatasetWithUserRowMapper implements RowMapper -{ - private static final String HDFS_BROWSER_URL = - Play.application().configuration().getString(DatasetsDAO.HDFS_BROWSER_URL_KEY); - public static final String DATASET_ID_COLUMN = "id"; - public static final String DATASET_NAME_COLUMN = "name"; - public static final String DATASET_URN_COLUMN = "urn"; - public static final String DATASET_SCHEMA_COLUMN = "schema"; - public static final String DATASET_SOURCE_COLUMN = "source"; - public static final String DATASET_CREATED_TIME_COLUMN = "created"; - public static final String DATASET_MODIFIED_TIME_COLUMN = "modified"; - public static final String DATASET_SOURCE_MODIFIED_TIME_COLUMN = "source_modified_time"; - public static final String FAVORITE_DATASET_ID_COLUMN = "dataset_id"; - public static final String DATASET_WATCH_ID_COLUMN = "watch_id"; - public static final String DATASET_PROPERTIES_COLUMN = "properties"; - public static final String SCHEMA_HISTORY_ID_COLUMN = "schema_history_id"; - public static final String DATASET_OWNER_ID_COLUMN = "owner_id"; - public static final String DATASET_OWNER_NAME_COLUMN = "owner_name"; - public static final String DATASET_OWNER_EMAIL_COLUMN = "owner_email"; - public static final String HDFS_PREFIX = "hdfs"; +public class DatasetWithUserRowMapper implements RowMapper { - @Override - public Dataset mapRow(ResultSet rs, int rowNum) throws SQLException - { - int id = rs.getInt(DATASET_ID_COLUMN); - String name = rs.getString(DATASET_NAME_COLUMN); - String urn = rs.getString(DATASET_URN_COLUMN); - String source = rs.getString(DATASET_SOURCE_COLUMN); - String schema = rs.getString(DATASET_SCHEMA_COLUMN); - Time created = rs.getTime(DATASET_CREATED_TIME_COLUMN); - Time modified = rs.getTime(DATASET_MODIFIED_TIME_COLUMN); - Integer favoriteId = rs.getInt(FAVORITE_DATASET_ID_COLUMN); - Integer schemaHistoryId = rs.getInt(SCHEMA_HISTORY_ID_COLUMN); - Long watchId = rs.getLong(DATASET_WATCH_ID_COLUMN); - Long sourceModifiedTime = rs.getLong(DATASET_SOURCE_MODIFIED_TIME_COLUMN); - String strOwner = rs.getString(DATASET_OWNER_ID_COLUMN); - String strOwnerName = rs.getString(DATASET_OWNER_NAME_COLUMN); - String strOwnerEmail = rs.getString(DATASET_OWNER_EMAIL_COLUMN); + @Override + public Dataset mapRow(ResultSet rs, int rowNum) throws SQLException { - Dataset dataset = new Dataset(); - dataset.id = id; - dataset.name = name; - dataset.urn = urn; - dataset.schema = schema; - String[] owners = StringUtils.isNotBlank(strOwner) ? strOwner.split(",") : null; - String[] ownerNames = StringUtils.isNotBlank(strOwnerName) ? strOwnerName.split(",") : null; - String[] ownerEmail = StringUtils.isNotBlank(strOwnerEmail) ? strOwnerEmail.split(",") : null; + Dataset dataset = new Dataset(); + dataset.id = rs.getInt(DatasetRowMapper.DATASET_ID_COLUMN); + dataset.name = rs.getString(DatasetRowMapper.DATASET_NAME_COLUMN); + dataset.urn = rs.getString(DatasetRowMapper.DATASET_URN_COLUMN); + dataset.schema = rs.getString(DatasetRowMapper.DATASET_SCHEMA_COLUMN); + dataset.source = rs.getString(DatasetRowMapper.DATASET_SOURCE_COLUMN); - dataset.owners = new ArrayList<>(); - if (owners != null && ownerNames != null && ownerEmail != null && owners.length == ownerNames.length - && owners.length == ownerEmail.length) - { - for (int i = 0; i < owners.length; i++) - { - User user = new User(); - user.setUserName(owners[i]); - user.setName(ownerNames[i]); - user.setEmail(ownerEmail[i]); - dataset.owners.add(user); - } - } - - if (StringUtils.isNotBlank(dataset.urn) && dataset.urn.substring(0, 4).equalsIgnoreCase(HDFS_PREFIX)) - { - dataset.hdfsBrowserLink = HDFS_BROWSER_URL + dataset.urn.substring(DatasetRowMapper.HDFS_URN_PREFIX_LEN); - } - - dataset.source = source; - if (modified != null && sourceModifiedTime != null && sourceModifiedTime > 0) - { - dataset.modified = new java.util.Date(modified.getTime()); - dataset.formatedModified = dataset.modified.toString(); - } - if (created != null) - { - dataset.created = new java.util.Date(created.getTime()); - } else if (modified != null) { - dataset.created = new java.util.Date(modified.getTime()); - } - - dataset.isFavorite = favoriteId != null && favoriteId > 0; - - if (watchId != null && watchId > 0) - { - dataset.watchId = watchId; - dataset.isWatched = true; - } - else - { - dataset.watchId = 0L; - dataset.isWatched = false; - } - - dataset.hasSchemaHistory = schemaHistoryId != null && schemaHistoryId > 0; - - return dataset; + if (StringUtils.isNotBlank(dataset.urn) && dataset.urn.substring(0, 4) + .equalsIgnoreCase(DatasetRowMapper.HDFS_PREFIX)) { + dataset.hdfsBrowserLink = + DatasetRowMapper.HDFS_BROWSER_URL + dataset.urn.substring(DatasetRowMapper.HDFS_URN_PREFIX_LEN); } + + String strOwner = rs.getString(DatasetRowMapper.DATASET_OWNER_ID_COLUMN); + String strOwnerName = rs.getString(DatasetRowMapper.DATASET_OWNER_NAME_COLUMN); + String strOwnerEmail = rs.getString(DatasetRowMapper.DATASET_OWNER_EMAIL_COLUMN); + String[] owners = StringUtils.isNotBlank(strOwner) ? strOwner.split(",") : null; + String[] ownerNames = StringUtils.isNotBlank(strOwnerName) ? strOwnerName.split(",") : null; + String[] ownerEmail = StringUtils.isNotBlank(strOwnerEmail) ? strOwnerEmail.split(",") : null; + + dataset.owners = utils.Dataset.fillDatasetOwnerList(owners, ownerNames, ownerEmail); + + Time created = rs.getTime(DatasetRowMapper.DATASET_CREATED_TIME_COLUMN); + Time modified = rs.getTime(DatasetRowMapper.DATASET_MODIFIED_TIME_COLUMN); + Long sourceModifiedTime = rs.getLong(DatasetRowMapper.DATASET_SOURCE_MODIFIED_TIME_COLUMN); + Integer schemaHistoryId = rs.getInt(DatasetRowMapper.SCHEMA_HISTORY_ID_COLUMN); + + if (modified != null && sourceModifiedTime != null && sourceModifiedTime > 0) { + dataset.modified = new java.util.Date(modified.getTime()); + dataset.formatedModified = dataset.modified.toString(); + } + if (created != null) { + dataset.created = new java.util.Date(created.getTime()); + } else if (modified != null) { + dataset.created = new java.util.Date(modified.getTime()); + } + + dataset.hasSchemaHistory = schemaHistoryId != null && schemaHistoryId > 0; + + Long watchId = rs.getLong(DatasetRowMapper.DATASET_WATCH_ID_COLUMN); + Integer favoriteId = rs.getInt(DatasetRowMapper.FAVORITE_DATASET_ID_COLUMN); + + dataset.isFavorite = favoriteId != null && favoriteId > 0; + + if (watchId != null && watchId > 0) { + dataset.watchId = watchId; + dataset.isWatched = true; + } else { + dataset.watchId = 0L; + dataset.isWatched = false; + } + + return dataset; + } } diff --git a/wherehows-frontend/app/dao/DatasetsDAO.java b/wherehows-frontend/app/dao/DatasetsDAO.java index 4aaa833afa..472076aa07 100644 --- a/wherehows-frontend/app/dao/DatasetsDAO.java +++ b/wherehows-frontend/app/dao/DatasetsDAO.java @@ -439,17 +439,17 @@ public class DatasetsDAO extends AbstractMySQLOpenSourceDAO for (Map row : rows) { Dataset ds = new Dataset(); - Timestamp modified = (Timestamp)row.get(DatasetWithUserRowMapper.DATASET_MODIFIED_TIME_COLUMN); - ds.id = (Long)row.get(DatasetWithUserRowMapper.DATASET_ID_COLUMN); - ds.name = (String)row.get(DatasetWithUserRowMapper.DATASET_NAME_COLUMN); - ds.source = (String)row.get(DatasetWithUserRowMapper.DATASET_SOURCE_COLUMN); - ds.urn = (String)row.get(DatasetWithUserRowMapper.DATASET_URN_COLUMN); - ds.schema = (String)row.get(DatasetWithUserRowMapper.DATASET_SCHEMA_COLUMN); - String strOwner = (String)row.get(DatasetWithUserRowMapper.DATASET_OWNER_ID_COLUMN); - String strOwnerName = (String)row.get(DatasetWithUserRowMapper.DATASET_OWNER_NAME_COLUMN); + Timestamp modified = (Timestamp)row.get(DatasetRowMapper.DATASET_MODIFIED_TIME_COLUMN); + ds.id = (Long)row.get(DatasetRowMapper.DATASET_ID_COLUMN); + ds.name = (String)row.get(DatasetRowMapper.DATASET_NAME_COLUMN); + ds.source = (String)row.get(DatasetRowMapper.DATASET_SOURCE_COLUMN); + ds.urn = (String)row.get(DatasetRowMapper.DATASET_URN_COLUMN); + ds.schema = (String)row.get(DatasetRowMapper.DATASET_SCHEMA_COLUMN); + String strOwner = (String)row.get(DatasetRowMapper.DATASET_OWNER_ID_COLUMN); + String strOwnerName = (String)row.get(DatasetRowMapper.DATASET_OWNER_NAME_COLUMN); Long sourceModifiedTime = - (Long)row.get(DatasetWithUserRowMapper.DATASET_SOURCE_MODIFIED_TIME_COLUMN); - String properties = (String)row.get(DatasetWithUserRowMapper.DATASET_PROPERTIES_COLUMN); + (Long)row.get(DatasetRowMapper.DATASET_SOURCE_MODIFIED_TIME_COLUMN); + String properties = (String)row.get(DatasetRowMapper.DATASET_PROPERTIES_COLUMN); try { if (StringUtils.isNotBlank(properties)) @@ -509,8 +509,8 @@ public class DatasetsDAO extends AbstractMySQLOpenSourceDAO } } - Integer favoriteId = (Integer)row.get(DatasetWithUserRowMapper.FAVORITE_DATASET_ID_COLUMN); - Long watchId = (Long)row.get(DatasetWithUserRowMapper.DATASET_WATCH_ID_COLUMN); + Integer favoriteId = (Integer)row.get(DatasetRowMapper.FAVORITE_DATASET_ID_COLUMN); + Long watchId = (Long)row.get(DatasetRowMapper.DATASET_WATCH_ID_COLUMN); Long schemaHistoryRecordCount = 0L; try @@ -622,7 +622,7 @@ public class DatasetsDAO extends AbstractMySQLOpenSourceDAO int sortId = 0; for (Map row : rows) { - String ownerId = (String)row.get(DatasetWithUserRowMapper.DATASET_OWNER_ID_COLUMN); + String ownerId = (String)row.get(DatasetRowMapper.DATASET_OWNER_ID_COLUMN); String namespace = (String)row.get("namespace"); int ret = getJdbcTemplate().update(UPDATE_DATASET_OWNER_SORT_ID, ++sortId, id, ownerId, namespace); if (ret <= 0) @@ -656,7 +656,7 @@ public class DatasetsDAO extends AbstractMySQLOpenSourceDAO List owners = new ArrayList(); for (Map row : rows) { - String ownerId = (String)row.get(DatasetWithUserRowMapper.DATASET_OWNER_ID_COLUMN); + String ownerId = (String)row.get(DatasetRowMapper.DATASET_OWNER_ID_COLUMN); String dislayName = (String)row.get("display_name"); if (StringUtils.isBlank(dislayName)) { @@ -694,7 +694,7 @@ public class DatasetsDAO extends AbstractMySQLOpenSourceDAO int sortId = 0; for (Map row : rows) { - String ownerId = (String)row.get(DatasetWithUserRowMapper.DATASET_OWNER_ID_COLUMN); + String ownerId = (String)row.get(DatasetRowMapper.DATASET_OWNER_ID_COLUMN); String dislayName = (String)row.get("display_name"); String namespace = (String)row.get("namespace"); if (StringUtils.isBlank(dislayName)) @@ -1792,9 +1792,9 @@ public class DatasetsDAO extends AbstractMySQLOpenSourceDAO for (Map row : rows) { DatasetListViewNode node = new DatasetListViewNode(); - node.datasetId = (Long) row.get(DatasetWithUserRowMapper.DATASET_ID_COLUMN); - node.nodeName = (String) row.get(DatasetWithUserRowMapper.DATASET_NAME_COLUMN); - String nodeUrn = (String) row.get(DatasetWithUserRowMapper.DATASET_URN_COLUMN); + node.datasetId = (Long) row.get(DatasetRowMapper.DATASET_ID_COLUMN); + node.nodeName = (String) row.get(DatasetRowMapper.DATASET_NAME_COLUMN); + String nodeUrn = (String) row.get(DatasetRowMapper.DATASET_URN_COLUMN); if (node.datasetId != null && node.datasetId > 0) { node.nodeUrl = "#/datasets/" + node.datasetId; diff --git a/wherehows-frontend/app/utils/Dataset.java b/wherehows-frontend/app/utils/Dataset.java index fed7ea5e5c..dc0dbe5559 100644 --- a/wherehows-frontend/app/utils/Dataset.java +++ b/wherehows-frontend/app/utils/Dataset.java @@ -17,15 +17,22 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.BufferedReader; import java.io.FileReader; +import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import org.apache.commons.lang3.StringUtils; import play.Logger; import play.cache.Cache; +import wherehows.models.table.User; public class Dataset { + + private Dataset() { + } + public static final String URNIDMAPKey = "impactUrnIDMap"; private static Cache currentCache = null; @@ -90,4 +97,20 @@ public class Dataset { } return impacts; } -} \ No newline at end of file + + public static List fillDatasetOwnerList(String[] owners, String[] ownerNames, String[] ownerEmails) { + List users = new ArrayList<>(); + + if (owners != null && ownerNames != null && ownerEmails != null && owners.length == ownerNames.length + && owners.length == ownerEmails.length) { + for (int i = 0; i < owners.length; i++) { + User user = new User(); + user.setUserName(owners[i]); + user.setName(ownerNames[i]); + user.setEmail(ownerEmails[i]); + users.add(user); + } + } + return users; + } +} diff --git a/wherehows-frontend/conf/routes b/wherehows-frontend/conf/routes index 64044f4f18..f1a53fd1f3 100644 --- a/wherehows-frontend/conf/routes +++ b/wherehows-frontend/conf/routes @@ -88,6 +88,10 @@ HEAD /api/v1/datasets/urntoid/:urn GET /api/v1/datasets/:id controllers.api.v1.Dataset.getDatasetByID(id:Int) +GET /api/v1/datasets/:id/view controllers.api.v1.Dataset.getDatasetViewById(id:Int) + +PUT /api/v1/datasets/:id/deprecate controllers.api.v1.Dataset.updateDatasetDeprecation(id:Int) + GET /api/v1/datasets/:id/owners controllers.api.v1.Dataset.getDatasetOwnersByID(id:Int) POST /api/v1/datasets/:id/owners controllers.api.v1.Dataset.updateDatasetOwners(id:Int)