From 795f0ea97af78d8355439aa797a37d391520e08a Mon Sep 17 00:00:00 2001 From: "Yi (Alan) Wang" Date: Thu, 26 Apr 2018 13:18:45 -0700 Subject: [PATCH] Add dataset retention API (#1120) --- .../dao/table/DatasetComplianceDao.java | 10 +++++ .../models/view/DatasetRetention.java | 33 +++++++++++++++ .../dao/DatasetComplianceDaoTest.java | 2 +- .../src/main/avro/ComplianceEntity.avsc | 25 ++++++++--- .../app/controllers/api/v2/Dataset.java | 42 ++++++++++++++++++- wherehows-frontend/conf/routes | 4 ++ 6 files changed, 109 insertions(+), 7 deletions(-) create mode 100644 wherehows-dao/src/main/java/wherehows/models/view/DatasetRetention.java diff --git a/wherehows-dao/src/main/java/wherehows/dao/table/DatasetComplianceDao.java b/wherehows-dao/src/main/java/wherehows/dao/table/DatasetComplianceDao.java index a11cd41c74..0e74a4f5d1 100644 --- a/wherehows-dao/src/main/java/wherehows/dao/table/DatasetComplianceDao.java +++ b/wherehows-dao/src/main/java/wherehows/dao/table/DatasetComplianceDao.java @@ -31,6 +31,7 @@ import wherehows.models.table.DictDataset; import wherehows.models.table.DsCompliance; import wherehows.models.view.DatasetCompliance; import wherehows.models.view.DatasetFieldEntity; +import wherehows.models.view.DatasetRetention; import wherehows.models.view.DsComplianceSuggestion; import static wherehows.util.JsonUtil.*; @@ -164,6 +165,15 @@ public class DatasetComplianceDao extends BaseDao { throw new UnsupportedOperationException("Not implemented yet"); } + @Nullable + public DatasetRetention getDatasetRetention(@Nonnull String datasetUrn) throws Exception { + throw new UnsupportedOperationException("Not implemented yet"); + } + + public void updateDatasetRetention(@Nonnull DatasetRetention record, @Nonnull String user) throws Exception { + throw new UnsupportedOperationException("Not implemented yet"); + } + /** * Convert DatasetCompliance view to DsCompliance entity. Serialize certain fields using json. * @param compliance DatasetCompliance diff --git a/wherehows-dao/src/main/java/wherehows/models/view/DatasetRetention.java b/wherehows-dao/src/main/java/wherehows/models/view/DatasetRetention.java new file mode 100644 index 0000000000..ebe4cb70aa --- /dev/null +++ b/wherehows-dao/src/main/java/wherehows/models/view/DatasetRetention.java @@ -0,0 +1,33 @@ +/** + * Copyright 2015 LinkedIn Corp. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ +package wherehows.models.view; + +import lombok.Data; + + +@Data +public class DatasetRetention { + + private Integer datasetId; + + private String datasetUrn; + + private String purgeType; + + private String purgeNote; + + private String modifiedBy; + + private Long modifiedTime; +} diff --git a/wherehows-dao/src/test/java/wherehows/dao/DatasetComplianceDaoTest.java b/wherehows-dao/src/test/java/wherehows/dao/DatasetComplianceDaoTest.java index 685a883bc1..3a9ba7c114 100644 --- a/wherehows-dao/src/test/java/wherehows/dao/DatasetComplianceDaoTest.java +++ b/wherehows-dao/src/test/java/wherehows/dao/DatasetComplianceDaoTest.java @@ -78,7 +78,7 @@ public class DatasetComplianceDaoTest { policy.datasetConfidentiality = SecurityClassification.HIGHLY_CONFIDENTIAL; String complianceEntityStr = "[{fieldPath: field1, complianceDataType: ADDRESS, complianceDataTypeUrn: null, " - + "fieldFormat: null, securityClassification: CONFIDENTIAL}]"; + + "fieldFormat: null, securityClassification: CONFIDENTIAL, valuePattern: null}]"; complianceDao.fillDsComplianceByCompliancePolicy(dsCompliance, policy, datasetUrn2, actor2); diff --git a/wherehows-data-model/src/main/avro/ComplianceEntity.avsc b/wherehows-data-model/src/main/avro/ComplianceEntity.avsc index 6e16a38eda..44966f90c3 100644 --- a/wherehows-data-model/src/main/avro/ComplianceEntity.avsc +++ b/wherehows-data-model/src/main/avro/ComplianceEntity.avsc @@ -106,13 +106,19 @@ "NUMERIC", "URN", "REVERSED_URN", - "COMPOSITE_URN" + "COMPOSITE_URN", + "CUSTOM", + "ENCODED", + "HASHED" ], "symbolDocs": { - "NUMERIC": "Numerical format", - "URN": "URN format", - "REVERSED_URN": "Reversed URN format", - "COMPOSITE_URN": "Composite URN format" + "NUMERIC": "Numerical format, 12345", + "URN": "URN format, urn:li:member:12345", + "REVERSED_URN": "Reversed URN format, 12345:member:li:urn", + "COMPOSITE_URN": "[Deprecated] Use CUSTOM format + pattern instead", + "CUSTOM": "Any other non-standard format. A pattern for the value is expected to be provided", + "ENCODED": "Data is stored in reversible encoded/serialized/encrypted format", + "HASHED": "Data is stored in irreversible hashed format" }, "doc": "The field format" } @@ -122,6 +128,15 @@ "name": "securityClassification", "type": "SecurityClassification", "doc": "Field security classification that governs data handling." + }, + { + "name": "valuePattern", + "default": null, + "doc": "Optional pattern for the value. Required for CUSTOM fieldFormat", + "type": [ + "null", + "string" + ] } ] } diff --git a/wherehows-frontend/app/controllers/api/v2/Dataset.java b/wherehows-frontend/app/controllers/api/v2/Dataset.java index 1314f3c64b..672bf4a929 100644 --- a/wherehows-frontend/app/controllers/api/v2/Dataset.java +++ b/wherehows-frontend/app/controllers/api/v2/Dataset.java @@ -39,6 +39,7 @@ import wherehows.models.table.AccessControlEntry; import wherehows.models.view.DatasetCompliance; import wherehows.models.view.DatasetOwner; import wherehows.models.view.DatasetOwnership; +import wherehows.models.view.DatasetRetention; import wherehows.models.view.DatasetSchema; import wherehows.models.view.DatasetView; import wherehows.models.view.DsComplianceSuggestion; @@ -316,7 +317,6 @@ public class Dataset extends Controller { Logger.error("Update Compliance Info fail", e); return Promise.promise(() -> internalServerError(errorResponse(e))); } - return Promise.promise(() -> ok(_EMPTY_RESPONSE)); } @@ -356,6 +356,46 @@ public class Dataset extends Controller { return Promise.promise(() -> ok(_EMPTY_RESPONSE)); } + public static Promise getDatasetRetention(@Nonnull String datasetUrn) { + final DatasetRetention record; + try { + record = COMPLIANCE_DAO.getDatasetRetention(datasetUrn); + } catch (Exception e) { + if (e.toString().contains("Response status 404")) { + return Promise.promise(() -> notFound(_EMPTY_RESPONSE)); + } + + Logger.error("Fetch compliance fail", e); + return Promise.promise(() -> internalServerError(errorResponse(e))); + } + + if (record == null) { + return Promise.promise(() -> notFound(_EMPTY_RESPONSE)); + } + return Promise.promise(() -> ok(Json.newObject().set("complianceInfo", Json.toJson(record)))); + } + + public static Promise updateDatasetRetention(@Nonnull String datasetUrn) { + final String username = session("user"); + if (StringUtils.isBlank(username)) { + return Promise.promise(() -> unauthorized(_EMPTY_RESPONSE)); + } + + try { + DatasetRetention record = Json.mapper().convertValue(request().body().asJson(), DatasetRetention.class); + + if (record.getDatasetUrn() == null || !record.getDatasetUrn().equals(datasetUrn)) { + throw new IllegalArgumentException("Dataset Urn not exist or doesn't match."); + } + + COMPLIANCE_DAO.updateDatasetRetention(record, username); + } catch (Exception e) { + Logger.error("Update Retention Policy fail", e); + return Promise.promise(() -> internalServerError(errorResponse(e))); + } + return Promise.promise(() -> ok(_EMPTY_RESPONSE)); + } + public static Promise getDatasetAcls(@Nonnull String datasetUrn) { final List acls; try { diff --git a/wherehows-frontend/conf/routes b/wherehows-frontend/conf/routes index 9944ee18ae..f91eadf598 100644 --- a/wherehows-frontend/conf/routes +++ b/wherehows-frontend/conf/routes @@ -306,6 +306,10 @@ GET /api/v2/datasets/:urn/compliance POST /api/v2/datasets/:urn/compliance controllers.api.v2.Dataset.updateDatasetCompliance(urn: String) +GET /api/v2/datasets/:urn/retention controllers.api.v2.Dataset.getDatasetRetention(urn: String) + +POST /api/v2/datasets/:urn/retention controllers.api.v2.Dataset.updateDatasetRetention(urn: String) + GET /api/v2/datasets/:urn/compliance/suggestion controllers.api.v2.Dataset.getDatasetSuggestedCompliance(urn: String) POST /api/v2/datasets/:urn/compliance/suggestion/feedback controllers.api.v2.Dataset.sendDatasetSuggestedComplianceFeedback(urn: String)