Add dataset retention API (#1120)

wherehows-dao/src/main/java/wherehows/dao/table/DatasetComplianceDao.java

@@ -31,6 +31,7 @@ import wherehows.models.table.DictDataset;
 import wherehows.models.table.DsCompliance;
 import wherehows.models.view.DatasetCompliance;
 import wherehows.models.view.DatasetFieldEntity;
+import wherehows.models.view.DatasetRetention;
 import wherehows.models.view.DsComplianceSuggestion;
 
 import static wherehows.util.JsonUtil.*;
@@ -164,6 +165,15 @@ public class DatasetComplianceDao extends BaseDao {
     throw new UnsupportedOperationException("Not implemented yet");
   }
 
+  @Nullable
+  public DatasetRetention getDatasetRetention(@Nonnull String datasetUrn) throws Exception {
+    throw new UnsupportedOperationException("Not implemented yet");
+  }
+
+  public void updateDatasetRetention(@Nonnull DatasetRetention record, @Nonnull String user) throws Exception {
+    throw new UnsupportedOperationException("Not implemented yet");
+  }
+
   /**
    * Convert DatasetCompliance view to DsCompliance entity. Serialize certain fields using json.
    * @param compliance DatasetCompliance

wherehows-dao/src/main/java/wherehows/models/view/DatasetRetention.java

@@ -0,0 +1,33 @@
+/**
+ * Copyright 2015 LinkedIn Corp. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ */
+package wherehows.models.view;
+
+import lombok.Data;
+
+
+@Data
+public class DatasetRetention {
+
+  private Integer datasetId;
+
+  private String datasetUrn;
+
+  private String purgeType;
+
+  private String purgeNote;
+
+  private String modifiedBy;
+
+  private Long modifiedTime;
+}

wherehows-dao/src/test/java/wherehows/dao/DatasetComplianceDaoTest.java

@@ -78,7 +78,7 @@ public class DatasetComplianceDaoTest {
     policy.datasetConfidentiality = SecurityClassification.HIGHLY_CONFIDENTIAL;
 
     String complianceEntityStr = "[{fieldPath: field1, complianceDataType: ADDRESS, complianceDataTypeUrn: null, "
-        + "fieldFormat: null, securityClassification: CONFIDENTIAL}]";
+        + "fieldFormat: null, securityClassification: CONFIDENTIAL, valuePattern: null}]";
 
     complianceDao.fillDsComplianceByCompliancePolicy(dsCompliance, policy, datasetUrn2, actor2);
 

wherehows-data-model/src/main/avro/ComplianceEntity.avsc

@@ -106,13 +106,19 @@
             "NUMERIC",
             "URN",
             "REVERSED_URN",
-            "COMPOSITE_URN"
+            "COMPOSITE_URN",
+            "CUSTOM",
+            "ENCODED",
+            "HASHED"
           ],
           "symbolDocs": {
-            "NUMERIC": "Numerical format",
-            "URN": "URN format",
-            "REVERSED_URN": "Reversed URN format",
-            "COMPOSITE_URN": "Composite URN format"
+            "NUMERIC": "Numerical format, 12345",
+            "URN": "URN format, urn:li:member:12345",
+            "REVERSED_URN": "Reversed URN format, 12345:member:li:urn",
+            "COMPOSITE_URN": "[Deprecated] Use CUSTOM format + pattern instead",
+            "CUSTOM": "Any other non-standard format. A pattern for the value is expected to be provided",
+            "ENCODED": "Data is stored in reversible encoded/serialized/encrypted format",
+            "HASHED": "Data is stored in irreversible hashed format"
           },
           "doc": "The field format"
         }
@@ -122,6 +128,15 @@
       "name": "securityClassification",
       "type": "SecurityClassification",
       "doc": "Field security classification that governs data handling."
+    },
+    {
+      "name": "valuePattern",
+      "default": null,
+      "doc": "Optional pattern for the value. Required for CUSTOM fieldFormat",
+      "type": [
+        "null",
+        "string"
+      ]
     }
   ]
 }

wherehows-frontend/app/controllers/api/v2/Dataset.java

@@ -39,6 +39,7 @@ import wherehows.models.table.AccessControlEntry;
 import wherehows.models.view.DatasetCompliance;
 import wherehows.models.view.DatasetOwner;
 import wherehows.models.view.DatasetOwnership;
+import wherehows.models.view.DatasetRetention;
 import wherehows.models.view.DatasetSchema;
 import wherehows.models.view.DatasetView;
 import wherehows.models.view.DsComplianceSuggestion;
@@ -316,7 +317,6 @@ public class Dataset extends Controller {
       Logger.error("Update Compliance Info fail", e);
       return Promise.promise(() -> internalServerError(errorResponse(e)));
     }
-
     return Promise.promise(() -> ok(_EMPTY_RESPONSE));
   }
 
@@ -356,6 +356,46 @@ public class Dataset extends Controller {
     return Promise.promise(() -> ok(_EMPTY_RESPONSE));
   }
 
+  public static Promise<Result> getDatasetRetention(@Nonnull String datasetUrn) {
+    final DatasetRetention record;
+    try {
+      record = COMPLIANCE_DAO.getDatasetRetention(datasetUrn);
+    } catch (Exception e) {
+      if (e.toString().contains("Response status 404")) {
+        return Promise.promise(() -> notFound(_EMPTY_RESPONSE));
+      }
+
+      Logger.error("Fetch compliance fail", e);
+      return Promise.promise(() -> internalServerError(errorResponse(e)));
+    }
+
+    if (record == null) {
+      return Promise.promise(() -> notFound(_EMPTY_RESPONSE));
+    }
+    return Promise.promise(() -> ok(Json.newObject().set("complianceInfo", Json.toJson(record))));
+  }
+
+  public static Promise<Result> updateDatasetRetention(@Nonnull String datasetUrn) {
+    final String username = session("user");
+    if (StringUtils.isBlank(username)) {
+      return Promise.promise(() -> unauthorized(_EMPTY_RESPONSE));
+    }
+
+    try {
+      DatasetRetention record = Json.mapper().convertValue(request().body().asJson(), DatasetRetention.class);
+
+      if (record.getDatasetUrn() == null || !record.getDatasetUrn().equals(datasetUrn)) {
+        throw new IllegalArgumentException("Dataset Urn not exist or doesn't match.");
+      }
+
+      COMPLIANCE_DAO.updateDatasetRetention(record, username);
+    } catch (Exception e) {
+      Logger.error("Update Retention Policy fail", e);
+      return Promise.promise(() -> internalServerError(errorResponse(e)));
+    }
+    return Promise.promise(() -> ok(_EMPTY_RESPONSE));
+  }
+
   public static Promise<Result> getDatasetAcls(@Nonnull String datasetUrn) {
     final List<AccessControlEntry> acls;
     try {

wherehows-frontend/conf/routes

@@ -306,6 +306,10 @@ GET           /api/v2/datasets/:urn/compliance
 
 POST          /api/v2/datasets/:urn/compliance                                  controllers.api.v2.Dataset.updateDatasetCompliance(urn: String)
 
+GET           /api/v2/datasets/:urn/retention                                   controllers.api.v2.Dataset.getDatasetRetention(urn: String)
+
+POST          /api/v2/datasets/:urn/retention                                   controllers.api.v2.Dataset.updateDatasetRetention(urn: String)
+
 GET           /api/v2/datasets/:urn/compliance/suggestion                       controllers.api.v2.Dataset.getDatasetSuggestedCompliance(urn: String)
 
 POST          /api/v2/datasets/:urn/compliance/suggestion/feedback              controllers.api.v2.Dataset.sendDatasetSuggestedComplianceFeedback(urn: String)