Merge pull request #1181 from theseyi/c-json-upload

updates schema for json metadata. initial refactor of function to valdate metadata schema properties. refactors action handler for upload: moves to container component

wherehows-web/app/components/dataset-compliance.ts

@@ -35,7 +35,6 @@ import {
   singleTagsInChangeSet,
   tagsForIdentifierField
 } from 'wherehows-web/constants';
-import { isPolicyExpectedShape } from 'wherehows-web/utils/datasets/compliance-policy';
 import { getTagsSuggestions } from 'wherehows-web/utils/datasets/compliance-suggestions';
 import { arrayMap, compact, isListUnique, iterateArrayAsync } from 'wherehows-web/utils/array';
 import noop from 'wherehows-web/utils/noop';
@@ -73,8 +72,6 @@ const {
   complianceFieldNotUnique,
   missingTypes,
   helpText,
-  successUploading,
-  invalidPolicyData,
   missingPurgePolicy,
   missingDatasetSecurityClassification
 } = compliancePolicyStrings;
@@ -127,6 +124,9 @@ export default class DatasetCompliance extends Component {
   schemaFieldNamesMappedToDataTypes: Array<Pick<IDatasetColumn, 'dataType' | 'fieldName'>>;
   onReset: <T>() => Promise<T>;
   onSave: <T>() => Promise<T>;
+
+  onComplianceUpload: (jsonString: string) => void;
+
   notifyOnChangeSetSuggestions: (hasSuggestions: boolean) => void;
   notifyOnChangeSetRequiresReview: (hasChangeSetDrift: boolean) => void;
 
@@ -815,9 +815,9 @@ export default class DatasetCompliance extends Component {
    * checked in the function. If criteria is not met, an the returned promise is settled
    * in a rejected state, otherwise fulfilled
    * @method
-   * @return {Promise<never> | void}
+   * @return {Promise<never | void>}
    */
-  validateFields(this: DatasetCompliance): Promise<never> | void {
+  async validateFields(this: DatasetCompliance): Promise<never | void> {
     const { notify } = get(this, 'notifications');
     const { complianceEntities = [] } = get(this, 'complianceInfo') || {};
     const idTypeComplianceEntities = complianceEntities.filter(isTagIdType(get(this, 'complianceDataTypes')));
@@ -1161,47 +1161,6 @@ export default class DatasetCompliance extends Component {
       set(tag, 'suggestionAuthority', intent);
     },
 
-    /**
-     * Receives the json representation for compliance and applies each key to the policy
-     * @param {string} textString string representation for the JSON file
-     */
-    onComplianceJsonUpload(this: DatasetCompliance, textString: string): void {
-      const complianceInfo = get(this, 'complianceInfo');
-      const { notify } = get(this, 'notifications');
-      let policy;
-
-      if (!complianceInfo) {
-        notify(NotificationEvent.error, {
-          content: 'Could not find compliance current compliance policy for this dataset'
-        });
-
-        return;
-      }
-
-      try {
-        policy = JSON.parse(textString);
-      } catch (e) {
-        notify(NotificationEvent.error, {
-          content: invalidPolicyData
-        });
-      }
-
-      if (isPolicyExpectedShape(policy)) {
-        setProperties(complianceInfo, {
-          complianceEntities: policy.complianceEntities,
-          datasetClassification: policy.datasetClassification
-        });
-
-        notify(NotificationEvent.info, {
-          content: successUploading
-        });
-      }
-
-      notify(NotificationEvent.error, {
-        content: invalidPolicyData
-      });
-    },
-
     /**
      * Handles the compliance policy download action
      */
@@ -1237,6 +1196,14 @@ export default class DatasetCompliance extends Component {
       anchor.click();
     },
 
+    /**
+     * Receives the json representation for compliance and applies each key to the policy
+     * @param {string} jsonString string representation for the JSON file
+     */
+    onComplianceJsonUpload(this: DatasetCompliance, jsonString: string): void {
+      get(this, 'onComplianceUpload')(jsonString);
+    },
+
     /**
      * Updates the source object representing the current datasetClassification map
      * @param {keyof typeof DatasetClassifiers} classifier the property on the datasetClassification to update

wherehows-web/app/components/datasets/containers/dataset-compliance.ts

@@ -1,5 +1,5 @@
 import Component from '@ember/component';
-import { get, set, setProperties } from '@ember/object';
+import { get, set, setProperties, getProperties } from '@ember/object';
 import ComputedProperty from '@ember/object/computed';
 import { inject } from '@ember/service';
 import { task } from 'ember-concurrency';
@@ -28,6 +28,10 @@ import {
   SuggestionIntent
 } from 'wherehows-web/constants';
 import { iterateArrayAsync } from 'wherehows-web/utils/array';
+import validateMetadataObject, {
+  datasetComplianceMetadataTaxonomy
+} from 'wherehows-web/utils/datasets/compliance/metadata-schema';
+import { notificationDialogActionFactory } from 'wherehows-web/utils/notifications/notifications';
 
 /**
  * Type alias for the response when container data items are batched
@@ -52,7 +56,7 @@ type BatchContainerDataResult = Pick<
   | 'schemaless'
 >;
 
-const { successUpdating, failedUpdating } = compliancePolicyStrings;
+const { successUpdating, failedUpdating, successUploading, invalidPolicyData } = compliancePolicyStrings;
 
 export default class DatasetComplianceContainer extends Component {
   /**
@@ -336,4 +340,43 @@ export default class DatasetComplianceContainer extends Component {
   onSuggestionsComplianceFeedback(uid: string | null = null, feedback: SuggestionIntent) {
     saveDatasetComplianceSuggestionFeedbackByUrn(get(this, 'urn'), uid, feedback);
   }
+
+  /**
+   * Reapplies the uploaded compliance policy to the container property
+   * @param {string} jsonString string representation for the JSON file
+   * @memberof DatasetComplianceContainer
+   */
+  @action
+  onComplianceUpload(this: DatasetComplianceContainer, jsonString: string): void {
+    const {
+      complianceInfo,
+      notifications: { notify }
+    } = getProperties(this, ['complianceInfo', 'notifications']);
+
+    if (complianceInfo) {
+      try {
+        const policy = JSON.parse(jsonString);
+
+        if (validateMetadataObject(policy, datasetComplianceMetadataTaxonomy)) {
+          const { complianceEntities, datasetClassification } = policy;
+          const resolvedComplianceInfo = { ...complianceInfo, complianceEntities, datasetClassification };
+          const { dialogActions } = notificationDialogActionFactory();
+
+          set(this, 'complianceInfo', resolvedComplianceInfo);
+
+          notify(NotificationEvent.confirm, {
+            header: 'Successfully applied uploaded metadata',
+            content: successUploading,
+            dialogActions,
+            dismissButtonText: false,
+            confirmButtonText: 'Dismiss'
+          });
+        }
+      } catch (e) {
+        notify(NotificationEvent.error, {
+          content: invalidPolicyData
+        });
+      }
+    }
+  }
 }

wherehows-web/app/constants/dataset-compliance.ts

@@ -35,7 +35,7 @@ const compliancePolicyStrings = {
   missingTypes: 'Looks like you may have forgotten to specify a `Field Format` for all ID fields?',
   successUpdating: 'Changes have been successfully saved!',
   failedUpdating: 'An error occurred while saving.',
-  successUploading: 'Metadata successfully updated! Please "Save" when ready.',
+  successUploading: 'Metadata successfully updated! Please confirm and "Save" when ready.',
   invalidPolicyData: 'Received policy in an unexpected format! Please check the provided attributes and try again.',
   helpText: {
     classification:
@@ -107,7 +107,7 @@ const getComplianceSteps = (hasSchema: boolean = true): { [x: number]: { name: s
  * @param {IComplianceEntity} { readonly }
  * @returns {boolean}
  */
-const isEditableComplianceEntity = ({ readonly }: IComplianceEntity): boolean => readonly !== true;
+const isEditableComplianceEntity = ({ readonly }: IComplianceEntity): boolean => readonly !== true; // do not simplify, readonly may be undefined
 
 /**
  * Filters out from a list of compliance entities, entities that are editable
@@ -299,26 +299,26 @@ const idTypeFieldsHaveLogicalType = arrayEvery(idTypeFieldHasLogicalType);
 
 /**
  * Describes the function interface for tagsForIdentifierField
- * @interface TagsForIdentifierFieldFn
+ * @interface ITagsForIdentifierFieldFn
  */
-interface TagsForIdentifierFieldFn {
+interface ITagsForIdentifierFieldFn {
   (identifierField: string): (tags: Array<IComplianceChangeSet>) => Array<IComplianceChangeSet>;
 }
 /**
  * Gets the tags for a specific identifier field
  * @param {string} identifierField
- * @return {TagsForIdentifierFieldFn}
+ * @return {ITagsForIdentifierFieldFn}
  */
-const tagsForIdentifierField: TagsForIdentifierFieldFn = (identifierField: string) =>
+const tagsForIdentifierField: ITagsForIdentifierFieldFn = (identifierField: string) =>
   arrayFilter(isSchemaFieldTag<IComplianceChangeSet>(identifierField));
 
 /**
  * Lists tags that occur for only one identifier type in the list of tags
  * @param {Array<IComplianceChangeSet>} tags the full list of tags to iterate through
- * @param {TagsForIdentifierFieldFn} tagsForIdentifierFieldFn
+ * @param {ITagsForIdentifierFieldFn} tagsForIdentifierFieldFn
  * @return {(singleTags: Array<IComplianceChangeSet>, { identifierField }: IComplianceChangeSet) => (any)[] | Array<IComplianceChangeSet>}
  */
-const singleTagsIn = (tags: Array<IComplianceChangeSet>, tagsForIdentifierFieldFn: TagsForIdentifierFieldFn) => (
+const singleTagsIn = (tags: Array<IComplianceChangeSet>, tagsForIdentifierFieldFn: ITagsForIdentifierFieldFn) => (
   singleTags: Array<IComplianceChangeSet>,
   { identifierField }: IComplianceChangeSet
 ): Array<IComplianceChangeSet> => {
@@ -329,12 +329,12 @@ const singleTagsIn = (tags: Array<IComplianceChangeSet>, tagsForIdentifierFieldF
 /**
  * Lists the tags in a list of tags that occur for only one identifier type
  * @param {Array<IComplianceChangeSet>} tags
- * @param {TagsForIdentifierFieldFn} tagsForIdentifierFieldFn
+ * @param {ITagsForIdentifierFieldFn} tagsForIdentifierFieldFn
  * @return {Array<IComplianceChangeSet>}
  */
 const singleTagsInChangeSet = (
   tags: Array<IComplianceChangeSet>,
-  tagsForIdentifierFieldFn: TagsForIdentifierFieldFn
+  tagsForIdentifierFieldFn: ITagsForIdentifierFieldFn
 ): Array<IComplianceChangeSet> => arrayReduce(singleTagsIn(tags, tagsForIdentifierFieldFn), [])(tags);
 
 /**
@@ -415,21 +415,18 @@ const foldComplianceChangeSets = async (
 /**
  * Builds a default shape for securitySpecification & privacyCompliancePolicy with default / unset values
  *   for non null properties as per Avro schema
- * @param {string} datasetId identifier for the dataset that this privacy object applies to
+ * @param {string} datasetUrn identifier for the dataset that this privacy object applies to
+ * @return {IComplianceInfo}
  */
-const createInitialComplianceInfo = (datasetId: string): IComplianceInfo => {
-  const identifier = typeof datasetId === 'string' ? { datasetUrn: decodeUrn(datasetId) } : { datasetId };
-
-  return {
-    ...identifier,
-    datasetId: null,
-    confidentiality: null,
-    complianceType: '',
-    compliancePurgeNote: '',
-    complianceEntities: [],
-    datasetClassification: null
-  };
-};
+const initialComplianceObjectFactory = (datasetUrn: string): IComplianceInfo => ({
+  datasetUrn: decodeUrn(datasetUrn),
+  datasetId: null,
+  confidentiality: null,
+  complianceType: '',
+  compliancePurgeNote: '',
+  complianceEntities: [],
+  datasetClassification: null
+});
 
 /**
  * Maps the fields found in the column property on the schema api to the values returned in the current privacy policy
@@ -581,7 +578,7 @@ export {
   tagsHaveNoneType,
   fieldTagsRequiringReview,
   tagsHaveNoneAndNotNoneType,
-  createInitialComplianceInfo,
+  initialComplianceObjectFactory,
   getIdTypeDataTypes,
   fieldTagsHaveIdentifierType,
   idTypeFieldHasLogicalType,

wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs

@@ -33,6 +33,7 @@
       notifyOnChangeSetRequiresReview=(action "onCompliancePolicyChangeSetDrift")
       onSave=(action "savePrivacyCompliancePolicy")
       onReset=(action "resetPrivacyCompliancePolicy")
+      onComplianceUpload=(action "onComplianceUpload")
     }}
 
   {{/if}}

wherehows-web/app/typings/generic.d.ts

@@ -10,4 +10,12 @@ type StringUnionKeyToValue<U extends string> = { [K in U]: K };
  */
 type StringEnumKeyToEnumValue<T extends string, V> = { [K in T]: V };
 
-export { StringUnionKeyToValue, StringEnumKeyToEnumValue };
+/**
+ * Describes the index signature for a generic object
+ * @interface IObject
+ */
+interface IObject<T> {
+  [K: string]: T;
+}
+
+export { StringUnionKeyToValue, StringEnumKeyToEnumValue, IObject };

wherehows-web/app/utils/api/datasets/compliance.ts

@@ -1,4 +1,4 @@
-import { createInitialComplianceInfo } from 'wherehows-web/constants';
+import { initialComplianceObjectFactory } from 'wherehows-web/constants';
 import { SuggestionIntent } from 'wherehows-web/constants/datasets/compliance';
 import { notFoundApiError } from 'wherehows-web/utils/api';
 import { datasetUrlById, datasetUrlByUrn } from 'wherehows-web/utils/api/datasets/shared';
@@ -62,7 +62,7 @@ export interface IReadComplianceResult {
  * @return {Promise<IReadComplianceResult>}
  */
 const readDatasetComplianceByUrn = async (urn: string): Promise<IReadComplianceResult> => {
-  let complianceInfo: IComplianceGetResponse['complianceInfo'] = createInitialComplianceInfo(urn);
+  let complianceInfo: IComplianceGetResponse['complianceInfo'] = initialComplianceObjectFactory(urn);
   let isNewComplianceInfo = false;
 
   try {
@@ -71,7 +71,7 @@ const readDatasetComplianceByUrn = async (urn: string): Promise<IReadComplianceR
     }));
   } catch (e) {
     if (notFoundApiError(e)) {
-      complianceInfo = createInitialComplianceInfo(urn);
+      complianceInfo = initialComplianceObjectFactory(urn);
       isNewComplianceInfo = true;
     } else {
       throw e;

wherehows-web/app/utils/datasets/compliance/metadata-schema.ts

@@ -0,0 +1,116 @@
+import { typeOf } from '@ember/utils';
+import { DatasetClassifiers } from 'wherehows-web/constants';
+import { arrayEvery, arrayReduce } from 'wherehows-web/utils/array';
+import { Classification } from 'wherehows-web/constants/datasets/compliance';
+import { IObject } from 'wherehows-web/typings/generic';
+
+/**
+ * Describes the interface for schemas that are used for compliance metadata objects
+ * @interface IMetadataTaxonomy
+ */
+interface IMetadataTaxonomy {
+  readonly '@props': Array<string>;
+  readonly '@type': string;
+
+  readonly [K: string]: IMetadataTaxonomy | any;
+}
+
+/**
+ * Defines the shape of the dataset compliance metadata json object using the IMetadataTaxonomy interface
+ * @type {({'@type': string; '@props': string[]} | {'@type': string; '@props': string[]; securityClassification: {'@type': string; '@props': any[]}})[]}
+ */
+const datasetComplianceMetadataTaxonomy: Array<IMetadataTaxonomy> = [
+  {
+    '@type': 'datasetClassification:object',
+    '@props': Object.keys(DatasetClassifiers).map(key => `${key}:boolean`)
+  },
+  {
+    '@type': 'complianceEntities:array',
+    '@props': [
+      'identifierField:string',
+      'identifierType:string|null',
+      'securityClassification:string|null',
+      'logicalType:string|null',
+      'nonOwner:boolean|null',
+      'valuePattern:string|null'
+    ],
+    securityClassification: {
+      '@type': 'securityClassification:string',
+      '@props': Object.values(Classification)
+    }
+  }
+];
+
+/**
+ * Checks that a value type matches an expected pattern string
+ * @param {*} value the value to check
+ * @param {string} expectedTypePattern the pattern string to match against
+ * @returns {boolean}
+ */
+const valueEquiv = (value: any, expectedTypePattern: string): boolean => expectedTypePattern.includes(typeOf(value));
+
+/**
+ * Extracts the type key and the pattern string from the string mapping into a tuple pair
+ * @param {string} objectKeyTypePattern string value consisting of a pair of key/property name and allowed types separated by a colon ":"
+ * @returns {[string, string]}
+ */
+const typePatternMap = (objectKeyTypePattern: string): [string, string] =>
+  <[string, string]>objectKeyTypePattern.split(':');
+
+/**
+ * Returns a iteratee bound to an object that checks that a key matches the expected value in the typeMap
+ * @param {IObject<any>} object the object with keys to check
+ * @return {(typeMap: string) => boolean}
+ */
+const keyValueHasMatch = (object: IObject<any>) => (typeMap: string) => {
+  const [key, typeString] = typePatternMap(typeMap);
+  return valueEquiv(object[key], typeString);
+};
+
+/**
+ * Checks each key on an object matches the expected types in the typeMap
+ * @param {IObject<any>} object the object with keys to check
+ * @param {Array<string>} typeMaps the colon delimited type string
+ * @returns {boolean}
+ */
+const keysEquiv = (object: IObject<any>, typeMaps: Array<string>): boolean =>
+  arrayEvery(keyValueHasMatch(object))(typeMaps);
+
+/**
+ * Checks that a compliance metadata object has a schema that matches the taxonomy / schema provided
+ * @param {IObject<any>} object an instance of a compliance metadata object
+ * @param {Array<IMetadataTaxonomy>} taxonomy schema shape to check against
+ * @return {boolean}
+ */
+const validateMetadataObject = (object: IObject<any>, taxonomy: Array<IMetadataTaxonomy>): boolean => {
+  const rootTypeMaps = taxonomy.map(category => category['@type']);
+  let isValid = keysEquiv(object, rootTypeMaps);
+
+  if (isValid) {
+    const downlevelAccumulator = (validity: boolean, typeMap: string): boolean => {
+      const [key, pattern]: [string, string] = typePatternMap(typeMap);
+
+      if (pattern.includes('object')) {
+        validity = keysEquiv(object[key], taxonomy.findBy('@type', typeMap)!['@props']);
+      }
+
+      if (pattern.includes('array') && Array.isArray(object[key])) {
+        validity = arrayReduce(
+          (validity: boolean, value: IObject<string>) =>
+            validity && keysEquiv(value, taxonomy.findBy('@type', typeMap)!['@props']),
+          validity
+        )(object[key]);
+      }
+
+      return validity;
+    };
+
+    return arrayReduce(downlevelAccumulator, isValid)(rootTypeMaps);
+  }
+
+  return isValid;
+};
+
+export default validateMetadataObject;
+
+export { datasetComplianceMetadataTaxonomy };

wherehows-web/tests/integration/components/datasets/containers/dataset-compliance-test.js

@@ -2,7 +2,7 @@ import { moduleForComponent, test } from 'ember-qunit';
 import hbs from 'htmlbars-inline-precompile';
 import { waitUntil, find } from 'ember-native-dom-helpers';
 import { urn } from 'wherehows-web/mirage/fixtures/urn';
-import { createInitialComplianceInfo } from 'wherehows-web/constants';
+import { initialComplianceObjectFactory } from 'wherehows-web/constants';
 import sinon from 'sinon';
 
 moduleForComponent(
@@ -27,7 +27,7 @@ test('it renders', async function(assert) {
     200,
     { 'Content-Type': 'application/json' },
     JSON.stringify({
-      complianceInfo: createInitialComplianceInfo(urn)
+      complianceInfo: initialComplianceObjectFactory(urn)
     })
   ]);
   this.server.respondWith(/.*\/compliance-data-types/, [

wherehows-web/tests/unit/constants/dataset-compliance-test.js

@@ -6,20 +6,22 @@ import {
   getFieldIdentifierOptions,
   isAutoGeneratedPolicy,
   PurgePolicy,
-  createInitialComplianceInfo,
+  initialComplianceObjectFactory,
   isRecentSuggestion,
   tagNeedsReview
 } from 'wherehows-web/constants';
 import complianceDataTypes from 'wherehows-web/mirage/fixtures/compliance-data-types';
 import { mockTimeStamps } from 'wherehows-web/tests/helpers/datasets/compliance-policy/recent-suggestions-constants';
 import { mockFieldChangeSets } from 'wherehows-web/tests/helpers/datasets/compliance-policy/field-changeset-constants';
+import { hdfsUrn } from 'wherehows-web/mirage/fixtures/urn';
 
 module('Unit | Constants | dataset compliance');
 
-test('createInitialComplianceInfo', function(assert) {
+test('initialComplianceObjectFactory', function(assert) {
   assert.expect(2);
-  const mockId = 1337;
+  const mockUrn = hdfsUrn;
   const initialComplianceInfo = {
+    datasetUrn: mockUrn,
     datasetId: null,
     complianceType: '',
     compliancePurgeNote: '',
@@ -28,8 +30,12 @@ test('createInitialComplianceInfo', function(assert) {
     confidentiality: null
   };
 
-  assert.ok(typeof createInitialComplianceInfo === 'function', 'createInitialComplianceInfo is a function');
-  assert.deepEqual(createInitialComplianceInfo(mockId), initialComplianceInfo, 'generates policy in expected shape');
+  assert.ok(typeof initialComplianceObjectFactory === 'function', 'initialComplianceObjectFactory is a function');
+  assert.deepEqual(
+    initialComplianceObjectFactory(mockUrn),
+    initialComplianceInfo,
+    'generates policy in expected shape'
+  );
 });
 
 test('isRecentSuggestion exists', function(assert) {