Merge pull request #771 from theseyi/compliance-reviewable

adds reviewable filter option. updates filter logic to create diff between fields on policy and column field values. refactors implementation for client working copy / change-set: reduces number of steps. modifies compliance entity table styles

wherehows-web/app/components/dataset-compliance-row.js

@@ -1,36 +1,19 @@
 import Ember from 'ember';
 import DatasetTableRow from 'wherehows-web/components/dataset-table-row';
 import {
-  fieldIdentifierTypes,
+  fieldIdentifierTypeValues,
+  fieldIdentifierTypeIds,
   defaultFieldDataTypeClassification,
   isMixedId,
   isCustomId,
   hasPredefinedFieldFormat,
   logicalTypesForIds,
-  logicalTypesForGeneric
+  logicalTypesForGeneric,
+  SuggestionIntent
 } from 'wherehows-web/constants';
-import { fieldIdentifierTypeIds } from 'wherehows-web/components/dataset-compliance';
+import { fieldChangeSetRequiresReview } from 'wherehows-web/utils/datasets/compliance-policy';
 
-const { computed, get, getProperties, set } = Ember;
+const { computed, get, getProperties } = Ember;
-/**
- * String indicating that the user affirms a suggestion
- * @type {string}
- */
-const acceptIntent = 'accept';
-
-/**
- * String indicating that the user ignored a suggestion
- * @type {string}
- */
-const ignoreIntent = 'ignore';
-
-/**
- * Caches a list of fieldIdentifierTypes values
- * @type {Array<string>}
- */
-const fieldIdentifierTypeValues = Object.keys(fieldIdentifierTypes)
-  .map(fieldIdentifierType => fieldIdentifierTypes[fieldIdentifierType])
-  .mapBy('value');
 
 /**
  * Extracts the suggestions for identifierType, logicalType suggestions, and confidence from a list of predictions
@@ -77,23 +60,23 @@ export default DatasetTableRow.extend({
   suggestionAuthority: computed.alias('field.suggestionAuthority'),
 
   /**
-   * Checks that the field does not have a recently input value
+   * Checks that the field does not have a current policy value
    * @type {Ember.computed}
    * @return {boolean}
    */
-  isNewField: computed('isNewComplianceInfo', 'isModified', function() {
+  isReviewRequested: computed('field.{isDirty,suggestion,privacyPolicyExists,suggestionAuthority}', function() {
-    const { isNewComplianceInfo, isModified } = getProperties(this, ['isNewComplianceInfo', 'isModified']);
+    return fieldChangeSetRequiresReview(get(this, 'field'));
-    return isNewComplianceInfo && !isModified;
   }),
 
   /**
    * Maps the suggestion response to a string resolution
    * @type {Ember.computed}
+   * @return {string|void}
    */
   suggestionResolution: computed('field.suggestionAuthority', function() {
     return {
-      [acceptIntent]: 'Accepted',
+      [SuggestionIntent.accept]: 'Accepted',
-      [ignoreIntent]: 'Discarded'
+      [SuggestionIntent.ignore]: 'Discarded'
     }[get(this, 'field.suggestionAuthority')];
   }),
 
@@ -204,7 +187,6 @@ export default DatasetTableRow.extend({
       const { onFieldIdentifierTypeChange } = this.attrs;
       if (typeof onFieldIdentifierTypeChange === 'function') {
         onFieldIdentifierTypeChange(get(this, 'field'), { value });
-        set(this, 'isModified', true);
       }
     },
 
@@ -217,7 +199,6 @@ export default DatasetTableRow.extend({
       const { onFieldLogicalTypeChange } = this.attrs;
       if (typeof onFieldLogicalTypeChange === 'function') {
         onFieldLogicalTypeChange(get(this, 'field'), { value });
-        set(this, 'isModified', true);
       }
     },
 
@@ -229,7 +210,6 @@ export default DatasetTableRow.extend({
       const { onFieldClassificationChange } = this.attrs;
       if (typeof onFieldClassificationChange === 'function') {
         onFieldClassificationChange(get(this, 'field'), { value });
-        set(this, 'isModified', true);
       }
     },
 
@@ -242,7 +222,7 @@ export default DatasetTableRow.extend({
       const { onSuggestionIntent } = this.attrs;
 
       // Accept the suggestion for either identifierType and/or logicalType
-      if (intent === acceptIntent) {
+      if (intent === SuggestionIntent.accept) {
         const { identifierType, logicalType } = get(this, 'prediction');
         if (identifierType) {
           this.actions.onFieldIdentifierTypeChange.call(this, { value: identifierType });

wherehows-web/app/components/dataset-compliance.js

@@ -4,6 +4,7 @@ import {
   classifiers,
   datasetClassifiers,
   fieldIdentifierTypes,
+  fieldIdentifierTypeIds,
   idLogicalTypes,
   nonIdFieldLogicalTypes,
   defaultFieldDataTypeClassification,
@@ -13,8 +14,14 @@ import {
   hasPredefinedFieldFormat,
   getDefaultLogicalType
 } from 'wherehows-web/constants';
-import { isPolicyExpectedShape } from 'wherehows-web/utils/datasets/functions';
+import {
+  isPolicyExpectedShape,
+  fieldChangeSetRequiresReview,
+  mergeMappedColumnFieldsWithSuggestions
+} from 'wherehows-web/utils/datasets/compliance-policy';
 import scrollMonitor from 'scrollmonitor';
+import { hasEnumerableKeys } from 'wherehows-web/utils/object';
+import { arrayFilter, isListUnique } from 'wherehows-web/utils/array';
 
 const {
   assert,
@@ -75,17 +82,17 @@ const datasetClassifiersKeys = Object.keys(datasetClassifiers);
  * @type {string}
  */
 const policyComplianceEntitiesKey = 'complianceInfo.complianceEntities';
-/**
- * Duplicate check using every to short-circuit iteration
- * @param {Array} list = [] the list to check for dupes
- * @return {Boolean} true is unique, false otherwise
- */
-const listIsUnique = (list = []) => new Set(list).size === list.length;
 
 assert('`fieldIdentifierTypes` contains an object with a key `none`', typeof fieldIdentifierTypes.none === 'object');
 const fieldIdentifierTypeKeysBarNone = Object.keys(fieldIdentifierTypes).filter(k => k !== 'none');
 const fieldDisplayKeys = ['none', '_', ...fieldIdentifierTypeKeysBarNone];
 
+/**
+ * Returns a list of changeSet fields that requires user attention
+ * @type {function({}): Array<{ isDirty, suggestion, privacyPolicyExists, suggestionAuthority }>}
+ */
+const changeSetFieldsRequiringReview = arrayFilter(fieldChangeSetRequiresReview);
+
 /**
  * A list of field identifier types mapped to label, value options for select display
  * @type {any[]|Array.<{value: String, label: String}>}
@@ -103,15 +110,6 @@ const fieldIdentifierOptions = fieldDisplayKeys.map(fieldIdentifierType => {
   };
 });
 
-/**
- * A list of field identifier types that are Ids i.e member ID, org ID, group ID
- * @type {any[]|Array.<String>}
- */
-export const fieldIdentifierTypeIds = Object.keys(fieldIdentifierTypes)
-  .map(fieldIdentifierType => fieldIdentifierTypes[fieldIdentifierType])
-  .filter(({ isId }) => isId)
-  .mapBy('value');
-
 export default Component.extend({
   sortColumnWithName: 'identifierField',
   filterBy: 'identifierField',
@@ -142,6 +140,7 @@ export default Component.extend({
    * @return {boolean}
    */
   isReadOnly: computed.not('isEditing'),
+
   /**
    * Flag indicating that the component is currently saving / attempting to save the privacy policy
    * @type {String}
@@ -183,6 +182,25 @@ export default Component.extend({
     }));
   }),
 
+  /**
+   * Returns a list of ui values and labels for review filter drop-down
+   * @type {Ember.computed}
+   */
+  fieldReviewOptions: computed(function() {
+    return [
+      { value: 'showAll', label: 'Showing all fields' },
+      {
+        value: 'showReview',
+        label: 'Showing only fields to review'
+      }
+    ];
+  }),
+
+  /**
+   * @type {string}
+   */
+  fieldReviewOption: 'showAll',
+
   /**
    * Reference to the application notifications Service
    * @type {Ember.Service}
@@ -272,7 +290,7 @@ export default Component.extend({
     const fieldNames = getWithDefault(this, 'schemaFieldNamesMappedToDataTypes', []).mapBy('fieldName');
 
     // identifier field names from the column api should be unique
-    if (listIsUnique(fieldNames.sort())) {
+    if (isListUnique(fieldNames.sort())) {
       return set(this, '_hasBadData', false);
     }
 
@@ -400,48 +418,13 @@ export default Component.extend({
     }, []);
   }),
 
-  /**
-   * Lists all dataset fields found in the `columns` performs an intersection
-   * of fields with the currently persisted and/or updated
-   * privacyCompliancePolicy.complianceEntities.
-   * The returned list is a map of fields with current or default privacy properties
-   */
-  mergeComplianceEntitiesAndColumnFields(
-    columnIdFieldsToCurrentPrivacyPolicy = {},
-    truncatedColumnFields = [],
-    identifierFieldMappedToSuggestions = {}
-  ) {
-    return truncatedColumnFields.map(({ fieldName: identifierField, dataType }) => {
-      const {
-        [identifierField]: { identifierType, logicalType, securityClassification }
-      } = columnIdFieldsToCurrentPrivacyPolicy;
-
-      //Cache the mapped suggestion into a local
-      const suggestion = identifierFieldMappedToSuggestions[identifierField];
-      let field = {
-        identifierField,
-        dataType,
-        identifierType,
-        logicalType,
-        classification: securityClassification
-      };
-
-      // If a suggestion exists for this field add the suggestion attribute to the field properties
-      if (suggestion) {
-        field = { ...field, suggestion };
-      }
-
-      return field;
-    });
-  },
-
   /**
    *
-   * @param {Array} columnFieldNames
+   * @param {Array<object>} columnFieldProps
-   * @return {*|{}|any}
+   * @param {Array<object>} complianceEntities
+   * @return {object}
    */
-  mapColumnIdFieldsToCurrentPrivacyPolicy(columnFieldNames) {
+  mapColumnIdFieldsToCurrentPrivacyPolicy(columnFieldProps, complianceEntities) {
-    const complianceEntities = get(this, policyComplianceEntitiesKey) || [];
     const getKeysOnField = (keys = [], fieldName, source = []) => {
       const sourceField = source.find(({ identifierField }) => identifierField === fieldName) || {};
       let ret = {};
@@ -454,14 +437,23 @@ export default Component.extend({
       return ret;
     };
 
-    return columnFieldNames.reduce((acc, identifierField) => {
+    return columnFieldProps.reduce((acc, { identifierField, dataType }) => {
       const currentPrivacyAttrs = getKeysOnField(
         ['identifierType', 'logicalType', 'securityClassification'],
         identifierField,
         complianceEntities
       );
 
-      return { ...acc, ...{ [identifierField]: currentPrivacyAttrs } };
+      return {
+        ...acc,
+        [identifierField]: {
+          identifierField,
+          dataType,
+          ...currentPrivacyAttrs,
+          privacyPolicyExists: hasEnumerableKeys(currentPrivacyAttrs),
+          isDirty: false
+        }
+      };
     }, {});
   },
 
@@ -473,29 +465,58 @@ export default Component.extend({
     'truncatedColumnFields',
     `${policyComplianceEntitiesKey}.[]`,
     function() {
-      const columnFieldNames = get(this, 'truncatedColumnFields').map(({ fieldName }) => fieldName);
+      // Truncated list of Dataset field names and data types currently returned from the column endpoint
-      return this.mapColumnIdFieldsToCurrentPrivacyPolicy(columnFieldNames);
+      const columnFieldProps = get(this, 'truncatedColumnFields').map(({ fieldName, dataType }) => ({
+        identifierField: fieldName,
+        dataType
+      }));
+      // Dataset fields that currently have a compliance policy
+      const currentComplianceEntities = get(this, policyComplianceEntitiesKey) || [];
+
+      return this.mapColumnIdFieldsToCurrentPrivacyPolicy(columnFieldProps, currentComplianceEntities);
     }
   ),
 
   /**
    * Caches a reference to the generated list of merged data between the column api and the current compliance entities list
-   * @type {Ember.computed}
+   * @type {Array<{identifierType: string, logicalType: string, securityClassification: string, privacyPolicyExists: boolean, isDirty: boolean, [suggestion]: object}>}
    */
-  mergedComplianceEntitiesAndColumnFields: computed('columnIdFieldsToCurrentPrivacyPolicy', function() {
+  compliancePolicyChangeSet: computed('columnIdFieldsToCurrentPrivacyPolicy', function() {
     // truncatedColumnFields is a dependency for cp columnIdFieldsToCurrentPrivacyPolicy, so no need to dep on that directly
-    return this.mergeComplianceEntitiesAndColumnFields(
+    return mergeMappedColumnFieldsWithSuggestions(
       get(this, 'columnIdFieldsToCurrentPrivacyPolicy'),
-      get(this, 'truncatedColumnFields'),
       get(this, 'identifierFieldToSuggestion')
     );
   }),
 
+  /**
+   * Returns a list of changeSet fields that meets the user selected filter criteria
+   * @type {Ember.computed}
+   * @return {Array<{}>}
+   */
+  filteredChangeSet: computed('changeSetReviewCount', 'fieldReviewOption', 'compliancePolicyChangeSet', function() {
+    const changeSet = get(this, 'compliancePolicyChangeSet');
+
+    return get(this, 'fieldReviewOption') === 'showReview' ? changeSetFieldsRequiringReview(changeSet) : changeSet;
+  }),
+
+  /**
+   * Returns a count of changeSet fields that require user attention
+   * @type {Ember.computed}
+   * @return {Array<{}>}
+   */
+  changeSetReviewCount: computed(
+    'compliancePolicyChangeSet.@each.{isDirty,suggestion,privacyPolicyExists,suggestionAuthority}',
+    function() {
+      return changeSetFieldsRequiringReview(get(this, 'compliancePolicyChangeSet')).length;
+    }
+  ),
+
   /**
    * Creates a mapping of compliance suggestions to identifierField
    * This improves performance in a subsequent merge op since this loop
    * happens only once and is cached
-   * @type {Ember.computed}
+   * @type {object}
    */
   identifierFieldToSuggestion: computed('complianceSuggestion', function() {
     const identifierFieldToSuggestion = {};
@@ -603,7 +624,7 @@ export default Component.extend({
     // All candidate fields that can be on policy, excluding tracking type fields
     const datasetFields = get(
       this,
-      'mergedComplianceEntitiesAndColumnFields'
+      'compliancePolicyChangeSet'
     ).map(({ identifierField, identifierType, logicalType, classification }) => ({
       identifierField,
       identifierType,
@@ -681,7 +702,7 @@ export default Component.extend({
       complianceEntities.filter(({ identifierType }) => fieldIdentifierTypeIds.includes(identifierType)),
       [...genericLogicalTypes, ...idLogicalTypes]
     );
-    const fieldIdentifiersAreUnique = listIsUnique(complianceEntities.mapBy('identifierField'));
+    const fieldIdentifiersAreUnique = isListUnique(complianceEntities.mapBy('identifierField'));
     const schemaFieldLengthGreaterThanComplianceEntities = this.isSchemaFieldLengthGreaterThanComplianceEntities();
 
     if (!fieldIdentifiersAreUnique || !schemaFieldLengthGreaterThanComplianceEntities) {
@@ -751,6 +772,14 @@ export default Component.extend({
       return set(this, 'showAllDatasetMemberData', true);
     },
 
+    /**
+     * Updates the fieldReviewOption with the user selected value
+     * @param {string} value
+     */
+    onFieldReviewChange({ value }) {
+      return set(this, 'fieldReviewOption', value);
+    },
+
     /**
      * Handler for setting the compliance policy into edit mode and rendering
      */
@@ -862,17 +891,19 @@ export default Component.extend({
      * @param {String} identifierType
      */
     onFieldIdentifierTypeChange({ identifierField }, { value: identifierType }) {
-      const currentComplianceEntities = get(this, 'mergedComplianceEntitiesAndColumnFields');
+      const currentComplianceEntities = get(this, 'compliancePolicyChangeSet');
-      let logicalType;
       // A reference to the current field in the compliance list, it should exist even for empty complianceEntities
-      // since this is a reference created in the working copy: mergedComplianceEntitiesAndColumnFields
+      // since this is a reference created in the working copy: compliancePolicyChangeSet
       const currentFieldInComplianceList = currentComplianceEntities.findBy('identifierField', identifierField);
+      let logicalType;
       if (hasPredefinedFieldFormat(identifierType)) {
         logicalType = getDefaultLogicalType(identifierType);
       }
+
       setProperties(currentFieldInComplianceList, {
         identifierType,
-        logicalType
+        logicalType,
+        isDirty: true
       });
       // Set the defaultClassification for the identifierField,
       // although the classification is based on the logicalType,
@@ -898,7 +929,7 @@ export default Component.extend({
           { value: fieldIdentifierTypes.none.value }
         );
       } else {
-        set(field, 'logicalType', logicalType);
+        setProperties(field, { logicalType, isDirty: true });
       }
 
       return this.setDefaultClassification({ identifierField }, { value: logicalType });
@@ -911,7 +942,7 @@ export default Component.extend({
      * @return {*}
      */
     onFieldClassificationChange({ identifierField }, { value: classification = null }) {
-      const currentFieldInComplianceList = get(this, 'mergedComplianceEntitiesAndColumnFields').findBy(
+      const currentFieldInComplianceList = get(this, 'compliancePolicyChangeSet').findBy(
         'identifierField',
         identifierField
       );
@@ -919,7 +950,7 @@ export default Component.extend({
       this.clearMessages();
 
       // Apply the updated classification value to the current instance of the field in working copy
-      set(currentFieldInComplianceList, 'classification', classification);
+      setProperties(currentFieldInComplianceList, { classification, isDirty: true });
     },
 
     /**

wherehows-web/app/components/disable-bubble-input.ts

@@ -0,0 +1,10 @@
+import Ember from 'ember';
+
+const { TextField } = Ember;
+
+export default TextField.extend({
+  /**
+   * Prevents click event bubbling
+   */
+  click: () => false
+});

wherehows-web/app/constants/metadata-acquisition.ts

@@ -1,4 +1,5 @@
 import Ember from 'ember';
+import { arrayMap } from 'wherehows-web/utils/array';
 
 /**
  * Defines the string values that are allowed for a classification
@@ -9,6 +10,14 @@ enum Classification {
   HighlyConfidential = 'highlyConfidential'
 }
 
+/**
+ * String indicating that the user affirms or ignored a field suggestion
+ */
+enum SuggestionIntent {
+  accept = 'accept',
+  ignore = 'ignore'
+}
+
 /**
  * Describes the index signature for the nonIdFieldLogicalTypes object
  */
@@ -18,6 +27,23 @@ interface INonIdLogicalTypes {
     displayAs: string;
   };
 }
+
+/**
+ * Describes the properties on a field identifier object for ui rendering
+ */
+interface IFieldIdProps {
+  value: string;
+  isId: boolean;
+  displayAs: string;
+}
+
+/**
+ * Describes the index signature for fieldIdentifierTypes
+ */
+interface IFieldIdTypes {
+  [prop: string]: IFieldIdProps;
+}
+
 /**
  * A list of id logical types
  * @type {Array.<String>}
@@ -154,12 +180,11 @@ const defaultFieldDataTypeClassification = Object.assign(
 const classifiers = Object.values(defaultFieldDataTypeClassification).filter(
   (classifier, index, iter) => iter.indexOf(classifier) === index
 );
-
 /**
  * A map of identifier types for fields on a dataset
  * @type {{none: {value: string, isId: boolean, displayAs: string}, member: {value: string, isId: boolean, displayAs: string}, subjectMember: {value: string, isId: boolean, displayAs: string}, group: {value: string, isId: boolean, displayAs: string}, organization: {value: string, isId: boolean, displayAs: string}, generic: {value: string, isId: boolean, displayAs: string}}}
  */
-const fieldIdentifierTypes = {
+const fieldIdentifierTypes: IFieldIdTypes = {
   none: {
     value: 'NONE',
     isId: false,
@@ -282,10 +307,36 @@ const logicalTypesForIds = logicalTypeValueLabel('id');
 // Map generic logical type to options consumable in DOM
 const logicalTypesForGeneric = logicalTypeValueLabel('generic');
 
+/**
+ * Caches a list of field identifiers
+ * @type {Array<IFieldIdProps>}
+ */
+const fieldIdentifierTypesList: Array<IFieldIdProps> = arrayMap(
+  (fieldIdentifierType: string) => fieldIdentifierTypes[fieldIdentifierType]
+)(Object.keys(fieldIdentifierTypes));
+
+/**
+ * A list of field identifier types that are Ids i.e member ID, org ID, group ID
+ * @type {Array<Pick<IFieldIdProps, 'value'>>}
+ */
+const fieldIdentifierTypeIds: Array<Pick<IFieldIdProps, 'value'>> = fieldIdentifierTypesList
+  .filter(({ isId }) => isId)
+  .map(({ value }) => ({ value }));
+
+/**
+ * Caches a list of fieldIdentifierTypes values
+ * @type {Array<Pick<IFieldIdProps, 'value'>>}
+ */
+const fieldIdentifierTypeValues: Array<Pick<IFieldIdProps, 'value'>> = fieldIdentifierTypesList.map(({ value }) => ({
+  value
+}));
+
 export {
   defaultFieldDataTypeClassification,
   classifiers,
   fieldIdentifierTypes,
+  fieldIdentifierTypeIds,
+  fieldIdentifierTypeValues,
   idLogicalTypes,
   customIdLogicalTypes,
   nonIdFieldLogicalTypes,
@@ -294,5 +345,6 @@ export {
   hasPredefinedFieldFormat,
   logicalTypesForIds,
   logicalTypesForGeneric,
-  getDefaultLogicalType
+  getDefaultLogicalType,
+  SuggestionIntent
 };

wherehows-web/app/styles/components/dataset-compliance/_compliance-container.scss

@@ -35,3 +35,9 @@
     visibility: hidden;
   }
 }
+
+.compliance-entities-meta {
+  padding: item-spacing(2 0 2);
+  color: #777777;
+  text-align: left;
+}

wherehows-web/app/styles/components/dataset-compliance/_compliance-table.scss

@@ -1,12 +1,13 @@
-@import "../variables";
+@import '../variables';
 
 .dataset-compliance-fields {
   &__has-suggestions {
     color: $compliance-suggestion-hint;
+    margin-left: item-spacing(2);
   }
 
   &__notification-column {
-    width: 5%
+    width: 5%;
   }
 
   &__classification-column {

wherehows-web/app/styles/components/nacho/_nacho-select.scss

@@ -1,5 +1,5 @@
-@import "../../abstracts/variables";
+@import '../../abstracts/variables';
-@import "../../abstracts/mixins";
+@import '../../abstracts/mixins';
 
 /**
  * <select> primary color, borders, icons, etcetera...
@@ -8,7 +8,7 @@ $color: set-color(grey, light);
 $default-border: (1px solid shade($color, 20%));
 
 .nacho-select {
-  display: flex;
+  display: inline-flex;
   align-items: center;
   height: 34px;
   width: 100%;

wherehows-web/app/styles/components/notifications/_notification-dot.scss

@@ -1,4 +1,4 @@
-@import "../variables";
+@import '../variables';
 
 /// Styles a dot that indicates that user attention is required
 .notification-dot {
@@ -15,7 +15,7 @@
     border-color: $compliance-suggestion-hint;
   }
 
-  &--is-unsaved {
+  &--needs-review {
     border-color: set-color(blue, blue5);
   }
 

wherehows-web/app/templates/components/dataset-compliance-row.hbs

@@ -7,10 +7,9 @@
           fieldFormats=fieldFormats
           logicalType=logicalType
           classification=classification
-          suggestionAuthority=suggestionAuthority
           suggestionResolution=suggestionResolution
           suggestion=prediction
-          isNewField=isNewField
+          isReviewRequested=isReviewRequested
           onFieldIdentifierTypeChange=(action 'onFieldIdentifierTypeChange')
           onFieldClassificationChange=(action 'onFieldClassificationChange')
           onFieldLogicalTypeChange=(action 'onFieldLogicalTypeChange')

wherehows-web/app/templates/components/disable-bubble-input.hbs

@@ -0,0 +1 @@
+{{yield}}

wherehows-web/app/templates/datasets/dataset-compliance/-dataset-compliance-entities.hbs

@@ -20,30 +20,31 @@
   {{/if}}
 </section>
 
-{{#if mergedComplianceEntitiesAndColumnFields.length}}
+{{#if compliancePolicyChangeSet.length}}
+
+  <section class="compliance-entities-meta">
+    {{ember-selector
+      values=fieldReviewOptions
+      selected=(readonly fieldReviewOption)
+      selectionDidChange=(action "onFieldReviewChange")
+    }}
+
+    {{#if changeSetReviewCount}}
+      <span class="dataset-compliance-fields__has-suggestions">
+          {{changeSetReviewCount}} fields to be reviewed
+        </span>
+    {{/if}}
+  </section>
+
   {{#dataset-table
     class="nacho-table--stripped dataset-compliance-fields"
-    fields=mergedComplianceEntitiesAndColumnFields
+    fields=filteredChangeSet
     sortColumnWithName=sortColumnWithName
     filterBy=filterBy
     sortDirection=sortDirection
     tableRowComponent='dataset-compliance-row'
-    searchTerm=searchTerm as |table|}}
+    searchTerm=searchTerm as |table|
-
+  }}
-    <caption>
-      <input
-        type="text"
-        title="Search fields"
-        placeholder="Search fields"
-        value="{{table.searchTerm}}"
-        oninput={{action table.filterDidChange value="target.value"}}>
-
-      {{#if (and hasRecentSuggestions (not isNewComplianceInfo))}}
-        <span class="dataset-compliance-fields__has-suggestions">
-            {{complianceSuggestion.complianceSuggestions.length}} fields to be reviewed
-          </span>
-      {{/if}}
-    </caption>
 
     {{#table.head as |head|}}
       {{#head.column class="dataset-compliance-fields__notification-column"}}{{/head.column}}
@@ -94,6 +95,18 @@
       {{/head.column}}
     {{/table.head}}
 
+    <tr>
+      <th></th>
+      <th colspan="6">
+        {{disable-bubble-input
+          title="Search field names"
+          placeholder="Search field names"
+          value=table.searchTerm
+          on-input=(action table.filterDidChange value="target.value")
+        }}
+      </th>
+    </tr>
+
     {{#table.body as |body|}}
       {{#each (sort-by table.sortBy table.data) as |field|}}
         {{#body.row
@@ -104,17 +117,22 @@
           onFieldLogicalTypeChange=(action 'onFieldLogicalTypeChange')
           onFieldClassificationChange=(action 'onFieldClassificationChange')
           onSuggestionIntent=(action 'onFieldSuggestionIntentChange')
-          onFieldIdentifierTypeChange=(action 'onFieldIdentifierTypeChange')as |row|}}
+          onFieldIdentifierTypeChange=(action 'onFieldIdentifierTypeChange') as |row|
+        }}
 
           {{#row.cell}}
-            {{#if row.suggestion}}
+            {{#if (and row.suggestion (not row.suggestionResolution))}}
+
               <span class="notification-dot notification-dot--has-prediction"
                     aria-label="Compliance fields have suggested values"></span>
-            {{/if}}
 
-            {{#if (and row.isNewField (not row.suggestion))}}
+            {{else}}
-              <span class="notification-dot notification-dot--is-unsaved"
+
-                    aria-label="Compliance has not been saved"></span>
+              {{#if row.isReviewRequested}}
+                <span class="notification-dot notification-dot--needs-review"
+                      aria-label="Compliance policy for field does not exist"></span>
+              {{/if}}
+
             {{/if}}
           {{/row.cell}}
 
@@ -134,8 +152,8 @@
                 {{auto-suggest-action action=(action row.onSuggestionAction)}}
               {{/if}}
             {{else}}
-              {{#if row.suggestionAuthority}}
+              {{#if row.suggestionResolution}}
-                {{capitalize row.suggestionResolution}}
+                {{row.suggestionResolution}}
               {{else}}
                 &mdash;
               {{/if}}

wherehows-web/app/typings/untyped-js-module.d.ts

@@ -2,7 +2,7 @@ declare module 'ember-modal-dialog/components/modal-dialog';
 
 declare module 'ember-simple-auth/mixins/authenticated-route-mixin';
 
-declare module 'wherehows-web/utils/datasets/functions';
+declare module 'wherehows-web/utils/datasets/compliance-policy';
 
 // https://github.com/ember-cli/ember-fetch/issues/72
 // TS assumes the mapping btw ES modules and CJS modules is 1:1

wherehows-web/app/utils/api/datasets/columns.ts

@@ -6,7 +6,7 @@ import {
 } from 'wherehows-web/typings/api/datasets/columns';
 import { getJSON } from 'wherehows-web/utils/api/fetcher';
 import { ApiStatus } from 'wherehows-web/utils/api';
-import { arrayMap } from 'wherehows-web/utils/array-map';
+import { arrayMap } from 'wherehows-web/utils/array';
 
 // TODO:  DSS-6122 Create and move to Error module
 

wherehows-web/app/utils/api/datasets/compliance.ts

@@ -1,5 +1,5 @@
 import Ember from 'ember';
-import { createInitialComplianceInfo } from 'wherehows-web/utils/datasets/functions';
+import { createInitialComplianceInfo } from 'wherehows-web/utils/datasets/compliance-policy';
 import { datasetUrlById } from 'wherehows-web/utils/api/datasets/shared';
 import { ApiStatus } from 'wherehows-web/utils/api/shared';
 import { IComplianceSuggestion, IComplianceSuggestionResponse } from 'wherehows-web/typings/api/datasets/compliance';

wherehows-web/app/utils/array-map.ts

@@ -1,10 +0,0 @@
-/**
- * Convenience utility takes a type-safe mapping function, and returns a list mapping function
- * @param {(param: T) => U} mappingFunction maps a single type T to type U
- * @return {(array: Array<T>) => Array<U>}
- */
-const arrayMap = <T, U>(mappingFunction: (param: T) => U): ((array: Array<T>) => Array<U>) => {
-  return array => array.map(mappingFunction);
-};
-
-export { arrayMap };

wherehows-web/app/utils/array.ts

@@ -0,0 +1,24 @@
+/**
+ * Convenience utility takes a type-safe mapping function, and returns a list mapping function
+ * @param {(param: T) => U} mappingFunction maps a single type T to type U
+ * @return {(array: Array<T>) => Array<U>}
+ */
+const arrayMap = <T, U>(mappingFunction: (param: T) => U): ((array: Array<T>) => Array<U>) => (array = []) =>
+  array.map(mappingFunction);
+
+/**
+ * Convenience utility takes a type-safe filter function, and returns a list filtering function
+ * @param {(param: T) => boolean} filtrationFunction
+ * @return {(array: Array<T>) => Array<T>}
+ */
+const arrayFilter = <T>(filtrationFunction: (param: T) => boolean): ((array: Array<T>) => Array<T>) => (array = []) =>
+  array.filter(filtrationFunction);
+
+/**
+ * Duplicate check using every to short-circuit iteration
+ * @param {Array<T>} [list = []] list to check for dupes
+ * @return {boolean} true is unique
+ */
+const isListUnique = <T>(list: Array<T> = []): boolean => new Set(list).size === list.length;
+
+export { arrayMap, arrayFilter, isListUnique };

wherehows-web/app/utils/datasets/compliance-policy.js

@@ -110,4 +110,64 @@ const isPolicyExpectedShape = (candidatePolicy = {}) => {
   return false;
 };
 
-export { createInitialComplianceInfo, isPolicyExpectedShape };
+/**
+ * Checks if a compliance policy changeSet field requires user attention: if a suggestion
+ * is available  but the user has not indicated intent or a policy for the field does not currently exist remotely
+ * and the related field changeSet has not been modified on the client
+ * @param {boolean} isDirty flag indicating the field changeSet has been modified on the client
+ * @param {object|void} suggestion the field suggestion properties
+ * @param {boolean} privacyPolicyExists flag indicating that the field has a current policy upstream
+ * @param {string} suggestionAuthority possibly empty string indicating the user intent for the suggestion
+ * @return {boolean}
+ */
+const fieldChangeSetRequiresReview = ({ isDirty, suggestion, privacyPolicyExists, suggestionAuthority }) => {
+  if (suggestion) {
+    return !suggestionAuthority;
+  }
+
+  return !privacyPolicyExists && !isDirty;
+};
+
+/**
+ * Merges the column fields with the suggestion for the field if available
+ * @param {object} mappedColumnFields a map of column fields to compliance entity properties
+ * @param {object} fieldSuggestionMap a map of field suggestion properties keyed by field name
+ * @return {Array<object>} mapped column field augmented with suggestion if available
+ */
+const mergeMappedColumnFieldsWithSuggestions = (mappedColumnFields = {}, fieldSuggestionMap = {}) =>
+  Object.keys(mappedColumnFields).map(fieldName => {
+    const {
+      identifierField,
+      dataType,
+      identifierType,
+      logicalType,
+      securityClassification,
+      privacyPolicyExists,
+      isDirty
+    } = mappedColumnFields[fieldName];
+    const suggestion = fieldSuggestionMap[identifierField];
+
+    const field = {
+      identifierField,
+      dataType,
+      identifierType,
+      logicalType,
+      privacyPolicyExists,
+      isDirty,
+      classification: securityClassification
+    };
+
+    // If a suggestion exists for this field add the suggestion attribute to the field properties
+    if (suggestion) {
+      return { ...field, suggestion };
+    }
+
+    return field;
+  });
+
+export {
+  createInitialComplianceInfo,
+  isPolicyExpectedShape,
+  fieldChangeSetRequiresReview,
+  mergeMappedColumnFieldsWithSuggestions
+};

wherehows-web/app/utils/object.ts

@@ -0,0 +1,15 @@
+/**
+ * Checks if a type is an object
+ * @param {any} candidate the entity to check
+ */
+const isObject = (candidate: any): candidate is object =>
+  candidate && Object.prototype.toString.call(candidate) === '[object Object]';
+
+/**
+ * Checks that an object has it own enumerable props
+ * @param {Object} object the object to the be tested
+ * @return {boolean} true if enumerable keys are present
+ */
+const hasEnumerableKeys = (object: object): boolean => isObject(object) && !!Object.keys(object).length;
+
+export { isObject, hasEnumerableKeys };

wherehows-web/package.json

@@ -42,7 +42,6 @@
     "ember-cli-typescript": "^1.0.3",
     "ember-cli-uglify": "^1.2.0",
     "ember-composable-helpers": "^2.0.0",
-    "ember-data": "^2.11.1",
     "ember-export-application-global": "^1.1.1",
     "ember-fetch": "^3.4.0",
     "ember-lodash-shim": "^2.0.2",