Merge branch 'master' of https://github.com/linkedin/WhereHows

wherehows-frontend/app/controllers/Application.java

@@ -66,6 +66,10 @@ public class Application extends Controller {
   private static final Boolean HTTPS_REDIRECT = Play.application().configuration().getBoolean("https.redirect", false);
   private static final Boolean WHZ_SHOW_LINEAGE =
       Play.application().configuration().getBoolean("linkedin.show.dataset.lineage", false);
+  private static final Boolean WHZ_SHOW_DS_HEALTH =
+      Play.application().configuration().getBoolean("linkedin.show.dataset.health", false);
+  private static final String WHZ_SUGGESTION_CONFIDENCE_THRESHOLD =
+      Play.application().configuration().getString("linkedin.suggestion.confidence.threshold", "50");
 
   private static final String WHZ_WIKI_LINKS__GDRP_PII =
       Play.application().configuration().getString("linkedin.links.wiki.gdprPii", "");
@@ -199,6 +203,8 @@ public class Application extends Controller {
     config.put("appVersion", APP_VERSION);
     config.put("isInternal", IS_INTERNAL);
     config.put("shouldShowDatasetLineage", WHZ_SHOW_LINEAGE);
+    config.put("shouldShowDatasetHealth", WHZ_SHOW_DS_HEALTH);
+    config.put("suggestionConfidenceThreshold", Integer.parseInt(WHZ_SUGGESTION_CONFIDENCE_THRESHOLD));
     config.set("wikiLinks", wikiLinks());
     config.set("JitAclAccessWhitelist", Json.toJson(StringUtils.split(JIT_ACL_WHITELIST, ',')));
     config.set("tracking", trackingInfo());

wherehows-web/app/components/dataset-compliance-field-tag.ts

@@ -67,6 +67,13 @@ export default class DatasetComplianceFieldTag extends Component {
    */
   parentHasSingleTag: boolean;
 
+  /**
+   * Confidence percentage number used to filter high quality suggestions versus lower quality
+   * @type {number}
+   * @memberof DatasetComplianceFieldTag
+   */
+  suggestionConfidenceThreshold: number;
+
   /**
    * Stores the value of error result if the valuePattern is invalid
    * @type {string}
@@ -204,8 +211,11 @@ export default class DatasetComplianceFieldTag extends Component {
     this: DatasetComplianceFieldTag
   ): boolean {
     const tagWithoutSuggestion = <IComplianceChangeSet>omit<IComplianceChangeSet>(get(this, 'tag'), ['suggestion']);
+    const suggestionConfidenceThreshold = get(this, 'suggestionConfidenceThreshold');
 
-    return tagNeedsReview(get(this, 'complianceDataTypes'))(tagWithoutSuggestion);
+    return tagNeedsReview(get(this, 'complianceDataTypes'), { checkSuggestions: true, suggestionConfidenceThreshold })(
+      tagWithoutSuggestion
+    );
   });
 
   /**

wherehows-web/app/components/dataset-compliance-rollup-row.ts

@@ -65,9 +65,17 @@ export default class DatasetComplianceRollupRow extends Component.extend({
   /**
    * Reference to the compliance data types
    * @type {Array<IComplianceDataType>}
+   * @memberof DatasetComplianceRollupRow
    */
   complianceDataTypes: Array<IComplianceDataType>;
 
+  /**
+   * Confidence percentage number used to filter high quality suggestions versus lower quality
+   * @type {number}
+   * @memberof DatasetComplianceRollupRow
+   */
+  suggestionConfidenceThreshold: number;
+
   /**
    * Flag indicating the field has a readonly attribute
    * @type ComputedProperty<boolean>
@@ -83,9 +91,16 @@ export default class DatasetComplianceRollupRow extends Component.extend({
   isReviewRequested = computed(
     `fieldChangeSet.@each.{${changeSetReviewableAttributeTriggers}}`,
     'complianceDataTypes',
+    'suggestionConfidenceThreshold',
     function(this: DatasetComplianceRollupRow): boolean {
-      const tags = get(this, 'fieldChangeSet');
+      const { fieldChangeSet: tags, suggestionConfidenceThreshold } = getProperties(this, [
-      const { length } = fieldTagsRequiringReview(get(this, 'complianceDataTypes'))(get(this, 'identifierField'))(tags);
+        'fieldChangeSet',
+        'suggestionConfidenceThreshold'
+      ]);
+      const { length } = fieldTagsRequiringReview(get(this, 'complianceDataTypes'), {
+        checkSuggestions: true,
+        suggestionConfidenceThreshold
+      })(get(this, 'identifierField'))(tags);
 
       return !!length || tagsHaveNoneAndNotNoneType(tags);
     }
@@ -172,10 +187,12 @@ export default class DatasetComplianceRollupRow extends Component.extend({
    * @type {(ComputedProperty<{ identifierType: ComplianceFieldIdValue; logicalType: string; confidence: number } | void>)}
    * @memberof DatasetComplianceRollupRow
    */
-  suggestion = computed('fieldProps.suggestion', 'suggestionAuthority', function(
+  suggestion = computed('fieldProps.suggestion', 'suggestionAuthority', 'suggestionConfidenceThreshold', function(
     this: DatasetComplianceRollupRow
   ): ISuggestedFieldTypeValues | void {
-    return getTagSuggestions(getWithDefault(this, 'fieldProps', <IComplianceChangeSet>{}));
+    const fieldProps = getWithDefault(this, 'fieldProps', <IComplianceChangeSet>{});
+
+    return getTagSuggestions({ suggestionConfidenceThreshold: get(this, 'suggestionConfidenceThreshold') })(fieldProps);
   });
 
   /**

wherehows-web/app/components/dataset-compliance.ts

@@ -35,7 +35,8 @@ import {
   singleTagsInChangeSet,
   tagsForIdentifierField,
   overrideTagReadonly,
-  editableTags
+  editableTags,
+  lowQualitySuggestionConfidenceThreshold
 } from 'wherehows-web/constants';
 import { getTagsSuggestions } from 'wherehows-web/utils/datasets/compliance-suggestions';
 import { arrayMap, compact, isListUnique, iterateArrayAsync } from 'wherehows-web/utils/array';
@@ -70,12 +71,12 @@ import { notificationDialogActionFactory } from 'wherehows-web/utils/notificatio
 import validateMetadataObject, {
   complianceEntitiesTaxonomy
 } from 'wherehows-web/utils/datasets/compliance/metadata-schema';
+import { typeOf } from '@ember/utils';
 
 const {
   complianceDataException,
   complianceFieldNotUnique,
   missingTypes,
-  helpText,
   missingPurgePolicy,
   missingDatasetSecurityClassification
 } = compliancePolicyStrings;
@@ -104,7 +105,6 @@ export default class DatasetCompliance extends Component {
   filterBy: string;
   sortDirection: string;
   searchTerm: string;
-  helpText = helpText;
   _hasBadData: boolean;
   platform: IDatasetView['platform'];
   isCompliancePolicyAvailable: boolean = false;
@@ -112,7 +112,7 @@ export default class DatasetCompliance extends Component {
   complianceInfo: undefined | IComplianceInfo;
 
   /**
-   * Lists the compliance entities that are entered via the advanced edititing interface
+   * Lists the compliance entities that are entered via the advanced editing interface
    * @type {Pick<IComplianceInfo, 'complianceEntities'>}
    * @memberof DatasetCompliance
    */
@@ -135,9 +135,17 @@ export default class DatasetCompliance extends Component {
   /**
    * Flag indicating the current compliance policy edit-view mode
    * @type {boolean}
+   * @memberof DatasetCompliance
    */
   showGuidedComplianceEditMode: boolean = true;
 
+  /**
+   * Confidence percentage number used to filter high quality suggestions versus lower quality
+   * @type {number}
+   * @memberof DatasetCompliance
+   */
+  suggestionConfidenceThreshold: number;
+
   /**
    * Formatted JSON string representing the compliance entities for this dataset
    * @type {ComputedProperty<string>}
@@ -146,7 +154,6 @@ export default class DatasetCompliance extends Component {
     this: DatasetCompliance
   ): string {
     const entityAttrs = ['identifierField', 'identifierType', 'logicalType', 'nonOwner', 'valuePattern', 'readonly'];
-    //@ts-ignore property access path using dot notation limitation
     const entityMap: ISchemaFieldsToPolicy = get(this, 'columnIdFieldsToCurrentPrivacyPolicy');
     const entitiesWithModifiableKeys = arrayMap((tag: IComplianceEntityWithMetadata) => pick(tag, entityAttrs))(
       (<Array<IComplianceEntityWithMetadata>>[]).concat(...Object.values(entityMap))
@@ -334,6 +341,8 @@ export default class DatasetCompliance extends Component {
     this.searchTerm || set(this, 'searchTerm', '');
     this.schemaFieldNamesMappedToDataTypes || (this.schemaFieldNamesMappedToDataTypes = []);
     this.complianceDataTypes || (this.complianceDataTypes = []);
+    typeOf(this.suggestionConfidenceThreshold) === 'number' ||
+      set(this, 'suggestionConfidenceThreshold', lowQualitySuggestionConfidenceThreshold);
   }
 
   /**
@@ -739,16 +748,26 @@ export default class DatasetCompliance extends Component {
     'columnIdFieldsToCurrentPrivacyPolicy',
     'complianceDataTypes',
     'identifierFieldToSuggestion',
+    'suggestionConfidenceThreshold',
     function(this: DatasetCompliance): Array<IComplianceChangeSet> {
       // schemaFieldNamesMappedToDataTypes is a dependency for CP columnIdFieldsToCurrentPrivacyPolicy, so no need to dep on that directly
       const changeSet = mergeComplianceEntitiesWithSuggestions(
         get(this, 'columnIdFieldsToCurrentPrivacyPolicy'),
         get(this, 'identifierFieldToSuggestion')
       );
+      const suggestionThreshold = get(this, 'suggestionConfidenceThreshold');
 
       // pass current changeSet state to parent handlers
-      run(() => next(this, 'notifyHandlerOfSuggestions', changeSet));
+      run(() => next(this, 'notifyHandlerOfSuggestions', suggestionThreshold, changeSet));
-      run(() => next(this, 'notifyHandlerOfFieldsRequiringReview', get(this, 'complianceDataTypes'), changeSet));
+      run(() =>
+        next(
+          this,
+          'notifyHandlerOfFieldsRequiringReview',
+          suggestionThreshold,
+          get(this, 'complianceDataTypes'),
+          changeSet
+        )
+      );
 
       return changeSet;
     }
@@ -764,14 +783,16 @@ export default class DatasetCompliance extends Component {
     'fieldReviewOption',
     'compliancePolicyChangeSet',
     'complianceDataTypes',
+    'suggestionConfidenceThreshold',
     function(this: DatasetCompliance): Array<IComplianceChangeSet> {
-      const { compliancePolicyChangeSet: changeSet, complianceDataTypes } = getProperties(this, [
+      const {
-        'compliancePolicyChangeSet',
+        compliancePolicyChangeSet: changeSet,
-        'complianceDataTypes'
+        complianceDataTypes,
-      ]);
+        suggestionConfidenceThreshold
+      } = getProperties(this, ['compliancePolicyChangeSet', 'complianceDataTypes', 'suggestionConfidenceThreshold']);
 
       return get(this, 'fieldReviewOption') === 'showReview'
-        ? tagsRequiringReview(complianceDataTypes)(changeSet)
+        ? tagsRequiringReview(complianceDataTypes, { checkSuggestions: true, suggestionConfidenceThreshold })(changeSet)
         : changeSet;
     }
   );
@@ -788,9 +809,10 @@ export default class DatasetCompliance extends Component {
   changeSetReviewWithoutSuggestionCheck = computed('changeSetReview', function(
     this: DatasetCompliance
   ): Array<IComplianceChangeSet> {
-    return tagsRequiringReview(get(this, 'complianceDataTypes'), { checkSuggestions: false })(
+    return tagsRequiringReview(get(this, 'complianceDataTypes'), {
-      get(this, 'changeSetReview')
+      checkSuggestions: false,
-    );
+      suggestionConfidenceThreshold: 0 // irrelevant value set to 0 since checkSuggestions flag is false above
+    })(get(this, 'changeSetReview'));
   });
 
   /**
@@ -801,8 +823,17 @@ export default class DatasetCompliance extends Component {
   changeSetReview = computed(
     `compliancePolicyChangeSet.@each.{${changeSetReviewableAttributeTriggers}}`,
     'complianceDataTypes',
+    'suggestionConfidenceThreshold',
     function(this: DatasetCompliance): Array<IComplianceChangeSet> {
-      return tagsRequiringReview(get(this, 'complianceDataTypes'))(get(this, 'compliancePolicyChangeSet'));
+      const { suggestionConfidenceThreshold, compliancePolicyChangeSet } = getProperties(this, [
+        'suggestionConfidenceThreshold',
+        'compliancePolicyChangeSet'
+      ]);
+
+      return tagsRequiringReview(get(this, 'complianceDataTypes'), {
+        checkSuggestions: true,
+        suggestionConfidenceThreshold
+      })(compliancePolicyChangeSet);
     }
   );
 
@@ -866,19 +897,25 @@ export default class DatasetCompliance extends Component {
 
   /**
    * Invokes external action with flag indicating that at least 1 suggestion exists for a field in the changeSet
+   * @param {number} suggestionConfidenceThreshold confidence threshold for filtering out higher quality suggestions
    * @param {Array<IComplianceChangeSet>} changeSet
    */
-  notifyHandlerOfSuggestions = (changeSet: Array<IComplianceChangeSet>): void => {
+  notifyHandlerOfSuggestions = (
-    const hasChangeSetSuggestions = !!compact(getTagsSuggestions(changeSet)).length;
+    suggestionConfidenceThreshold: number,
+    changeSet: Array<IComplianceChangeSet>
+  ): void => {
+    const hasChangeSetSuggestions = !!compact(getTagsSuggestions({ suggestionConfidenceThreshold })(changeSet)).length;
     this.notifyOnChangeSetSuggestions(hasChangeSetSuggestions);
   };
 
   /**
    * Invokes external action with flag indicating that a field in the tags requires user review
+   * @param {number} suggestionConfidenceThreshold confidence threshold for filtering out higher quality suggestions
    * @param {Array<IComplianceDataType>} complianceDataTypes
    * @param {Array<IComplianceChangeSet>} tags
    */
   notifyHandlerOfFieldsRequiringReview = (
+    suggestionConfidenceThreshold: number,
     complianceDataTypes: Array<IComplianceDataType>,
     tags: Array<IComplianceChangeSet>
   ): void => {
@@ -886,7 +923,10 @@ export default class DatasetCompliance extends Component {
     assert('expected complianceDataTypes to be of type `array`', Array.isArray(complianceDataTypes));
     assert('expected tags to be of type `array`', Array.isArray(tags));
 
-    const hasChangeSetDrift = !!tagsRequiringReview(complianceDataTypes)(tags).length;
+    const hasChangeSetDrift = !!tagsRequiringReview(complianceDataTypes, {
+      checkSuggestions: true,
+      suggestionConfidenceThreshold
+    })(tags).length;
 
     this.notifyOnChangeSetRequiresReview(hasChangeSetDrift);
   };

wherehows-web/app/components/datasets/containers/dataset-compliance.ts

@@ -21,12 +21,20 @@ import {
 import { columnDataTypesAndFieldNames } from 'wherehows-web/utils/api/datasets/columns';
 import { readDatasetSchemaByUrn } from 'wherehows-web/utils/api/datasets/schema';
 import { readComplianceDataTypes } from 'wherehows-web/utils/api/list/compliance-datatypes';
-import { compliancePolicyStrings, removeReadonlyAttr, editableTags, SuggestionIntent } from 'wherehows-web/constants';
+import {
+  compliancePolicyStrings,
+  removeReadonlyAttr,
+  editableTags,
+  SuggestionIntent,
+  lowQualitySuggestionConfidenceThreshold
+} from 'wherehows-web/constants';
 import { iterateArrayAsync } from 'wherehows-web/utils/array';
 import validateMetadataObject, {
   complianceEntitiesTaxonomy
 } from 'wherehows-web/utils/datasets/compliance/metadata-schema';
 import { notificationDialogActionFactory } from 'wherehows-web/utils/notifications/notifications';
+import Configurator from 'wherehows-web/services/configurator';
+import { typeOf } from '@ember/utils';
 
 /**
  * Type alias for the response when container data items are batched
@@ -49,6 +57,7 @@ type BatchContainerDataResult = Pick<
   | 'complianceSuggestion'
   | 'schemaFieldNamesMappedToDataTypes'
   | 'schemaless'
+  | 'suggestionConfidenceThreshold'
 >;
 
 const { successUpdating, failedUpdating, successUploading, invalidPolicyData } = compliancePolicyStrings;
@@ -128,6 +137,13 @@ export default class DatasetComplianceContainer extends Component {
    */
   datasetName: string = '';
 
+  /**
+   * Confidence percentage number used to filter high quality suggestions versus lower quality
+   * @type {number}
+   * @memberof DatasetComplianceContainer
+   */
+  suggestionConfidenceThreshold: number = lowQualitySuggestionConfidenceThreshold;
+
   /**
    * The urn identifier for the dataset
    * @type {string}
@@ -177,11 +193,16 @@ export default class DatasetComplianceContainer extends Component {
     ]);
     const schemaFieldNamesMappedToDataTypes = await iterateArrayAsync(columnDataTypesAndFieldNames)(columns);
     const { containingPersonalData, fromUpstream } = complianceInfo;
+    let suggestionConfidenceThreshold = Configurator.getConfig('suggestionConfidenceThreshold');
 
+    // convert to fractional percentage if valid number is present
+    typeOf(suggestionConfidenceThreshold) === 'number' &&
+      (suggestionConfidenceThreshold = suggestionConfidenceThreshold / 100);
     this.notifyPiiStatus(!!containingPersonalData);
     this.onCompliancePolicyStateChange.call(this, { isNewComplianceInfo, fromUpstream: !!fromUpstream });
 
     return setProperties(this, {
+      suggestionConfidenceThreshold,
       isNewComplianceInfo,
       complianceInfo,
       complianceDataTypes,

wherehows-web/app/components/datasets/containers/dataset-health.ts

@@ -0,0 +1,32 @@
+import Component from '@ember/component';
+import { get } from '@ember/object';
+import { task, TaskInstance } from 'ember-concurrency';
+
+/**
+ * This is the container component for the dataset health tab. It should contain the health bar graphs and a table
+ * depicting the detailed health scores. Aside from fetching the data, it also handles click interactions between
+ * the graphs and the table in terms of filtering and displaying of data
+ */
+export default class DatasetHealthContainer extends Component {
+  /**
+   * The urn identifier for the dataset
+   * @type {string}
+   */
+  urn: string;
+
+  didInsertElement() {
+    get(this, 'getContainerDataTask').perform();
+  }
+
+  didUpdateAttrs() {
+    get(this, 'getContainerDataTask').perform();
+  }
+
+  /**
+   * An async parent task to group all data tasks for this container component
+   * @type {Task<TaskInstance<Promise<any>>, (a?: any) => TaskInstance<TaskInstance<Promise<any>>>>}
+   */
+  getContainerDataTask = task(function*(this: DatasetHealthContainer): IterableIterator<TaskInstance<Promise<any>>> {
+    // Do something in the future
+  });
+}

wherehows-web/app/constants/dataset-compliance.ts

@@ -40,11 +40,6 @@ const compliancePolicyStrings = {
   failedUpdating: 'An error occurred while saving.',
   successUploading: 'Metadata successfully updated! Please confirm and complete subsequent metadata information.',
   invalidPolicyData: 'Received policy in an unexpected format! Please check the provided attributes and try again.',
-  helpText: {
-    classification:
-      'This security classification is from go/dht and should be good enough in most cases. ' +
-      'You can optionally override it if required by house security.'
-  },
   missingPurgePolicy: 'Please specify a Compliance Purge Policy',
   missingDatasetSecurityClassification: 'Please specify a security classification for this dataset.'
 };
@@ -183,10 +178,18 @@ const suggestedIdentifierTypesInList = (suggestion: ISuggestedFieldTypeValues |
  * @param {SchemaFieldToSuggestedValue} suggestion
  * @param {SuggestionIntent} suggestionAuthority
  * @param {ComplianceFieldIdValue | NonIdLogicalType | null} identifierType
+ * @param {number} suggestionConfidenceThreshold confidence threshold for filtering out higher quality suggestions
  * @return {boolean}
  */
-const tagSuggestionNeedsReview = ({ suggestion, suggestionAuthority, identifierType }: IComplianceChangeSet): boolean =>
+const tagSuggestionNeedsReview = ({
-  suggestion && suggestion.identifierType !== identifierType && isHighConfidenceSuggestion(suggestion)
+  suggestion,
+  suggestionAuthority,
+  identifierType,
+  suggestionConfidenceThreshold
+}: IComplianceChangeSet & { suggestionConfidenceThreshold: number }): boolean =>
+  suggestion &&
+  suggestion.identifierType !== identifierType &&
+  isHighConfidenceSuggestion(suggestion, suggestionConfidenceThreshold)
     ? !suggestionAuthority
     : false;
 
@@ -223,14 +226,14 @@ const tagValuePatternNeedsReview = ({ valuePattern }: IComplianceChangeSet): boo
  * checking steps
  * @return {(tag: IComplianceChangeSet) => boolean}
  */
-const tagNeedsReview = (complianceDataTypes: Array<IComplianceDataType>, options?: IComplianceTagReviewOptions) =>
+const tagNeedsReview = (complianceDataTypes: Array<IComplianceDataType>, options: IComplianceTagReviewOptions) =>
   /**
    * Checks if a compliance tag needs to be reviewed against a set of rules
    * @param {IComplianceChangeSet} tag
    * @return {boolean}
    */
   (tag: IComplianceChangeSet): boolean => {
-    const { checkSuggestions } = options || { checkSuggestions: true };
+    const { checkSuggestions, suggestionConfidenceThreshold } = options;
     const { isDirty, privacyPolicyExists, identifierType, logicalType } = tag;
     let isReviewRequired = false;
 
@@ -241,7 +244,7 @@ const tagNeedsReview = (complianceDataTypes: Array<IComplianceDataType>, options
 
     // Check that a hi confidence suggestion exists and the identifierType does not match the change set item
     if (checkSuggestions) {
-      isReviewRequired = isReviewRequired || tagSuggestionNeedsReview(tag);
+      isReviewRequired = isReviewRequired || tagSuggestionNeedsReview({ ...tag, suggestionConfidenceThreshold });
     }
 
     // Ensure that tag has a logical type and nonOwner flag is set when tag is of id type
@@ -391,17 +394,20 @@ const singleTagsInChangeSet = (
  * @param {IComplianceTagReviewOptions} options
  * @return {(array: Array<IComplianceChangeSet>) => Array<IComplianceChangeSet>}
  */
-const tagsRequiringReview = (complianceDataTypes: Array<IComplianceDataType>, options?: IComplianceTagReviewOptions) =>
+const tagsRequiringReview = (complianceDataTypes: Array<IComplianceDataType>, options: IComplianceTagReviewOptions) =>
   arrayFilter<IComplianceChangeSet>(tagNeedsReview(complianceDataTypes, options));
 
 /**
  * Lists the tags for a specific identifier field that need to be reviewed
  * @param {Array<IComplianceDataType>} complianceDataTypes
+ * @param {IComplianceTagReviewOptions} options
  * @return {(identifierField: string) => (tags: Array<IComplianceChangeSet>) => Array<IComplianceChangeSet>}
  */
-const fieldTagsRequiringReview = (complianceDataTypes: Array<IComplianceDataType>) => (identifierField: string) => (
+const fieldTagsRequiringReview = (
-  tags: Array<IComplianceChangeSet>
+  complianceDataTypes: Array<IComplianceDataType>,
-) => tagsRequiringReview(complianceDataTypes)(tagsForIdentifierField(identifierField)(tags));
+  options: IComplianceTagReviewOptions
+) => (identifierField: string) => (tags: Array<IComplianceChangeSet>) =>
+  tagsRequiringReview(complianceDataTypes, options)(tagsForIdentifierField(identifierField)(tags));
 
 /**
  * Extracts a suggestion for a field from a suggestion map and merges a compliance entity with the suggestion

wherehows-web/app/controllers/datasets/dataset.ts

@@ -87,6 +87,14 @@ export default class DatasetController extends Controller {
    */
   shouldShowDatasetLineage: boolean;
 
+  /**
+   * Flags the health feature for datasets, which is currently in the development stage so we should not
+   * have it appear in production
+   * @type {boolean}
+   * @memberof DatasetController
+   */
+  shouldShowDatasetHealth: boolean;
+
   /**
    * Flag indicating if the dataset contains personally identifiable information
    * @type {boolean}

wherehows-web/app/routes/datasets/dataset.ts

@@ -95,7 +95,8 @@ export default class DatasetRoute extends Route {
     setProperties(controller, {
       isInternal: !!getConfig('isInternal'),
       jitAclAccessWhitelist: getConfig('JitAclAccessWhitelist') || [],
-      shouldShowDatasetLineage: getConfig('shouldShowDatasetLineage')
+      shouldShowDatasetLineage: getConfig('shouldShowDatasetLineage'),
+      shouldShowDatasetHealth: getConfig('shouldShowDatasetHealth')
     });
   }
 

wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs

@@ -25,6 +25,7 @@
       platform=platform
       complianceInfo=complianceInfo
       complianceSuggestion=complianceSuggestion
+      suggestionConfidenceThreshold=suggestionConfidenceThreshold
       isNewComplianceInfo=isNewComplianceInfo
       schemaFieldNamesMappedToDataTypes=schemaFieldNamesMappedToDataTypes
       complianceDataTypes=complianceDataTypes

wherehows-web/app/templates/components/datasets/containers/dataset-health.hbs

@@ -0,0 +1 @@
+Coming Soon!

wherehows-web/app/templates/datasets/dataset-compliance/-dataset-compliance-entities.hbs

@@ -136,6 +136,7 @@
             field=field
             isNewComplianceInfo=isNewComplianceInfo
             complianceDataTypes=complianceDataTypes
+            suggestionConfidenceThreshold=suggestionConfidenceThreshold
             onFieldDblClick=(action "onFieldDblClick")
             onFieldTagAdded=(action "onFieldTagAdded")
             onFieldTagRemoved=(action "onFieldTagRemoved")
@@ -216,6 +217,7 @@
                             sourceTag=tag
                             parentHasSingleTag=row.hasSingleTag
                             tagDidChange=(action "tagPropertiesUpdated")
+                            suggestionConfidenceThreshold=suggestionConfidenceThreshold
                             complianceFieldIdDropdownOptions=complianceFieldIdDropdownOptions
                             complianceDataTypes=complianceDataTypes as |tagRowComponent|
                           }}

wherehows-web/app/templates/datasets/dataset.hbs

@@ -105,6 +105,12 @@
           {{/tablist.tab}}
         {{/if}}
 
+        {{#if shouldShowDatasetHealth}}
+          {{#tablist.tab tabIds.Health on-select=(action "tabSelectionChanged")}}
+            Health
+          {{/tablist.tab}}
+        {{/if}}
+
       {{/tabs.tablist}}
     </div>
   </div>
@@ -152,5 +158,11 @@
         {{datasets/dataset-relationships urn=encodedUrn}}
       {{/tabs.tabpanel}}
     {{/if}}
+
+    {{#if shouldShowDatasetHealth}}
+      {{#tabs.tabpanel tabIds.Health}}
+        {{datasets/containers/dataset-health urn=encodedUrn}}
+      {{/tabs.tabpanel}}
+    {{/if}}
   </div>
 {{/ivy-tabs}}

wherehows-web/app/typings/api/configurator/configurator.d.ts

@@ -9,6 +9,9 @@ interface IAppConfig {
   isInternal: boolean | void;
   JitAclAccessWhitelist: Array<DatasetPlatform> | void;
   shouldShowDatasetLineage: boolean;
+  shouldShowDatasetHealth: boolean;
+  // confidence threshold for filtering out higher quality suggestions
+  suggestionConfidenceThreshold: number;
   tracking: {
     isEnabled: boolean;
     trackers: {

wherehows-web/app/typings/app/dataset-compliance.d.ts

@@ -23,7 +23,10 @@ interface IDatasetComplianceActions {
  * @interface IComplianceTagReviewOptions
  */
 interface IComplianceTagReviewOptions {
+  // flag determines if suggested values are considered in tag(IComplianceChangeSet) review check
   checkSuggestions: boolean;
+  // confidence threshold for filtering out higher quality suggestions
+  suggestionConfidenceThreshold: number;
 }
 
 /**

wherehows-web/app/utils/array.ts

@@ -1,4 +1,4 @@
-import { identity } from 'wherehows-web/utils/helpers/functions';
+import { identity, not } from 'wherehows-web/utils/helpers/functions';
 
 /**
  * Composable function that will in turn consume an item from a list an emit a result of equal or same type
@@ -22,37 +22,53 @@ const take = <T>(n: number = 0) => (list: Array<T>): Array<T> => Array.prototype
 
 /**
  * Convenience utility takes a type-safe mapping function, and returns a list mapping function
- * @param {(param: T) => U} mappingFunction maps a single type T to type U
+ * @param {(param: T) => U} predicate maps a single type T to type U
  * @return {(array: Array<T>) => Array<U>}
  */
-const arrayMap = <T, U>(mappingFunction: (param: T) => U): ((array: Array<T>) => Array<U>) => (array = []) =>
+const arrayMap = <T, U>(predicate: (param: T) => U): ((array: Array<T>) => Array<U>) => (array = []) =>
-  array.map(mappingFunction);
+  array.map(predicate);
+
+/**
+ * Partitions an array into a tuple containing elements that meet the predicate in the zeroth index,
+ * and excluded elements in the next
+ * `iterate-first data-last` function
+ * @template T type of source element list
+ * @template U subtype of T in first partition
+ * @param {(param: T) => param is U} predicate is a type guard function
+ * @returns {((array: Array<T>) => [Array<U>, Array<Exclude<T, U>>])}
+ */
+const arrayPartition = <T, U extends T>(
+  predicate: (param: T) => param is U
+): ((array: Array<T>) => [Array<U>, Array<Exclude<T, U>>]) => (array = []) => [
+  array.filter(predicate),
+  array.filter<Exclude<T, U>>((v: T): v is Exclude<T, U> => not(predicate)(v))
+];
 
 /**
  * Convenience utility takes a type-safe filter function, and returns a list filtering function
- * @param {(param: T) => boolean} filtrationFunction
+ * @param {(param: T) => boolean} predicate
  * @return {(array: Array<T>) => Array<T>}
  */
-const arrayFilter = <T>(filtrationFunction: (param: T) => boolean): ((array: Array<T>) => Array<T>) => (array = []) =>
+const arrayFilter = <T>(predicate: (param: T) => boolean): ((array: Array<T>) => Array<T>) => (array = []) =>
-  array.filter(filtrationFunction);
+  array.filter(predicate);
 
 /**
  * Type safe utility `iterate-first data-last` function for array every
  * @template T
- * @param {(param: T) => boolean} filter
+ * @param {(param: T) => boolean} predicate
  * @returns {((array: Array<T>) => boolean)}
  */
-const arrayEvery = <T>(filter: (param: T) => boolean): ((array: Array<T>) => boolean) => (array = []) =>
+const arrayEvery = <T>(predicate: (param: T) => boolean): ((array: Array<T>) => boolean) => (array = []) =>
-  array.every(filter);
+  array.every(predicate);
 
 /**
  * Type safe utility `iterate-first data-last` function for array some
  * @template T
- * @param {(param: T) => boolean} filter
+ * @param {(param: T) => boolean} predicate
  * @return {(array: Array<T>) => boolean}
  */
-const arraySome = <T>(filter: (param: T) => boolean): ((array: Array<T>) => boolean) => (array = []) =>
+const arraySome = <T>(predicate: (param: T) => boolean): ((array: Array<T>) => boolean) => (array = []) =>
-  array.some(filter);
+  array.some(predicate);
 
 /**
  * Composable reducer abstraction, curries a reducing iteratee and returns a reducing function that takes a list
@@ -228,9 +244,10 @@ export { Many, Iteratee };
 export {
   take,
   arrayMap,
+  arrayPipe,
   arrayFilter,
   arrayReduce,
-  arrayPipe,
+  arrayPartition,
   isListUnique,
   compact,
   arrayEvery,

wherehows-web/app/utils/datasets/compliance-suggestions.ts

@@ -1,4 +1,3 @@
-import { lowQualitySuggestionConfidenceThreshold } from 'wherehows-web/constants';
 import { arrayMap } from 'wherehows-web/utils/array';
 import { IComplianceChangeSet, ISuggestedFieldTypeValues } from 'wherehows-web/typings/app/dataset-compliance';
 
@@ -6,23 +5,28 @@ import { IComplianceChangeSet, ISuggestedFieldTypeValues } from 'wherehows-web/t
  * Takes a list of suggestions with confidence values, and if the confidence is greater than
  * a low confidence threshold
  * @param {number} confidenceLevel percentage indicating how confidence the system is in the suggested value
+ * @param {number} suggestionConfidenceThreshold threshold number to consider as a valid suggestion
  * @return {boolean}
  */
-const isHighConfidenceSuggestion = ({ confidenceLevel = 0 }: { confidenceLevel: number }): boolean =>
+const isHighConfidenceSuggestion = (
-  confidenceLevel > lowQualitySuggestionConfidenceThreshold;
+  { confidenceLevel = 0 }: { confidenceLevel: number },
+  suggestionConfidenceThreshold: number = 0
+): boolean => confidenceLevel > suggestionConfidenceThreshold;
 
 /**
  * Extracts the tag suggestion from an IComplianceChangeSet tag.
  * If a suggestionAuthority property exists on the tag, then the user has already either accepted or ignored
  * the suggestion for this tag. It's value should not be taken into account on re-renders,
  * in place, this substitutes an empty suggestion
- * @param {IComplianceChangeSet} tag
+ * @param {number} suggestionConfidenceThreshold confidence threshold for filtering out higher quality suggestions
- * @return {{identifierType: IComplianceChangeSet.identifierType, logicalType: IComplianceChangeSet.logicalType, confidence: number} | void}
+ * @return {(tag?: IComplianceChangeSet) => (ISuggestedFieldTypeValues | void)}
  */
-const getTagSuggestions = (tag: IComplianceChangeSet = <IComplianceChangeSet>{}): ISuggestedFieldTypeValues | void => {
+const getTagSuggestions = ({ suggestionConfidenceThreshold }: { suggestionConfidenceThreshold: number }) => (
+  tag: IComplianceChangeSet = <IComplianceChangeSet>{}
+): ISuggestedFieldTypeValues | void => {
   const { suggestion } = tag;
 
-  if (suggestion && isHighConfidenceSuggestion(suggestion)) {
+  if (suggestion && isHighConfidenceSuggestion(suggestion, suggestionConfidenceThreshold)) {
     const { identifierType, logicalType, confidenceLevel: confidence } = suggestion;
     return { identifierType, logicalType, confidence: +(confidence * 100).toFixed(2) };
   }
@@ -30,8 +34,10 @@ const getTagSuggestions = (tag: IComplianceChangeSet = <IComplianceChangeSet>{})
 
 /**
  * Gets the suggestions for a list of IComplianceChangeSet fields
- * @type {(array: Array<IComplianceChangeSet>) => Array<{identifierType: ComplianceFieldIdValue | NonIdLogicalType | null; logicalType: IdLogicalType | null; confidence: number} | void>}
+ * @param {number} suggestionConfidenceThreshold
+ * @return {(array: Array<IComplianceChangeSet>) => Array<ISuggestedFieldTypeValues | void>}
  */
-const getTagsSuggestions = arrayMap(getTagSuggestions);
+const getTagsSuggestions = ({ suggestionConfidenceThreshold }: { suggestionConfidenceThreshold: number }) =>
+  arrayMap(getTagSuggestions({ suggestionConfidenceThreshold }));
 
 export { isHighConfidenceSuggestion, getTagSuggestions, getTagsSuggestions };

wherehows-web/app/utils/object.ts

@@ -27,7 +27,9 @@ interface IPartialOrIdentityTypeFn<T> {
  * @param {Array<K>} [droppedKeys=[]] the list of attributes on T to be dropped
  * @returns {IPartialOrIdentityTypeFn<T>}
  */
-const fleece = <T, K extends keyof T>(droppedKeys: Array<K> = []): IPartialOrIdentityTypeFn<T> => o => {
+const fleece = <T, K extends keyof T = keyof T>(droppedKeys: Array<K> = []): IPartialOrIdentityTypeFn<T> => (
+  o: T
+): Partial<T> | T => {
   const partialResult = Object.assign({}, o);
 
   return droppedKeys.reduce((partial, key) => {

wherehows-web/ember-cli-build.js

@@ -21,6 +21,15 @@ module.exports = function(defaults) {
       includePolyfill: true
     },
 
+    emberHighCharts: {
+      includedHighCharts: true,
+      // Note: Since we only need highcharts, excluding the other available modules in the addon
+      includeHighStock: false,
+      includeHighMaps: false,
+      includeHighChartsMore: false,
+      includeHighCharts3D: false
+    },
+
     storeConfigInMeta: false,
 
     SRI: {

wherehows-web/package.json

@@ -66,6 +66,7 @@
     "ember-export-application-global": "^2.0.0",
     "ember-fetch": "^3.4.4",
     "ember-font-awesome": "^4.0.0-rc.2",
+    "ember-highcharts": "^1.0.0",
     "ember-inflector": "^2.2.0",
     "ember-load-initializers": "^1.0.0",
     "ember-math-helpers": "^2.4.0",
@@ -87,6 +88,7 @@
     "eslint-plugin-prettier": "^2.5.0",
     "eyeglass": "^1.3.0",
     "eyeglass-restyle": "^1.1.0",
+    "highcharts": "^6.1.1",
     "husky": "^0.14.3",
     "ivy-tabs": "^3.1.0",
     "lint-staged": "^7.1.0",

wherehows-web/tests/integration/components/datasets/containers/dataset-health-test.js

@@ -0,0 +1,13 @@
+import { module, test } from 'qunit';
+import { setupRenderingTest } from 'ember-qunit';
+import { render } from '@ember/test-helpers';
+import hbs from 'htmlbars-inline-precompile';
+
+module('Integration | Component | datasets/containers/dataset-health', function(hooks) {
+  setupRenderingTest(hooks);
+  // TODO: More meaningful tests as we continue with development
+  test('it renders', async function(assert) {
+    await render(hbs`{{datasets/containers/dataset-health}}`);
+    assert.ok(this.element, 'Renders without errors');
+  });
+});

wherehows-web/tests/unit/constants/dataset-compliance-test.js

@@ -8,7 +8,8 @@ import {
   PurgePolicy,
   initialComplianceObjectFactory,
   isRecentSuggestion,
-  tagNeedsReview
+  tagNeedsReview,
+  lowQualitySuggestionConfidenceThreshold
 } from 'wherehows-web/constants';
 import complianceDataTypes from 'wherehows-web/mirage/fixtures/compliance-data-types';
 import { mockTimeStamps } from 'wherehows-web/tests/helpers/datasets/compliance-policy/recent-suggestions-constants';
@@ -17,6 +18,11 @@ import { hdfsUrn } from 'wherehows-web/mirage/fixtures/urn';
 
 module('Unit | Constants | dataset compliance');
 
+const complianceTagReviewOptions = {
+  checkSuggestions: false,
+  suggestionConfidenceThreshold: lowQualitySuggestionConfidenceThreshold
+};
+
 test('initialComplianceObjectFactory', function(assert) {
   assert.expect(2);
   const mockUrn = hdfsUrn;
@@ -55,14 +61,20 @@ test('isRecentSuggestion correctly determines if a suggestion is recent or not',
 test('tagNeedsReview exists', function(assert) {
   assert.ok(typeof tagNeedsReview === 'function', 'tagNeedsReview is a function');
 
-  assert.ok(typeof tagNeedsReview([])({}) === 'boolean', 'tagNeedsReview returns a boolean');
+  assert.ok(
+    typeof tagNeedsReview([], complianceTagReviewOptions)({}) === 'boolean',
+    'tagNeedsReview returns a boolean'
+  );
 });
 
 test('tagNeedsReview correctly determines if a fieldChangeSet requires review', function(assert) {
   assert.expect(mockFieldChangeSets.length);
 
   mockFieldChangeSets.forEach(changeSet =>
-    assert.ok(tagNeedsReview(complianceDataTypes)(changeSet) === changeSet.__requiresReview__, changeSet.__msg__)
+    assert.ok(
+      tagNeedsReview(complianceDataTypes, complianceTagReviewOptions)(changeSet) === changeSet.__requiresReview__,
+      changeSet.__msg__
+    )
   );
 });
 

wherehows-web/tests/unit/utils/datasets/compliance-suggestions-test.js

@@ -8,21 +8,30 @@ test('isHighConfidenceSuggestion correctly determines the confidence of a sugges
   let result = isHighConfidenceSuggestion({});
   assert.notOk(result, 'should be false if no arguments are supplied');
 
-  result = isHighConfidenceSuggestion({ confidenceLevel: lowQualitySuggestionConfidenceThreshold + 1 });
+  result = isHighConfidenceSuggestion(
+    { confidenceLevel: lowQualitySuggestionConfidenceThreshold + 1 },
+    lowQualitySuggestionConfidenceThreshold
+  );
 
   assert.ok(
     result,
     `should be true if the confidence value is greater than ${lowQualitySuggestionConfidenceThreshold}`
   );
 
-  result = isHighConfidenceSuggestion({ confidenceLevel: lowQualitySuggestionConfidenceThreshold - 1 });
+  result = isHighConfidenceSuggestion(
+    { confidenceLevel: lowQualitySuggestionConfidenceThreshold - 1 },
+    lowQualitySuggestionConfidenceThreshold
+  );
 
   assert.notOk(
     result,
     `should be false if the confidence value is less than ${lowQualitySuggestionConfidenceThreshold}`
   );
 
-  result = isHighConfidenceSuggestion({ confidenceLevel: lowQualitySuggestionConfidenceThreshold });
+  result = isHighConfidenceSuggestion(
+    { confidenceLevel: lowQualitySuggestionConfidenceThreshold },
+    lowQualitySuggestionConfidenceThreshold
+  );
 
   assert.notOk(
     result,
@@ -42,15 +51,17 @@ test('getTagSuggestions correctly extracts suggestions from a compliance field',
     suggestionAuthority: SuggestionIntent.accept
   };
 
-  let result = getTagSuggestions({});
+  let result = getTagSuggestions({ suggestionConfidenceThreshold: lowQualitySuggestionConfidenceThreshold })({});
 
   assert.ok(typeof result === 'undefined', 'expected undefined return when the argument is an empty object');
 
-  result = getTagSuggestions();
+  result = getTagSuggestions({ suggestionConfidenceThreshold: lowQualitySuggestionConfidenceThreshold })();
 
   assert.ok(typeof result === 'undefined', 'expected undefined return when no argument is supplied');
 
-  result = getTagSuggestions({ suggestion: changeSetField.suggestion });
+  result = getTagSuggestions({ suggestionConfidenceThreshold: lowQualitySuggestionConfidenceThreshold })({
+    suggestion: changeSetField.suggestion
+  });
 
   assert.deepEqual(
     result,

wherehows-web/yarn.lock

@@ -1382,6 +1382,10 @@ bootstrap-sass@^3.0.0:
   version "3.3.7"
   resolved "https://registry.yarnpkg.com/bootstrap-sass/-/bootstrap-sass-3.3.7.tgz#6596c7ab40f6637393323ab0bc80d064fc630498"
 
+bootstrap@3.3.7:
+  version "3.3.7"
+  resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-3.3.7.tgz#5a389394549f23330875a3b150656574f8a9eb71"
+
 bower-config@^1.3.0:
   version "1.4.1"
   resolved "https://registry.yarnpkg.com/bower-config/-/bower-config-1.4.1.tgz#85fd9df367c2b8dbbd0caa4c5f2bad40cd84c2cc"
@@ -1742,7 +1746,7 @@ broccoli-lint-eslint@^4.2.1:
     lodash.defaultsdeep "^4.6.0"
     md5-hex "^2.0.0"
 
-broccoli-merge-trees@^1.0.0, broccoli-merge-trees@^1.1.0, broccoli-merge-trees@^1.1.1, broccoli-merge-trees@^1.1.4:
+broccoli-merge-trees@^1.0.0, broccoli-merge-trees@^1.1.0, broccoli-merge-trees@^1.1.1, broccoli-merge-trees@^1.1.4, broccoli-merge-trees@^1.2.0:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/broccoli-merge-trees/-/broccoli-merge-trees-1.2.4.tgz#a001519bb5067f06589d91afa2942445a2d0fdb5"
   dependencies:
@@ -3516,6 +3520,16 @@ ember-hash-helper-polyfill@^0.1.1:
     ember-cli-babel "^5.1.7"
     ember-cli-version-checker "^1.2.0"
 
+ember-highcharts@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/ember-highcharts/-/ember-highcharts-1.0.0.tgz#d412af4d1f2f55e1cae0174c353852fd98b5bad9"
+  dependencies:
+    bootstrap "3.3.7"
+    broccoli-funnel "^2.0.1"
+    broccoli-merge-trees "^1.2.0"
+    ember-cli-babel "^6.6.0"
+    ember-cli-htmlbars "^2.0.1"
+
 ember-ignore-children-helper@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/ember-ignore-children-helper/-/ember-ignore-children-helper-1.0.1.tgz#f7c4aa17afb9c5685e1d4dcdb61c7b138ca7cdc3"
@@ -5170,6 +5184,10 @@ heimdalljs@^0.2.0, heimdalljs@^0.2.1, heimdalljs@^0.2.3, heimdalljs@^0.2.5:
   dependencies:
     rsvp "~3.2.1"
 
+highcharts@^6.1.1:
+  version "6.1.1"
+  resolved "https://registry.yarnpkg.com/highcharts/-/highcharts-6.1.1.tgz#49dc34f5e963744ecd7eb87603b6cdaf8304c13a"
+
 hoek@2.x.x:
   version "2.16.3"
   resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed"