diff --git a/wherehows-web/app/components/dataset-compliance-row.ts b/wherehows-web/app/components/dataset-compliance-row.ts index 27694bc6c7..f6fae05d8b 100644 --- a/wherehows-web/app/components/dataset-compliance-row.ts +++ b/wherehows-web/app/components/dataset-compliance-row.ts @@ -1,3 +1,4 @@ +import { action } from 'ember-decorators/object'; import { IComplianceChangeSet } from 'wherehows-web/components/dataset-compliance'; import DatasetTableRow from 'wherehows-web/components/dataset-table-row'; import ComputedProperty, { alias, bool } from '@ember/object/computed'; @@ -9,10 +10,13 @@ import { getDefaultSecurityClassification, IComplianceFieldFormatOption, IComplianceFieldIdentifierOption, - IFieldIdentifierOption + IFieldIdentifierOption, + fieldChangeSetRequiresReview, + isFieldIdType, + changeSetReviewableAttributeTriggers, + idTypeFieldHasLogicalType } from 'wherehows-web/constants'; import { IComplianceDataType } from 'wherehows-web/typings/api/list/compliance-datatypes'; -import { fieldChangeSetRequiresReview } from 'wherehows-web/utils/datasets/compliance-policy'; import { getFieldSuggestions } from 'wherehows-web/utils/datasets/compliance-suggestions'; import noop from 'wherehows-web/utils/noop'; import { hasEnumerableKeys } from 'wherehows-web/utils/object'; @@ -139,10 +143,19 @@ export default class DatasetComplianceRow extends DatasetTableRow { * @type {ComputedProperty} * @memberof DatasetComplianceRow */ - isReviewRequested = computed('field.{isDirty,suggestion,privacyPolicyExists,suggestionAuthority}', function( + isReviewRequested = computed(`field.{${changeSetReviewableAttributeTriggers}}`, 'complianceDataTypes', function( this: DatasetComplianceRow ): boolean { - return fieldChangeSetRequiresReview(get(this, 'field')); + return fieldChangeSetRequiresReview(get(this, 'complianceDataTypes'))(get(this, 'field')); + }); + + /** + * Checks if the field format / logical type for this field if missing if the field is of ID type + * @type {ComputedProperty} + * @memberof DatasetComplianceRow + */ + isFieldFormatMissing = computed('isIdType', 'field.logicalType', function(): boolean { + return get(this, 'isIdType') && !idTypeFieldHasLogicalType(get(this, 'field')); }); /** @@ -245,7 +258,9 @@ export default class DatasetComplianceRow extends DatasetTableRow { * @type ComputedProperty> * @memberof DatasetComplianceRow */ - fieldFormats = computed('isIdType', function(this: DatasetComplianceRow): Array { + fieldFormats = computed('isIdType', 'complianceDataTypes', function( + this: DatasetComplianceRow + ): Array { const identifierType = get(this, 'field')['identifierType'] || ''; const { isIdType, complianceDataTypes } = getProperties(this, ['isIdType', 'complianceDataTypes']); const complianceDataType = complianceDataTypes.findBy('id', identifierType); @@ -268,11 +283,11 @@ export default class DatasetComplianceRow extends DatasetTableRow { * @type {ComputedProperty} * @memberof DatasetComplianceRow */ - isIdType: ComputedProperty = computed('field.identifierType', function(this: DatasetComplianceRow): boolean { - const { field: { identifierType }, complianceDataTypes } = getProperties(this, ['field', 'complianceDataTypes']); - const { idType } = complianceDataTypes.findBy('id', identifierType || '') || { idType: false }; - - return idType; + isIdType: ComputedProperty = computed('field.identifierType', 'complianceDataTypes', function( + this: DatasetComplianceRow + ): boolean { + const { field, complianceDataTypes } = getProperties(this, ['field', 'complianceDataTypes']); + return isFieldIdType(complianceDataTypes)(field); }); /** @@ -339,77 +354,86 @@ export default class DatasetComplianceRow extends DatasetTableRow { return getFieldSuggestions(getWithDefault(this, 'field', {})); }); - actions = { - /** - * Handles UI changes to the field identifierType - * @param {{ value: ComplianceFieldIdValue }} { value } - */ - onFieldIdentifierTypeChange(this: DatasetComplianceRow, { value }: { value: ComplianceFieldIdValue | null }) { - const onFieldIdentifierTypeChange = get(this, 'onFieldIdentifierTypeChange'); - if (typeof onFieldIdentifierTypeChange === 'function') { - onFieldIdentifierTypeChange(get(this, 'field'), { value }); - } - }, - - /** - * Handles the updates when the field logical type changes on this field - * @param {(IComplianceChangeSet['logicalType'])} value contains the selected drop-down value - */ - onFieldLogicalTypeChange(this: DatasetComplianceRow, { value }: { value: IComplianceChangeSet['logicalType'] }) { - const onFieldLogicalTypeChange = get(this, 'onFieldLogicalTypeChange'); - if (typeof onFieldLogicalTypeChange === 'function') { - onFieldLogicalTypeChange(get(this, 'field'), value); - } - }, - - /** - * Handles UI change to field security classification - * @param {({ value: '' | Classification })} { value } contains the changed classification value - */ - onFieldClassificationChange(this: DatasetComplianceRow, { value }: { value: '' | Classification }) { - const onFieldClassificationChange = get(this, 'onFieldClassificationChange'); - if (typeof onFieldClassificationChange === 'function') { - onFieldClassificationChange(get(this, 'field'), { value }); - } - }, - - /** - * Handles the nonOwner flag update on the field - * @param {boolean} nonOwner - */ - onOwnerChange(this: DatasetComplianceRow, nonOwner: boolean) { - get(this, 'onFieldOwnerChange')(get(this, 'field'), nonOwner); - }, - - /** - * Handler for user interactions with a suggested value. Applies / ignores the suggestion - * Then invokes the parent supplied suggestion handler - * @param {string | void} intent a binary indicator to accept or ignore suggestion - * @param {SuggestionIntent} intent - */ - onSuggestionAction(this: DatasetComplianceRow, intent?: SuggestionIntent) { - const onSuggestionIntent = get(this, 'onSuggestionIntent'); - - // Accept the suggestion for either identifierType and/or logicalType - if (intent === SuggestionIntent.accept) { - const { identifierType, logicalType } = get(this, 'prediction') || { - identifierType: void 0, - logicalType: void 0 - }; - - if (identifierType) { - this.actions.onFieldIdentifierTypeChange.call(this, { value: identifierType }); - } - - if (logicalType) { - this.actions.onFieldLogicalTypeChange.call(this, logicalType); - } + /** + * Handles UI changes to the field identifierType + * @param {{ value: ComplianceFieldIdValue }} { value } + */ + @action + fieldIdentifierTypeDidChange(this: DatasetComplianceRow, { value }: { value: ComplianceFieldIdValue | null }) { + const onFieldIdentifierTypeChange = get(this, 'onFieldIdentifierTypeChange'); + if (typeof onFieldIdentifierTypeChange === 'function') { + // if the field has a predicted value, but the user changes the identifier type, + // ignore the suggestion + if (get(this, 'prediction')) { + this.onSuggestionAction(SuggestionIntent.ignore); } - // Invokes parent handle to runtime ignore future suggesting this suggestion - if (typeof onSuggestionIntent === 'function') { - onSuggestionIntent(get(this, 'field'), intent); + onFieldIdentifierTypeChange(get(this, 'field'), { value }); + } + } + + /** + * Handles the updates when the field logical type changes on this field + * @param {(IComplianceChangeSet['logicalType'])} value contains the selected drop-down value + */ + @action + fieldLogicalTypeDidChange(this: DatasetComplianceRow, { value }: { value: IComplianceChangeSet['logicalType'] }) { + const onFieldLogicalTypeChange = get(this, 'onFieldLogicalTypeChange'); + if (typeof onFieldLogicalTypeChange === 'function') { + onFieldLogicalTypeChange(get(this, 'field'), value); + } + } + + /** + * Handles UI change to field security classification + * @param {({ value: '' | Classification })} { value } contains the changed classification value + */ + @action + fieldClassificationDidChange(this: DatasetComplianceRow, { value }: { value: '' | Classification }) { + const onFieldClassificationChange = get(this, 'onFieldClassificationChange'); + if (typeof onFieldClassificationChange === 'function') { + onFieldClassificationChange(get(this, 'field'), { value }); + } + } + + /** + * Handles the nonOwner flag update on the field + * @param {boolean} nonOwner + */ + @action + onOwnerChange(this: DatasetComplianceRow, nonOwner: boolean) { + get(this, 'onFieldOwnerChange')(get(this, 'field'), nonOwner); + } + + /** + * Handler for user interactions with a suggested value. Applies / ignores the suggestion + * Then invokes the parent supplied suggestion handler + * @param {string | void} intent a binary indicator to accept or ignore suggestion + * @param {SuggestionIntent} intent + */ + @action + onSuggestionAction(this: DatasetComplianceRow, intent?: SuggestionIntent) { + const onSuggestionIntent = get(this, 'onSuggestionIntent'); + + // Accept the suggestion for either identifierType and/or logicalType + if (intent === SuggestionIntent.accept) { + const { identifierType, logicalType } = get(this, 'prediction') || { + identifierType: void 0, + logicalType: void 0 + }; + + if (identifierType) { + this.actions.fieldIdentifierTypeDidChange.call(this, { value: identifierType }); + } + + if (logicalType) { + this.actions.fieldLogicalTypeDidChange.call(this, logicalType); } } - }; + + // Invokes parent handle to runtime ignore future suggesting this suggestion + if (typeof onSuggestionIntent === 'function') { + onSuggestionIntent(get(this, 'field'), intent); + } + } } diff --git a/wherehows-web/app/components/dataset-compliance.ts b/wherehows-web/app/components/dataset-compliance.ts index 60c8abfa91..f796c83513 100644 --- a/wherehows-web/app/components/dataset-compliance.ts +++ b/wherehows-web/app/components/dataset-compliance.ts @@ -4,6 +4,7 @@ import ComputedProperty, { not, or } from '@ember/object/computed'; import { run, schedule, next } from '@ember/runloop'; import { inject } from '@ember/service'; import { classify } from '@ember/string'; +import { assert } from '@ember/debug'; import { IFieldIdentifierOption, ISecurityClassificationOption } from 'wherehows-web/constants/dataset-compliance'; import { IDatasetView } from 'wherehows-web/typings/api/datasets/dataset'; import { IDataPlatform } from 'wherehows-web/typings/api/list/platforms'; @@ -16,7 +17,6 @@ import { getDefaultSecurityClassification, compliancePolicyStrings, getComplianceSteps, - hiddenTrackingFields, isExempt, ComplianceFieldIdValue, IComplianceFieldIdentifierOption, @@ -24,18 +24,19 @@ import { DatasetClassification, SuggestionIntent, PurgePolicy, - getSupportedPurgePolicies -} from 'wherehows-web/constants'; -import { - isPolicyExpectedShape, - fieldChangeSetRequiresReview, + getSupportedPurgePolicies, mergeMappedColumnFieldsWithSuggestions, - getFieldsRequiringReview -} from 'wherehows-web/utils/datasets/compliance-policy'; + getFieldsRequiringReview, + isFieldIdType, + idTypeFieldsHaveLogicalType, + changeSetFieldsRequiringReview, + changeSetReviewableAttributeTriggers +} from 'wherehows-web/constants'; +import { isPolicyExpectedShape } from 'wherehows-web/utils/datasets/compliance-policy'; import scrollMonitor from 'scrollmonitor'; import { getFieldsSuggestions } from 'wherehows-web/utils/datasets/compliance-suggestions'; import { hasEnumerableKeys } from 'wherehows-web/utils/object'; -import { arrayFilter, isListUnique } from 'wherehows-web/utils/array'; +import { compact, isListUnique } from 'wherehows-web/utils/array'; import noop from 'wherehows-web/utils/noop'; import { IComplianceDataType } from 'wherehows-web/typings/api/list/compliance-datatypes'; import Notifications, { NotificationEvent, IConfirmOptions } from 'wherehows-web/services/notifications'; @@ -67,7 +68,9 @@ type SchemaFieldToPolicyValue = Pick< IComplianceEntity, 'identifierField' | 'identifierType' | 'logicalType' | 'securityClassification' | 'nonOwner' | 'readonly' > & { + // flag indicating that the field has a current policy upstream privacyPolicyExists: boolean; + // flag indicating the field changeSet has been modified on the client isDirty: boolean; policyModificationTime: IComplianceInfo['modifiedTime']; dataType: string; @@ -77,7 +80,7 @@ type SchemaFieldToPolicyValue = Pick< * Describes the interface for a mapping of field names to type, SchemaFieldToPolicyValue * @interface ISchemaFieldsToPolicy */ -interface ISchemaFieldsToPolicy { +export interface ISchemaFieldsToPolicy { [fieldName: string]: SchemaFieldToPolicyValue; } @@ -97,7 +100,7 @@ type SchemaFieldToSuggestedValue = Pick< * Describes the mapping of attributes to value types for a datasets schema field names to suggested property values * @interface ISchemaFieldsToSuggested */ -interface ISchemaFieldsToSuggested { +export interface ISchemaFieldsToSuggested { [fieldName: string]: SchemaFieldToSuggestedValue; } /** @@ -125,14 +128,6 @@ const { missingDatasetSecurityClassification } = compliancePolicyStrings; -/** - * Takes a list of compliance data types and maps a list of compliance id's with idType set to true - * @param {Array} [complianceDataTypes=[]] the list of compliance data types to transform - * @return {Array} - */ -const getIdTypeDataTypes = (complianceDataTypes: Array = []) => - complianceDataTypes.filter(complianceDataType => complianceDataType.idType).mapBy('id'); - /** * String constant referencing the datasetClassification on the privacy policy * @type {string} @@ -150,12 +145,6 @@ const datasetClassifiersKeys = >Object.ke */ const policyComplianceEntitiesKey = 'complianceInfo.complianceEntities'; -/** - * Returns a list of changeSet fields that requires user attention - * @type {function({}): Array<{ isDirty, suggestion, privacyPolicyExists, suggestionAuthority }>} - */ -const changeSetFieldsRequiringReview = arrayFilter(fieldChangeSetRequiresReview); - /** * The initial state of the compliance step for a zero based array * @type {number} @@ -173,8 +162,6 @@ export default class DatasetCompliance extends Component { watchers: Array<{ stateChange: (fn: () => void) => void; watchItem: Element; destroy?: Function }>; complianceWatchers: WeakMap; _hasBadData: boolean; - _message: string; - _alertType: string; platform: IDatasetView['platform']; isCompliancePolicyAvailable: boolean = false; showAllDatasetMemberData: boolean; @@ -202,12 +189,6 @@ export default class DatasetCompliance extends Component { */ notifications: ComputedProperty = inject(); - /** - * @type {Handlebars.SafeStringStatic} - * @memberof DatasetCompliance - */ - hiddenTrackingFields = hiddenTrackingFields; - /** * Flag indicating that the related dataset is schemaless or has a schema * @type {boolean} @@ -304,23 +285,39 @@ export default class DatasetCompliance extends Component { this: DatasetCompliance ): Array> { type NoneAndUnspecifiedOptions = Array>; + // object with interface IComplianceDataType and an index number indicative of position + type IndexedComplianceDataType = IComplianceDataType & { index: number }; const noneAndUnSpecifiedDropdownOptions: NoneAndUnspecifiedOptions = [ { value: null, label: 'Select Field Type...', isDisabled: true }, { value: ComplianceFieldIdValue.None, label: 'None' } ]; - const dataTypes = get(this, 'complianceDataTypes') || []; + // Creates a list of IComplianceDataType each with an index. The intent here is to perform a stable sort on + // the items in the list, Array#sort is not stable, so for items that equal on the primary comparator + // break the tie based on position in original list + const indexedDataTypes: Array = (get(this, 'complianceDataTypes') || []).map( + (type, index) => ({ + ...type, + index + }) + ); /** - * Compares each compliance data type + * Compares each compliance data type, ensure that positional order is maintained * @param {IComplianceDataType} a the compliance type to compare * @param {IComplianceDataType} b the other * @returns {number} 0, 1, -1 indicating sort order */ - const dataTypeComparator = (a: IComplianceDataType, b: IComplianceDataType): number => + const dataTypeComparator = (a: IndexedComplianceDataType, b: IndexedComplianceDataType): number => { + const { idType: aIdType, index: aIndex } = a; + const { idType: bIdType, index: bIndex } = b; // Convert boolean values to number type + const typeCompare = Number(aIdType) - Number(bIdType); + // True types first, hence negation - -(Number(a.idType) - Number(b.idType)); + // If types are same, then sort on original position i.e stable sort + return typeCompare ? -typeCompare : aIndex - bIndex; + }; /** * Inserts a divider in the list of compliance field identifier dropdown options @@ -343,7 +340,7 @@ export default class DatasetCompliance extends Component { return [ ...noneAndUnSpecifiedDropdownOptions, - ...insertDivider(getFieldIdentifierOptions(dataTypes.sort(dataTypeComparator))) + ...insertDivider(getFieldIdentifierOptions(indexedDataTypes.sort(dataTypeComparator))) ]; }); @@ -757,42 +754,70 @@ export default class DatasetCompliance extends Component { * @type {ComputedProperty} * @memberof DatasetCompliance */ - compliancePolicyChangeSet = computed('columnIdFieldsToCurrentPrivacyPolicy', function( - this: DatasetCompliance - ): Array { - // schemaFieldNamesMappedToDataTypes is a dependency for cp columnIdFieldsToCurrentPrivacyPolicy, so no need to dep on that directly - // TODO: move source to TS - const changeSet = mergeMappedColumnFieldsWithSuggestions( - get(this, 'columnIdFieldsToCurrentPrivacyPolicy'), - get(this, 'identifierFieldToSuggestion') - ); + compliancePolicyChangeSet = computed( + 'columnIdFieldsToCurrentPrivacyPolicy', + 'complianceDataTypes', + 'identifierFieldToSuggestion', + function(this: DatasetCompliance): Array { + // schemaFieldNamesMappedToDataTypes is a dependency for CP columnIdFieldsToCurrentPrivacyPolicy, so no need to dep on that directly + const changeSet = mergeMappedColumnFieldsWithSuggestions( + get(this, 'columnIdFieldsToCurrentPrivacyPolicy'), + get(this, 'identifierFieldToSuggestion') + ); - run(() => next(this, 'notifyHandlerOfSuggestions', changeSet)); - run(() => next(this, 'notifyHandlerOfFieldsRequiringReview', changeSet)); + // pass current changeSet state to parent handlers + run(() => next(this, 'notifyHandlerOfSuggestions', changeSet)); + run(() => next(this, 'notifyHandlerOfFieldsRequiringReview', changeSet, get(this, 'complianceDataTypes'))); - return changeSet; - }); + return changeSet; + } + ); /** * Returns a list of changeSet fields that meets the user selected filter criteria * @type {ComputedProperty} * @memberof DatasetCompliance */ - filteredChangeSet = computed('changeSetReviewCount', 'fieldReviewOption', 'compliancePolicyChangeSet', function( - this: DatasetCompliance - ): Array { - const changeSet = get(this, 'compliancePolicyChangeSet'); + filteredChangeSet = computed( + 'changeSetReviewCount', + 'fieldReviewOption', + 'compliancePolicyChangeSet', + 'complianceDataTypes', + function(this: DatasetCompliance): Array { + const { compliancePolicyChangeSet: changeSet, complianceDataTypes } = getProperties(this, [ + 'compliancePolicyChangeSet', + 'complianceDataTypes' + ]); - return get(this, 'fieldReviewOption') === 'showReview' ? changeSetFieldsRequiringReview(changeSet) : changeSet; - }); + return get(this, 'fieldReviewOption') === 'showReview' + ? changeSetFieldsRequiringReview(complianceDataTypes)(changeSet) + : changeSet; + } + ); - notifyHandlerOfSuggestions = (changeSet: Array) => { - const hasChangeSetSuggestions = getFieldsSuggestions(changeSet).some(suggestion => !!suggestion); + /** + * Invokes external action with flag indicating that at least 1 suggestion exists for a field in the changeSet + * @param {Array} changeSet + */ + notifyHandlerOfSuggestions = (changeSet: Array): void => { + const hasChangeSetSuggestions = !!compact(getFieldsSuggestions(changeSet)).length; this.notifyOnChangeSetSuggestions(hasChangeSetSuggestions); }; - notifyHandlerOfFieldsRequiringReview = (changeSet: Array) => { - const hasChangeSetDrift = getFieldsRequiringReview(changeSet).some((isReviewRequired: boolean) => isReviewRequired); + /** + * Invokes external action with flag indicating that a field in the changeSet requires user review + * @param {Array} complianceDataTypes + * @param {Array} changeSet + */ + notifyHandlerOfFieldsRequiringReview = ( + complianceDataTypes: Array, + changeSet: Array + ) => { + // adding assertions for run-loop callback invocation, because static type checks are bypassed + assert('expected complianceDataTypes to be of type `array`', Array.isArray(complianceDataTypes)); + assert('expected changeSet to be of type `array`', Array.isArray(changeSet)); + + const hasChangeSetDrift = !!getFieldsRequiringReview(complianceDataTypes)(changeSet).length; this.notifyOnChangeSetRequiresReview(hasChangeSetDrift); }; @@ -802,25 +827,14 @@ export default class DatasetCompliance extends Component { * @memberof DatasetCompliance */ changeSetReviewCount = computed( - 'compliancePolicyChangeSet.@each.{isDirty,suggestion,privacyPolicyExists,suggestionAuthority}', + `compliancePolicyChangeSet.@each.{${changeSetReviewableAttributeTriggers}}`, + 'complianceDataTypes', function(this: DatasetCompliance): number { - return changeSetFieldsRequiringReview(get(this, 'compliancePolicyChangeSet')).length; + return changeSetFieldsRequiringReview(get(this, 'complianceDataTypes'))(get(this, 'compliancePolicyChangeSet')) + .length; } ); - /** - * TODO:DSS-6719 refactor into mixin - * Clears recently shown user messages - * @returns {(Pick)} - * @memberof DatasetCompliance - */ - clearMessages(this: DatasetCompliance): Pick { - return setProperties(this, { - _message: '', - _alertType: '' - }); - } - /** * Sets the default classification for the given identifier field * Using the identifierType, determine the field's default security classification based on a values @@ -928,15 +942,12 @@ export default class DatasetCompliance extends Component { validateFields(this: DatasetCompliance) { const { notify } = get(this, 'notifications'); const { complianceEntities = [] } = get(this, 'complianceInfo') || {}; - const idTypeIdentifiers = getIdTypeDataTypes(get(this, 'complianceDataTypes')); - const idTypeComplianceEntities = complianceEntities.filter(({ identifierType }) => - idTypeIdentifiers.includes(identifierType) - ); + const idTypeComplianceEntities = complianceEntities.filter(isFieldIdType(get(this, 'complianceDataTypes'))); // Validation operations - const idFieldsHaveValidLogicalType = idTypeComplianceEntities.every(({ logicalType }) => !!logicalType); - const fieldIdentifiersAreUnique = isListUnique(complianceEntities.mapBy('identifierField')); - const schemaFieldLengthGreaterThanComplianceEntities = this.isSchemaFieldLengthGreaterThanComplianceEntities(); + const idFieldsHaveValidLogicalType: boolean = idTypeFieldsHaveLogicalType(idTypeComplianceEntities); + const fieldIdentifiersAreUnique: boolean = isListUnique(complianceEntities.mapBy('identifierField')); + const schemaFieldLengthGreaterThanComplianceEntities: boolean = this.isSchemaFieldLengthGreaterThanComplianceEntities(); if (!fieldIdentifiersAreUnique) { notify(NotificationEvent.error, { content: complianceFieldNotUnique }); @@ -1306,8 +1317,6 @@ export default class DatasetCompliance extends Component { 'identifierField', identifierField ); - // TODO:DSS-6719 refactor into mixin - this.clearMessages(); // Apply the updated classification value to the current instance of the field in working copy if (currentFieldInComplianceList) { diff --git a/wherehows-web/app/constants/dataset-compliance.ts b/wherehows-web/app/constants/dataset-compliance.ts index 980f6680e5..9e47d8bc45 100644 --- a/wherehows-web/app/constants/dataset-compliance.ts +++ b/wherehows-web/app/constants/dataset-compliance.ts @@ -1,12 +1,17 @@ -import Ember from 'ember'; +import { + IComplianceChangeSet, + ISchemaFieldsToPolicy, + ISchemaFieldsToSuggested +} from 'wherehows-web/components/dataset-compliance'; import { Classification, ComplianceFieldIdValue, IdLogicalType } from 'wherehows-web/constants/datasets/compliance'; import { PurgePolicy } from 'wherehows-web/constants/index'; import { IComplianceEntity, IComplianceInfo } from 'wherehows-web/typings/api/datasets/compliance'; import { IComplianceDataType } from 'wherehows-web/typings/api/list/compliance-datatypes'; -import { arrayFilter, arrayMap } from 'wherehows-web/utils/array'; +import { arrayEvery, arrayFilter, arrayMap } from 'wherehows-web/utils/array'; import { fleece } from 'wherehows-web/utils/object'; - -const { String: { htmlSafe } } = Ember; +import { lastSeenSuggestionInterval } from 'wherehows-web/constants/metadata-acquisition'; +import { pick } from 'lodash'; +import { decodeUrn } from 'wherehows-web/utils/validators/urn'; /** * Defines the generic interface field identifier drop downs @@ -64,6 +69,14 @@ const compliancePolicyStrings = { missingDatasetSecurityClassification: 'Please specify a security classification for this dataset.' }; +/** + * Field / changeSet attributes that will trigger a check if review is requested + * field `logicalType` in `changeSetReviewableAttributeTriggers` is used in the determination of idType fields + * without a logicalType as requiring review + * @type {string} + */ +const changeSetReviewableAttributeTriggers = 'isDirty,suggestion,privacyPolicyExists,suggestionAuthority,logicalType'; + /** * Takes a compliance data type and transforms it into a compliance field identifier option * @param {IComplianceDataType} complianceDataType @@ -80,17 +93,6 @@ const getFieldIdentifierOption = (complianceDataType: IComplianceDataType): ICom */ const getFieldIdentifierOptions = arrayMap(getFieldIdentifierOption); -/** - * Defines the html string for informing the user of hidden tracking fields - * @type {Ember.String.htmlSafe} - */ -const hiddenTrackingFields = htmlSafe( - '

Some fields in this dataset have been hidden from the table(s) below. ' + - "These are tracking fields for which we've been able to predetermine the compliance classification.

" + - '

For example: header.memberId, requestHeader. ' + - 'Hopefully, this saves you some scrolling!

' -); - /** * Defines the sequence of edit steps in the compliance policy component */ @@ -160,12 +162,142 @@ const isAutoGeneratedPolicy = (policy?: IComplianceInfo): boolean => { return false; }; +/** + * Takes a list of compliance data types and maps a list of compliance id's with idType set to true + * @param {Array} [complianceDataTypes=[]] the list of compliance data types to transform + * @return {Array} + */ +const getIdTypeDataTypes = (complianceDataTypes: Array = []): Array => + complianceDataTypes.filter(complianceDataType => complianceDataType.idType).mapBy('id'); + +/** + * Checks if the compliance suggestion has a date that is equal or exceeds the policy mod time by at least the + * ms time in lastSeenSuggestionInterval + * @param {string} [policyModificationTime = 0] timestamp for the policy modification date + * @param {number} suggestionModificationTime timestamp for the suggestion modification date + * @return {boolean} + */ +const isRecentSuggestion = (policyModificationTime: string = '0', suggestionModificationTime: number) => + !!suggestionModificationTime && + suggestionModificationTime - parseInt(policyModificationTime) >= lastSeenSuggestionInterval; + +/** + * Checks if a compliance policy changeSet field requires user attention: if a suggestion + * is available but the user has not indicated intent or a policy for the field does not currently exist remotely + * and the related field changeSet has not been modified on the client and isn't readonly + * @param {boolean} isDirty + * @return {boolean} + */ +/** + * + * @param {Array} complianceDataTypes + * @return {(changeSet: IComplianceChangeSet) => boolean} + */ +const fieldChangeSetRequiresReview = (complianceDataTypes: Array) => + /** + * + * @param {IComplianceChangeSet} changeSet + * @return {boolean} + */ + (changeSet: IComplianceChangeSet): boolean => { + const { isDirty, suggestion, privacyPolicyExists, suggestionAuthority, readonly } = changeSet; + let isReviewRequired = false; + + if (readonly) { + return false; + } + + if (suggestion) { + isReviewRequired = isReviewRequired || !suggestionAuthority; + } + + if (isFieldIdType(complianceDataTypes)(changeSet)) { + isReviewRequired = isReviewRequired || !idTypeFieldHasLogicalType(changeSet); + } + // If either the privacy policy doesn't exists, or user hasn't made changes, then review is required + return isReviewRequired || !(privacyPolicyExists || isDirty); + }; + +const isFieldIdType = (complianceDataTypes: Array = []) => ({ + identifierType +}: IComplianceChangeSet): boolean => getIdTypeDataTypes(complianceDataTypes).includes(identifierType); + +const idTypeFieldHasLogicalType = ({ logicalType }: IComplianceEntity): boolean => !!logicalType; + +const idTypeFieldsHaveLogicalType = arrayEvery(idTypeFieldHasLogicalType); +/** + * Gets the fields requiring review + * @type {(array: Array) => Array} + */ +const getFieldsRequiringReview = (complianceDataTypes: Array) => + arrayMap(fieldChangeSetRequiresReview(complianceDataTypes)); + +/** + * Returns a list of changeSet fields that requires user attention + * @type {function({}): Array<{ isDirty, suggestion, privacyPolicyExists, suggestionAuthority }>} + */ +const changeSetFieldsRequiringReview = (complianceDataTypes: Array) => + arrayFilter(fieldChangeSetRequiresReview(complianceDataTypes)); + +/** + * Merges the column fields with the suggestion for the field if available + * @param {object} mappedColumnFields a map of column fields to compliance entity properties + * @param {object} fieldSuggestionMap a map of field suggestion properties keyed by field name + * @return {Array} mapped column field augmented with suggestion if available + */ +const mergeMappedColumnFieldsWithSuggestions = ( + mappedColumnFields: ISchemaFieldsToPolicy = {}, + fieldSuggestionMap: ISchemaFieldsToSuggested = {} +): Array => + Object.keys(mappedColumnFields).map(fieldName => { + const field = pick(mappedColumnFields[fieldName], [ + 'identifierField', + 'dataType', + 'identifierType', + 'logicalType', + 'securityClassification', + 'policyModificationTime', + 'privacyPolicyExists', + 'isDirty', + 'nonOwner', + 'readonly' + ]); + const { identifierField, policyModificationTime } = field; + const suggestion = fieldSuggestionMap[identifierField]; + + // If a suggestion exists for this field add the suggestion attribute to the field properties / changeSet + // Check if suggestion isRecent before augmenting, otherwise, suggestion will not be considered on changeSet + if (suggestion && isRecentSuggestion(policyModificationTime, suggestion.suggestionsModificationTime)) { + return { ...field, suggestion }; + } + + return field; + }); + +/** + * Builds a default shape for securitySpecification & privacyCompliancePolicy with default / unset values + * for non null properties as per Avro schema + * @param {string} datasetId identifier for the dataset that this privacy object applies to + */ +const createInitialComplianceInfo = (datasetId: string): IComplianceInfo => { + const identifier = typeof datasetId === 'string' ? { datasetUrn: decodeUrn(datasetId) } : { datasetId }; + + return { + ...identifier, + datasetId: null, + confidentiality: null, + complianceType: '', + compliancePurgeNote: '', + complianceEntities: [], + datasetClassification: null + }; +}; + export { compliancePolicyStrings, getFieldIdentifierOption, getFieldIdentifierOptions, complianceSteps, - hiddenTrackingFields, getComplianceSteps, filterEditableEntities, isAutoGeneratedPolicy, @@ -173,5 +305,16 @@ export { IComplianceFieldIdentifierOption, IComplianceFieldFormatOption, ISecurityClassificationOption, - IFieldIdentifierOption + IFieldIdentifierOption, + fieldChangeSetRequiresReview, + isFieldIdType, + mergeMappedColumnFieldsWithSuggestions, + isRecentSuggestion, + getFieldsRequiringReview, + createInitialComplianceInfo, + getIdTypeDataTypes, + idTypeFieldHasLogicalType, + idTypeFieldsHaveLogicalType, + changeSetFieldsRequiringReview, + changeSetReviewableAttributeTriggers }; diff --git a/wherehows-web/app/controllers/datasets/dataset.js b/wherehows-web/app/controllers/datasets/dataset.js index 2f24c022ee..897dbd8242 100644 --- a/wherehows-web/app/controllers/datasets/dataset.js +++ b/wherehows-web/app/controllers/datasets/dataset.js @@ -13,7 +13,6 @@ import { } from 'wherehows-web/utils/api'; import { encodeUrn } from 'wherehows-web/utils/validators/urn'; import { updateDatasetDeprecation } from 'wherehows-web/utils/api/datasets/properties'; -import { readDatasetOwners, updateDatasetOwners } from 'wherehows-web/utils/api/datasets/owners'; import { Tabs } from 'wherehows-web/constants/datasets/shared'; import { action } from 'ember-decorators/object'; import Notifications from 'wherehows-web/services/notifications'; diff --git a/wherehows-web/app/styles/components/dataset-compliance/_compliance-table.scss b/wherehows-web/app/styles/components/dataset-compliance/_compliance-table.scss index 7cee05ad8c..39e725cb3c 100644 --- a/wherehows-web/app/styles/components/dataset-compliance/_compliance-table.scss +++ b/wherehows-web/app/styles/components/dataset-compliance/_compliance-table.scss @@ -62,6 +62,21 @@ color: $compliance-ok-color; } } + + &--missing-selection { + &#{&}#{&} { + $invalid-color: get-color(red7, 0.6); + + background-color: $invalid-color; + border-color: $invalid-color; + color: get-color(white); + + &::after { + content: '?'; + color: get-color(white); + } + } + } } .compliance-depends { diff --git a/wherehows-web/app/templates/components/dataset-compliance-row.hbs b/wherehows-web/app/templates/components/dataset-compliance-row.hbs index caf57d45d4..02b9021ff1 100644 --- a/wherehows-web/app/templates/components/dataset-compliance-row.hbs +++ b/wherehows-web/app/templates/components/dataset-compliance-row.hbs @@ -16,9 +16,10 @@ identifierTypeBeforeSuggestion=identifierTypeBeforeSuggestion logicalTypeBeforeSuggestion=logicalTypeBeforeSuggestion isReviewRequested=isReviewRequested - onFieldIdentifierTypeChange=(action 'onFieldIdentifierTypeChange') - onFieldClassificationChange=(action 'onFieldClassificationChange') - onFieldLogicalTypeChange=(action 'onFieldLogicalTypeChange') + isFieldFormatMissing=isFieldFormatMissing + fieldIdentifierTypeDidChange=(action 'fieldIdentifierTypeDidChange') + fieldClassificationDidChange=(action 'fieldClassificationDidChange') + fieldLogicalTypeDidChange=(action 'fieldLogicalTypeDidChange') onSuggestionAction=(action 'onSuggestionAction') onOwnerChange=(action 'onOwnerChange') )}} diff --git a/wherehows-web/app/templates/datasets/dataset-compliance/-dataset-compliance-entities.hbs b/wherehows-web/app/templates/datasets/dataset-compliance/-dataset-compliance-entities.hbs index e59d31fd33..e8804f1528 100644 --- a/wherehows-web/app/templates/datasets/dataset-compliance/-dataset-compliance-entities.hbs +++ b/wherehows-web/app/templates/datasets/dataset-compliance/-dataset-compliance-entities.hbs @@ -92,7 +92,7 @@ {{else}} {{#if (and row.suggestion (not row.suggestionResolution))}} - + {{tooltip-on-element text="Has suggestions" }} @@ -102,9 +102,9 @@ {{#if row.isReviewRequested}} - + {{tooltip-on-element - text="New field" + text="Please review" }} @@ -167,7 +167,7 @@ disabled=(or (not isEditing) row.isReadonly) values=complianceFieldIdDropdownOptions selected=(readonly row.identifierType) - selectionDidChange=(action row.onFieldIdentifierTypeChange) + selectionDidChange=(action row.fieldIdentifierTypeDidChange) }} {{#if row.identifierTypeBeforeSuggestion}} @@ -194,7 +194,8 @@ disabled=(or (not isEditing) row.isReadonly) values=row.fieldFormats selected=(readonly row.logicalType) - selectionDidChange=(action row.onFieldLogicalTypeChange) + selectionDidChange=(action row.fieldLogicalTypeDidChange) + class=(if row.isFieldFormatMissing "dataset-compliance-fields--missing-selection") }} {{#if row.logicalTypeBeforeSuggestion}} @@ -239,7 +240,7 @@ disabled=(or (not isEditing) row.isReadonly) values=classifiers selected=row.classification - selectionDidChange=(action row.onFieldClassificationChange) + selectionDidChange=(action row.fieldClassificationDidChange) }} {{/row.cell}} diff --git a/wherehows-web/app/utils/datasets/compliance-policy.js b/wherehows-web/app/utils/datasets/compliance-policy.js index 2373d10d5f..75112cdf5e 100644 --- a/wherehows-web/app/utils/datasets/compliance-policy.js +++ b/wherehows-web/app/utils/datasets/compliance-policy.js @@ -1,25 +1,5 @@ import { DatasetClassifiers } from 'wherehows-web/constants/dataset-classification'; -import { lastSeenSuggestionInterval } from 'wherehows-web/constants/metadata-acquisition'; import { assert, warn } from '@ember/debug'; -import { decodeUrn } from 'wherehows-web/utils/validators/urn'; -import { arrayMap } from 'wherehows-web/utils/array'; - -/** - * Builds a default shape for securitySpecification & privacyCompliancePolicy with default / unset values - * for non null properties as per Avro schema - * @param {number} datasetId identifier for the dataset that this privacy object applies to - */ -const createInitialComplianceInfo = datasetId => { - const identifier = typeof datasetId === 'string' ? { datasetUrn: decodeUrn(datasetId) } : { datasetId }; - - return { - ...identifier, - complianceType: '', - compliancePurgeNote: '', - complianceEntities: [], - datasetClassification: {} - }; -}; /** * @@ -43,6 +23,7 @@ const policyShape = { /** * Checks that a policy is valid + * TODO: Extract to TypeScript * @param candidatePolicy * @return {boolean} */ @@ -111,95 +92,4 @@ const isPolicyExpectedShape = (candidatePolicy = {}) => { return false; }; -/** - * Checks if the compliance suggestion has a date that is equal or exceeds the policy mod time by at least the - * ms time in lastSeenSuggestionInterval - * @param {number} [policyModificationTime = 0] timestamp for the policy modification date - * @param {number} suggestionModificationTime timestamp for the suggestion modification date - * @return {boolean} - */ -const isRecentSuggestion = (policyModificationTime = 0, suggestionModificationTime) => - !!suggestionModificationTime && suggestionModificationTime - policyModificationTime >= lastSeenSuggestionInterval; - -/** - * Checks if a compliance policy changeSet field requires user attention: if a suggestion - * is available but the user has not indicated intent or a policy for the field does not currently exist remotely - * and the related field changeSet has not been modified on the client and isn't readonly - * @param {boolean} isDirty flag indicating the field changeSet has been modified on the client - * @param {object|void} suggestion the field suggestion properties - * @param {boolean} privacyPolicyExists flag indicating that the field has a current policy upstream - * @param {string} suggestionAuthority possibly empty string indicating the user intent for the suggestion - * @return {boolean} - */ -const fieldChangeSetRequiresReview = ({ - isDirty, - suggestion, - privacyPolicyExists, - suggestionAuthority, - readonly -} = {}) => { - if (suggestion) { - return !suggestionAuthority && !readonly; - } - - // If either the privacy policy exists, or user has made changes, and field is not readonly then no review is required - return !(privacyPolicyExists || isDirty) && !readonly; -}; - -/** - * Gets the fields requiring review - * @type {(array: Array) => Array} - */ -const getFieldsRequiringReview = arrayMap(fieldChangeSetRequiresReview); - -/** - * Merges the column fields with the suggestion for the field if available - * @param {object} mappedColumnFields a map of column fields to compliance entity properties - * @param {object} fieldSuggestionMap a map of field suggestion properties keyed by field name - * @return {Array} mapped column field augmented with suggestion if available - */ -const mergeMappedColumnFieldsWithSuggestions = (mappedColumnFields = {}, fieldSuggestionMap = {}) => - Object.keys(mappedColumnFields).map(fieldName => { - const { - identifierField, - dataType, - identifierType, - logicalType, - securityClassification, - policyModificationTime, - privacyPolicyExists, - isDirty, - nonOwner, - readonly - } = mappedColumnFields[fieldName]; - const suggestion = fieldSuggestionMap[identifierField]; - - const field = { - identifierField, - dataType, - identifierType, - logicalType, - privacyPolicyExists, - isDirty, - nonOwner, - securityClassification, - readonly - }; - - // If a suggestion exists for this field add the suggestion attribute to the field properties / changeSet - // Check if suggestion isRecent before augmenting, otherwise, suggestion will not be considered on changeSet - if (suggestion && isRecentSuggestion(policyModificationTime, suggestion.suggestionsModificationTime)) { - return { ...field, suggestion }; - } - - return field; - }); - -export { - createInitialComplianceInfo, - isPolicyExpectedShape, - fieldChangeSetRequiresReview, - mergeMappedColumnFieldsWithSuggestions, - isRecentSuggestion, - getFieldsRequiringReview -}; +export { isPolicyExpectedShape };