diff --git a/wherehows-web/app/components/dataset-compliance.ts b/wherehows-web/app/components/dataset-compliance.ts index f2530e5b46..3a71b99189 100644 --- a/wherehows-web/app/components/dataset-compliance.ts +++ b/wherehows-web/app/components/dataset-compliance.ts @@ -35,7 +35,6 @@ import { singleTagsInChangeSet, tagsForIdentifierField } from 'wherehows-web/constants'; -import { isPolicyExpectedShape } from 'wherehows-web/utils/datasets/compliance-policy'; import { getTagsSuggestions } from 'wherehows-web/utils/datasets/compliance-suggestions'; import { arrayMap, compact, isListUnique, iterateArrayAsync } from 'wherehows-web/utils/array'; import noop from 'wherehows-web/utils/noop'; @@ -73,8 +72,6 @@ const { complianceFieldNotUnique, missingTypes, helpText, - successUploading, - invalidPolicyData, missingPurgePolicy, missingDatasetSecurityClassification } = compliancePolicyStrings; @@ -127,6 +124,9 @@ export default class DatasetCompliance extends Component { schemaFieldNamesMappedToDataTypes: Array>; onReset: () => Promise; onSave: () => Promise; + + onComplianceUpload: (jsonString: string) => void; + notifyOnChangeSetSuggestions: (hasSuggestions: boolean) => void; notifyOnChangeSetRequiresReview: (hasChangeSetDrift: boolean) => void; @@ -815,9 +815,9 @@ export default class DatasetCompliance extends Component { * checked in the function. If criteria is not met, an the returned promise is settled * in a rejected state, otherwise fulfilled * @method - * @return {Promise | void} + * @return {Promise} */ - validateFields(this: DatasetCompliance): Promise | void { + async validateFields(this: DatasetCompliance): Promise { const { notify } = get(this, 'notifications'); const { complianceEntities = [] } = get(this, 'complianceInfo') || {}; const idTypeComplianceEntities = complianceEntities.filter(isTagIdType(get(this, 'complianceDataTypes'))); @@ -1161,47 +1161,6 @@ export default class DatasetCompliance extends Component { set(tag, 'suggestionAuthority', intent); }, - /** - * Receives the json representation for compliance and applies each key to the policy - * @param {string} textString string representation for the JSON file - */ - onComplianceJsonUpload(this: DatasetCompliance, textString: string): void { - const complianceInfo = get(this, 'complianceInfo'); - const { notify } = get(this, 'notifications'); - let policy; - - if (!complianceInfo) { - notify(NotificationEvent.error, { - content: 'Could not find compliance current compliance policy for this dataset' - }); - - return; - } - - try { - policy = JSON.parse(textString); - } catch (e) { - notify(NotificationEvent.error, { - content: invalidPolicyData - }); - } - - if (isPolicyExpectedShape(policy)) { - setProperties(complianceInfo, { - complianceEntities: policy.complianceEntities, - datasetClassification: policy.datasetClassification - }); - - notify(NotificationEvent.info, { - content: successUploading - }); - } - - notify(NotificationEvent.error, { - content: invalidPolicyData - }); - }, - /** * Handles the compliance policy download action */ @@ -1237,6 +1196,14 @@ export default class DatasetCompliance extends Component { anchor.click(); }, + /** + * Receives the json representation for compliance and applies each key to the policy + * @param {string} jsonString string representation for the JSON file + */ + onComplianceJsonUpload(this: DatasetCompliance, jsonString: string): void { + get(this, 'onComplianceUpload')(jsonString); + }, + /** * Updates the source object representing the current datasetClassification map * @param {keyof typeof DatasetClassifiers} classifier the property on the datasetClassification to update diff --git a/wherehows-web/app/components/datasets/containers/dataset-compliance.ts b/wherehows-web/app/components/datasets/containers/dataset-compliance.ts index cd0c4f4703..13e2251e63 100644 --- a/wherehows-web/app/components/datasets/containers/dataset-compliance.ts +++ b/wherehows-web/app/components/datasets/containers/dataset-compliance.ts @@ -1,5 +1,5 @@ import Component from '@ember/component'; -import { get, set, setProperties } from '@ember/object'; +import { get, set, setProperties, getProperties } from '@ember/object'; import ComputedProperty from '@ember/object/computed'; import { inject } from '@ember/service'; import { task } from 'ember-concurrency'; @@ -28,6 +28,10 @@ import { SuggestionIntent } from 'wherehows-web/constants'; import { iterateArrayAsync } from 'wherehows-web/utils/array'; +import validateMetadataObject, { + datasetComplianceMetadataTaxonomy +} from 'wherehows-web/utils/datasets/compliance/metadata-schema'; +import { notificationDialogActionFactory } from 'wherehows-web/utils/notifications/notifications'; /** * Type alias for the response when container data items are batched @@ -52,7 +56,7 @@ type BatchContainerDataResult = Pick< | 'schemaless' >; -const { successUpdating, failedUpdating } = compliancePolicyStrings; +const { successUpdating, failedUpdating, successUploading, invalidPolicyData } = compliancePolicyStrings; export default class DatasetComplianceContainer extends Component { /** @@ -336,4 +340,43 @@ export default class DatasetComplianceContainer extends Component { onSuggestionsComplianceFeedback(uid: string | null = null, feedback: SuggestionIntent) { saveDatasetComplianceSuggestionFeedbackByUrn(get(this, 'urn'), uid, feedback); } + + /** + * Reapplies the uploaded compliance policy to the container property + * @param {string} jsonString string representation for the JSON file + * @memberof DatasetComplianceContainer + */ + @action + onComplianceUpload(this: DatasetComplianceContainer, jsonString: string): void { + const { + complianceInfo, + notifications: { notify } + } = getProperties(this, ['complianceInfo', 'notifications']); + + if (complianceInfo) { + try { + const policy = JSON.parse(jsonString); + + if (validateMetadataObject(policy, datasetComplianceMetadataTaxonomy)) { + const { complianceEntities, datasetClassification } = policy; + const resolvedComplianceInfo = { ...complianceInfo, complianceEntities, datasetClassification }; + const { dialogActions } = notificationDialogActionFactory(); + + set(this, 'complianceInfo', resolvedComplianceInfo); + + notify(NotificationEvent.confirm, { + header: 'Successfully applied uploaded metadata', + content: successUploading, + dialogActions, + dismissButtonText: false, + confirmButtonText: 'Dismiss' + }); + } + } catch (e) { + notify(NotificationEvent.error, { + content: invalidPolicyData + }); + } + } + } } diff --git a/wherehows-web/app/constants/dataset-compliance.ts b/wherehows-web/app/constants/dataset-compliance.ts index 669bc85712..cdd93d7397 100644 --- a/wherehows-web/app/constants/dataset-compliance.ts +++ b/wherehows-web/app/constants/dataset-compliance.ts @@ -35,7 +35,7 @@ const compliancePolicyStrings = { missingTypes: 'Looks like you may have forgotten to specify a `Field Format` for all ID fields?', successUpdating: 'Changes have been successfully saved!', failedUpdating: 'An error occurred while saving.', - successUploading: 'Metadata successfully updated! Please "Save" when ready.', + successUploading: 'Metadata successfully updated! Please confirm and "Save" when ready.', invalidPolicyData: 'Received policy in an unexpected format! Please check the provided attributes and try again.', helpText: { classification: @@ -107,7 +107,7 @@ const getComplianceSteps = (hasSchema: boolean = true): { [x: number]: { name: s * @param {IComplianceEntity} { readonly } * @returns {boolean} */ -const isEditableComplianceEntity = ({ readonly }: IComplianceEntity): boolean => readonly !== true; +const isEditableComplianceEntity = ({ readonly }: IComplianceEntity): boolean => readonly !== true; // do not simplify, readonly may be undefined /** * Filters out from a list of compliance entities, entities that are editable @@ -299,26 +299,26 @@ const idTypeFieldsHaveLogicalType = arrayEvery(idTypeFieldHasLogicalType); /** * Describes the function interface for tagsForIdentifierField - * @interface TagsForIdentifierFieldFn + * @interface ITagsForIdentifierFieldFn */ -interface TagsForIdentifierFieldFn { +interface ITagsForIdentifierFieldFn { (identifierField: string): (tags: Array) => Array; } /** * Gets the tags for a specific identifier field * @param {string} identifierField - * @return {TagsForIdentifierFieldFn} + * @return {ITagsForIdentifierFieldFn} */ -const tagsForIdentifierField: TagsForIdentifierFieldFn = (identifierField: string) => +const tagsForIdentifierField: ITagsForIdentifierFieldFn = (identifierField: string) => arrayFilter(isSchemaFieldTag(identifierField)); /** * Lists tags that occur for only one identifier type in the list of tags * @param {Array} tags the full list of tags to iterate through - * @param {TagsForIdentifierFieldFn} tagsForIdentifierFieldFn + * @param {ITagsForIdentifierFieldFn} tagsForIdentifierFieldFn * @return {(singleTags: Array, { identifierField }: IComplianceChangeSet) => (any)[] | Array} */ -const singleTagsIn = (tags: Array, tagsForIdentifierFieldFn: TagsForIdentifierFieldFn) => ( +const singleTagsIn = (tags: Array, tagsForIdentifierFieldFn: ITagsForIdentifierFieldFn) => ( singleTags: Array, { identifierField }: IComplianceChangeSet ): Array => { @@ -329,12 +329,12 @@ const singleTagsIn = (tags: Array, tagsForIdentifierFieldF /** * Lists the tags in a list of tags that occur for only one identifier type * @param {Array} tags - * @param {TagsForIdentifierFieldFn} tagsForIdentifierFieldFn + * @param {ITagsForIdentifierFieldFn} tagsForIdentifierFieldFn * @return {Array} */ const singleTagsInChangeSet = ( tags: Array, - tagsForIdentifierFieldFn: TagsForIdentifierFieldFn + tagsForIdentifierFieldFn: ITagsForIdentifierFieldFn ): Array => arrayReduce(singleTagsIn(tags, tagsForIdentifierFieldFn), [])(tags); /** @@ -415,21 +415,18 @@ const foldComplianceChangeSets = async ( /** * Builds a default shape for securitySpecification & privacyCompliancePolicy with default / unset values * for non null properties as per Avro schema - * @param {string} datasetId identifier for the dataset that this privacy object applies to + * @param {string} datasetUrn identifier for the dataset that this privacy object applies to + * @return {IComplianceInfo} */ -const createInitialComplianceInfo = (datasetId: string): IComplianceInfo => { - const identifier = typeof datasetId === 'string' ? { datasetUrn: decodeUrn(datasetId) } : { datasetId }; - - return { - ...identifier, - datasetId: null, - confidentiality: null, - complianceType: '', - compliancePurgeNote: '', - complianceEntities: [], - datasetClassification: null - }; -}; +const initialComplianceObjectFactory = (datasetUrn: string): IComplianceInfo => ({ + datasetUrn: decodeUrn(datasetUrn), + datasetId: null, + confidentiality: null, + complianceType: '', + compliancePurgeNote: '', + complianceEntities: [], + datasetClassification: null +}); /** * Maps the fields found in the column property on the schema api to the values returned in the current privacy policy @@ -581,7 +578,7 @@ export { tagsHaveNoneType, fieldTagsRequiringReview, tagsHaveNoneAndNotNoneType, - createInitialComplianceInfo, + initialComplianceObjectFactory, getIdTypeDataTypes, fieldTagsHaveIdentifierType, idTypeFieldHasLogicalType, diff --git a/wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs b/wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs index 66a8187ffe..853d0f342a 100644 --- a/wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs +++ b/wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs @@ -33,6 +33,7 @@ notifyOnChangeSetRequiresReview=(action "onCompliancePolicyChangeSetDrift") onSave=(action "savePrivacyCompliancePolicy") onReset=(action "resetPrivacyCompliancePolicy") + onComplianceUpload=(action "onComplianceUpload") }} {{/if}} diff --git a/wherehows-web/app/typings/generic.d.ts b/wherehows-web/app/typings/generic.d.ts index 7f6dc30a81..9ea889548c 100644 --- a/wherehows-web/app/typings/generic.d.ts +++ b/wherehows-web/app/typings/generic.d.ts @@ -10,4 +10,12 @@ type StringUnionKeyToValue = { [K in U]: K }; */ type StringEnumKeyToEnumValue = { [K in T]: V }; -export { StringUnionKeyToValue, StringEnumKeyToEnumValue }; +/** + * Describes the index signature for a generic object + * @interface IObject + */ +interface IObject { + [K: string]: T; +} + +export { StringUnionKeyToValue, StringEnumKeyToEnumValue, IObject }; diff --git a/wherehows-web/app/utils/api/datasets/compliance.ts b/wherehows-web/app/utils/api/datasets/compliance.ts index 698834495a..e7f4ea5c9c 100644 --- a/wherehows-web/app/utils/api/datasets/compliance.ts +++ b/wherehows-web/app/utils/api/datasets/compliance.ts @@ -1,4 +1,4 @@ -import { createInitialComplianceInfo } from 'wherehows-web/constants'; +import { initialComplianceObjectFactory } from 'wherehows-web/constants'; import { SuggestionIntent } from 'wherehows-web/constants/datasets/compliance'; import { notFoundApiError } from 'wherehows-web/utils/api'; import { datasetUrlById, datasetUrlByUrn } from 'wherehows-web/utils/api/datasets/shared'; @@ -62,7 +62,7 @@ export interface IReadComplianceResult { * @return {Promise} */ const readDatasetComplianceByUrn = async (urn: string): Promise => { - let complianceInfo: IComplianceGetResponse['complianceInfo'] = createInitialComplianceInfo(urn); + let complianceInfo: IComplianceGetResponse['complianceInfo'] = initialComplianceObjectFactory(urn); let isNewComplianceInfo = false; try { @@ -71,7 +71,7 @@ const readDatasetComplianceByUrn = async (urn: string): Promise; + readonly '@type': string; + + readonly [K: string]: IMetadataTaxonomy | any; +} + +/** + * Defines the shape of the dataset compliance metadata json object using the IMetadataTaxonomy interface + * @type {({'@type': string; '@props': string[]} | {'@type': string; '@props': string[]; securityClassification: {'@type': string; '@props': any[]}})[]} + */ +const datasetComplianceMetadataTaxonomy: Array = [ + { + '@type': 'datasetClassification:object', + '@props': Object.keys(DatasetClassifiers).map(key => `${key}:boolean`) + }, + { + '@type': 'complianceEntities:array', + '@props': [ + 'identifierField:string', + 'identifierType:string|null', + 'securityClassification:string|null', + 'logicalType:string|null', + 'nonOwner:boolean|null', + 'valuePattern:string|null' + ], + securityClassification: { + '@type': 'securityClassification:string', + '@props': Object.values(Classification) + } + } +]; + +/** + * Checks that a value type matches an expected pattern string + * @param {*} value the value to check + * @param {string} expectedTypePattern the pattern string to match against + * @returns {boolean} + */ +const valueEquiv = (value: any, expectedTypePattern: string): boolean => expectedTypePattern.includes(typeOf(value)); + +/** + * Extracts the type key and the pattern string from the string mapping into a tuple pair + * @param {string} objectKeyTypePattern string value consisting of a pair of key/property name and allowed types separated by a colon ":" + * @returns {[string, string]} + */ +const typePatternMap = (objectKeyTypePattern: string): [string, string] => + <[string, string]>objectKeyTypePattern.split(':'); + +/** + * Returns a iteratee bound to an object that checks that a key matches the expected value in the typeMap + * @param {IObject} object the object with keys to check + * @return {(typeMap: string) => boolean} + */ +const keyValueHasMatch = (object: IObject) => (typeMap: string) => { + const [key, typeString] = typePatternMap(typeMap); + return valueEquiv(object[key], typeString); +}; + +/** + * Checks each key on an object matches the expected types in the typeMap + * @param {IObject} object the object with keys to check + * @param {Array} typeMaps the colon delimited type string + * @returns {boolean} + */ +const keysEquiv = (object: IObject, typeMaps: Array): boolean => + arrayEvery(keyValueHasMatch(object))(typeMaps); + +/** + * Checks that a compliance metadata object has a schema that matches the taxonomy / schema provided + * @param {IObject} object an instance of a compliance metadata object + * @param {Array} taxonomy schema shape to check against + * @return {boolean} + */ +const validateMetadataObject = (object: IObject, taxonomy: Array): boolean => { + const rootTypeMaps = taxonomy.map(category => category['@type']); + let isValid = keysEquiv(object, rootTypeMaps); + + if (isValid) { + const downlevelAccumulator = (validity: boolean, typeMap: string): boolean => { + const [key, pattern]: [string, string] = typePatternMap(typeMap); + + if (pattern.includes('object')) { + validity = keysEquiv(object[key], taxonomy.findBy('@type', typeMap)!['@props']); + } + + if (pattern.includes('array') && Array.isArray(object[key])) { + validity = arrayReduce( + (validity: boolean, value: IObject) => + validity && keysEquiv(value, taxonomy.findBy('@type', typeMap)!['@props']), + validity + )(object[key]); + } + + return validity; + }; + + return arrayReduce(downlevelAccumulator, isValid)(rootTypeMaps); + } + + return isValid; +}; + +export default validateMetadataObject; + +export { datasetComplianceMetadataTaxonomy }; diff --git a/wherehows-web/tests/integration/components/datasets/containers/dataset-compliance-test.js b/wherehows-web/tests/integration/components/datasets/containers/dataset-compliance-test.js index c8095776b1..93e69a7d6b 100644 --- a/wherehows-web/tests/integration/components/datasets/containers/dataset-compliance-test.js +++ b/wherehows-web/tests/integration/components/datasets/containers/dataset-compliance-test.js @@ -2,7 +2,7 @@ import { moduleForComponent, test } from 'ember-qunit'; import hbs from 'htmlbars-inline-precompile'; import { waitUntil, find } from 'ember-native-dom-helpers'; import { urn } from 'wherehows-web/mirage/fixtures/urn'; -import { createInitialComplianceInfo } from 'wherehows-web/constants'; +import { initialComplianceObjectFactory } from 'wherehows-web/constants'; import sinon from 'sinon'; moduleForComponent( @@ -27,7 +27,7 @@ test('it renders', async function(assert) { 200, { 'Content-Type': 'application/json' }, JSON.stringify({ - complianceInfo: createInitialComplianceInfo(urn) + complianceInfo: initialComplianceObjectFactory(urn) }) ]); this.server.respondWith(/.*\/compliance-data-types/, [ diff --git a/wherehows-web/tests/unit/constants/dataset-compliance-test.js b/wherehows-web/tests/unit/constants/dataset-compliance-test.js index 51f1b7a06b..c2d1fe0a19 100644 --- a/wherehows-web/tests/unit/constants/dataset-compliance-test.js +++ b/wherehows-web/tests/unit/constants/dataset-compliance-test.js @@ -6,20 +6,22 @@ import { getFieldIdentifierOptions, isAutoGeneratedPolicy, PurgePolicy, - createInitialComplianceInfo, + initialComplianceObjectFactory, isRecentSuggestion, tagNeedsReview } from 'wherehows-web/constants'; import complianceDataTypes from 'wherehows-web/mirage/fixtures/compliance-data-types'; import { mockTimeStamps } from 'wherehows-web/tests/helpers/datasets/compliance-policy/recent-suggestions-constants'; import { mockFieldChangeSets } from 'wherehows-web/tests/helpers/datasets/compliance-policy/field-changeset-constants'; +import { hdfsUrn } from 'wherehows-web/mirage/fixtures/urn'; module('Unit | Constants | dataset compliance'); -test('createInitialComplianceInfo', function(assert) { +test('initialComplianceObjectFactory', function(assert) { assert.expect(2); - const mockId = 1337; + const mockUrn = hdfsUrn; const initialComplianceInfo = { + datasetUrn: mockUrn, datasetId: null, complianceType: '', compliancePurgeNote: '', @@ -28,8 +30,12 @@ test('createInitialComplianceInfo', function(assert) { confidentiality: null }; - assert.ok(typeof createInitialComplianceInfo === 'function', 'createInitialComplianceInfo is a function'); - assert.deepEqual(createInitialComplianceInfo(mockId), initialComplianceInfo, 'generates policy in expected shape'); + assert.ok(typeof initialComplianceObjectFactory === 'function', 'initialComplianceObjectFactory is a function'); + assert.deepEqual( + initialComplianceObjectFactory(mockUrn), + initialComplianceInfo, + 'generates policy in expected shape' + ); }); test('isRecentSuggestion exists', function(assert) {