diff --git a/wherehows-web/app/components/dataset-compliance-row.js b/wherehows-web/app/components/dataset-compliance-row.js index 60a31591c9..fd216ef86f 100644 --- a/wherehows-web/app/components/dataset-compliance-row.js +++ b/wherehows-web/app/components/dataset-compliance-row.js @@ -1,7 +1,6 @@ import Ember from 'ember'; import DatasetTableRow from 'wherehows-web/components/dataset-table-row'; import { - fieldIdentifierTypeValues, fieldIdentifierTypeIds, defaultFieldDataTypeClassification, isMixedId, @@ -12,6 +11,11 @@ import { SuggestionIntent } from 'wherehows-web/constants'; import { fieldChangeSetRequiresReview } from 'wherehows-web/utils/datasets/compliance-policy'; +import { compact } from 'wherehows-web/utils/array'; +import { + highConfidenceSuggestions, + accumulateFieldSuggestions +} from 'wherehows-web/utils/datasets/compliance-suggestions'; const { computed, get, getProperties } = Ember; @@ -21,24 +25,7 @@ const { computed, get, getProperties } = Ember; * @param {Array} predictions * @returns Array */ -const getFieldSuggestions = predictions => - predictions.filter(prediction => prediction).reduce((suggested, { value, confidence = 0 }) => { - if (value) { - if (fieldIdentifierTypeValues.includes(value)) { - suggested = { ...suggested, identifierType: value }; - } else { - suggested = { ...suggested, logicalType: value }; - } - - return { - ...suggested, - // value is Percent. identifierType value should be the last element in the list - confidence: (confidence * 100).toFixed(2) - }; - } - - return suggested; - }, {}); +const getFieldSuggestions = predictions => accumulateFieldSuggestions(highConfidenceSuggestions(compact(predictions))); export default DatasetTableRow.extend({ /** diff --git a/wherehows-web/app/constants/metadata-acquisition.ts b/wherehows-web/app/constants/metadata-acquisition.ts index 23e30454e4..1df994359f 100644 --- a/wherehows-web/app/constants/metadata-acquisition.ts +++ b/wherehows-web/app/constants/metadata-acquisition.ts @@ -44,6 +44,12 @@ interface IFieldIdTypes { [prop: string]: IFieldIdProps; } +/** + * Percentage value for a compliance policy suggestion with a low confidence score + * @type {number} + */ +const lowQualitySuggestionConfidenceThreshold = 0.5; + /** * A list of id logical types * @type {Array.} @@ -344,5 +350,6 @@ export { logicalTypesForIds, logicalTypesForGeneric, getDefaultLogicalType, - SuggestionIntent + SuggestionIntent, + lowQualitySuggestionConfidenceThreshold }; diff --git a/wherehows-web/app/typings/api/datasets/compliance.d.ts b/wherehows-web/app/typings/api/datasets/compliance.d.ts index a5f3fba313..c2569149f4 100644 --- a/wherehows-web/app/typings/api/datasets/compliance.d.ts +++ b/wherehows-web/app/typings/api/datasets/compliance.d.ts @@ -10,6 +10,19 @@ interface IPrediction { value: string; } +/** + * Describes the interface for a field suggestion + * values for the keys are extracted from the JSON response + * from the compliance/suggestion endpoint and do not necessarily match up + * with the key [`identifierTypePrediction` | `logicalTypePrediction`] predicted values + * @link extractTypesSuggestion gives further detail + */ +interface IFieldSuggestion { + identifierType?: string; + logicalType?: string; + confidence: number; +} + /** * Describes shape of a compliance auto suggestion */ diff --git a/wherehows-web/app/utils/array.ts b/wherehows-web/app/utils/array.ts index 7ccf49f3a2..a4194617da 100644 --- a/wherehows-web/app/utils/array.ts +++ b/wherehows-web/app/utils/array.ts @@ -14,6 +14,17 @@ const arrayMap = (mappingFunction: (param: T) => U): ((array: Array) => const arrayFilter = (filtrationFunction: (param: T) => boolean): ((array: Array) => Array) => (array = []) => array.filter(filtrationFunction); +/** + * Composable reducer abstraction, curries a reducing iteratee and returns a reducing function that takes a list + * @param {(acc: U) => U} iteratee + * @param {U} init the initial value in the reduction sequence + * @return {(arr: Array) => U} + */ +const arrayReduce = ( + iteratee: (accumulator: U, element: T, index: number, collection: Array) => U, + init: U +): ((arr: Array) => U) => (array = []) => array.reduce(iteratee, init); + /** * Duplicate check using every to short-circuit iteration * @param {Array} [list = []] list to check for dupes @@ -21,4 +32,11 @@ const arrayFilter = (filtrationFunction: (param: T) => boolean): ((array: Arr */ const isListUnique = (list: Array = []): boolean => new Set(list).size === list.length; -export { arrayMap, arrayFilter, isListUnique }; +/** + * Extracts all non falsey values from a list. + * @param {Array} list the list of items to compact + * @return {Array} + */ +const compact = (list: Array = []): Array => list.filter(item => item); + +export { arrayMap, arrayFilter, arrayReduce, isListUnique, compact }; diff --git a/wherehows-web/app/utils/datasets/compliance-suggestions.ts b/wherehows-web/app/utils/datasets/compliance-suggestions.ts new file mode 100644 index 0000000000..490945fede --- /dev/null +++ b/wherehows-web/app/utils/datasets/compliance-suggestions.ts @@ -0,0 +1,49 @@ +import { IFieldSuggestion, IPrediction } from 'wherehows-web/typings/api/datasets/compliance'; +import { arrayFilter, arrayReduce } from 'wherehows-web/utils/array'; +import { fieldIdentifierTypeValues, lowQualitySuggestionConfidenceThreshold } from 'wherehows-web/constants'; + +/** + * Takes a list of suggestions with confidence values, and if the confidence is greater than + * a low confidence threshold + * @param {number} confidence + * @return {boolean} + */ +const isHighConfidenceSuggestion = ({ confidence = 0 }: IPrediction): boolean => + confidence > lowQualitySuggestionConfidenceThreshold; + +/** + * Filters out a list of IPrediction 's that have a confidence level higher than the low confidence threshold + * @type {(array: Array) => Array} + */ +const highConfidenceSuggestions = arrayFilter(isHighConfidenceSuggestion); + +/** + * Extracts the type (identifierType&|logicalType) suggestion and confidence value from a predicted or suggested object. + * A determination is made based of the type of the value string, rather than the keys of the wrapping object: + * `identifierTypePrediction` or `logicalTypePrediction`. + * This is important to pay attention to when modifying the implementation + * @param {IFieldSuggestion} suggestion the extracted suggestion + * @param {string} value the value in the api provided suggestion + * @param {number} confidence how confidence the system is in the suggested value + * @return {IFieldSuggestion} + */ +const extractTypesSuggestion = ( + suggestion: IFieldSuggestion, + { value, confidence = 0 }: IPrediction +): IFieldSuggestion => { + if (value) { + if (fieldIdentifierTypeValues.includes(value)) { + suggestion = { ...suggestion, identifierType: value }; + } else { + suggestion = { ...suggestion, logicalType: value }; + } + + // identifierType value should be the last element in the list + return { ...suggestion, confidence: +(confidence * 100).toFixed(2) }; + } + return suggestion; +}; + +const accumulateFieldSuggestions = arrayReduce(extractTypesSuggestion, { confidence: 0 }); + +export { highConfidenceSuggestions, accumulateFieldSuggestions }; diff --git a/wherehows-web/tests/unit/utils/array-test.js b/wherehows-web/tests/unit/utils/array-test.js index 24ca8182d1..56c628d8b9 100644 --- a/wherehows-web/tests/unit/utils/array-test.js +++ b/wherehows-web/tests/unit/utils/array-test.js @@ -1,5 +1,5 @@ import { module, test } from 'qunit'; -import { arrayMap, arrayFilter, isListUnique } from 'wherehows-web/utils/array'; +import { arrayMap, arrayFilter, arrayReduce, isListUnique } from 'wherehows-web/utils/array'; import { xRandomNumbers, numToString, isAString } from 'wherehows-web/tests/helpers/arrays/functions'; module('Unit | Utility | array'); @@ -45,3 +45,17 @@ test('isListUnique correctly tests uniqueness of a list', function(assert) { assert.notOk(isListUnique(listWithDuplicateNumbers), `${listWithDuplicateNumbers} has duplicates`); assert.ok(isListUnique(listWithoutDuplicateNumbers), `${listWithoutDuplicateNumbers} has no duplicates`); }); + +test('arrayReduce is a function', function(assert) { + assert.ok(typeof arrayReduce === 'function', 'module exports an array reducer function'); +}); + +test('arrayReduce should work as a reduction iteratee', function(assert) { + const array = [{ a: 1 }, { b: 2 }, { c: 3 }], + expected = { a: 1, b: 2, c: 3 }; + const reducer = arrayReduce(function(acc, el) { + return { ...acc, ...el }; + }, {}); + + assert.deepEqual(reducer(array), expected); +});