From 7a74e5b86c14b5d0f586b781975e1168fc50f63f Mon Sep 17 00:00:00 2001 From: Seyi Adebajo Date: Tue, 24 Jul 2018 13:33:27 -0700 Subject: [PATCH 1/9] adds typesafe iteratee-first-data-last util partition --- wherehows-web/app/utils/array.ts | 45 ++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/wherehows-web/app/utils/array.ts b/wherehows-web/app/utils/array.ts index abe87760c0..8edbb89596 100644 --- a/wherehows-web/app/utils/array.ts +++ b/wherehows-web/app/utils/array.ts @@ -1,4 +1,4 @@ -import { identity } from 'wherehows-web/utils/helpers/functions'; +import { identity, not } from 'wherehows-web/utils/helpers/functions'; /** * Composable function that will in turn consume an item from a list an emit a result of equal or same type @@ -22,37 +22,53 @@ const take = (n: number = 0) => (list: Array): Array => Array.prototype /** * Convenience utility takes a type-safe mapping function, and returns a list mapping function - * @param {(param: T) => U} mappingFunction maps a single type T to type U + * @param {(param: T) => U} predicate maps a single type T to type U * @return {(array: Array) => Array} */ -const arrayMap = (mappingFunction: (param: T) => U): ((array: Array) => Array) => (array = []) => - array.map(mappingFunction); +const arrayMap = (predicate: (param: T) => U): ((array: Array) => Array) => (array = []) => + array.map(predicate); + +/** + * Partitions an array into a tuple containing elements that meet the predicate in the zeroth index, + * and excluded elements in the next + * `iterate-first data-last` function + * @template T type of source element list + * @template U subtype of T in first partition + * @param {(param: T) => param is U} predicate is a type guard function + * @returns {((array: Array) => [Array, Array>])} + */ +const arrayPartition = ( + predicate: (param: T) => param is U +): ((array: Array) => [Array, Array>]) => (array = []) => [ + array.filter(predicate), + array.filter>((v: T): v is Exclude => not(predicate)(v)) +]; /** * Convenience utility takes a type-safe filter function, and returns a list filtering function - * @param {(param: T) => boolean} filtrationFunction + * @param {(param: T) => boolean} predicate * @return {(array: Array) => Array} */ -const arrayFilter = (filtrationFunction: (param: T) => boolean): ((array: Array) => Array) => (array = []) => - array.filter(filtrationFunction); +const arrayFilter = (predicate: (param: T) => boolean): ((array: Array) => Array) => (array = []) => + array.filter(predicate); /** * Type safe utility `iterate-first data-last` function for array every * @template T - * @param {(param: T) => boolean} filter + * @param {(param: T) => boolean} predicate * @returns {((array: Array) => boolean)} */ -const arrayEvery = (filter: (param: T) => boolean): ((array: Array) => boolean) => (array = []) => - array.every(filter); +const arrayEvery = (predicate: (param: T) => boolean): ((array: Array) => boolean) => (array = []) => + array.every(predicate); /** * Type safe utility `iterate-first data-last` function for array some * @template T - * @param {(param: T) => boolean} filter + * @param {(param: T) => boolean} predicate * @return {(array: Array) => boolean} */ -const arraySome = (filter: (param: T) => boolean): ((array: Array) => boolean) => (array = []) => - array.some(filter); +const arraySome = (predicate: (param: T) => boolean): ((array: Array) => boolean) => (array = []) => + array.some(predicate); /** * Composable reducer abstraction, curries a reducing iteratee and returns a reducing function that takes a list @@ -228,9 +244,10 @@ export { Many, Iteratee }; export { take, arrayMap, + arrayPipe, arrayFilter, arrayReduce, - arrayPipe, + arrayPartition, isListUnique, compact, arrayEvery, From 882bcbdcfb20e64425a56050bfe1b1e41478f3bb Mon Sep 17 00:00:00 2001 From: cptran777 Date: Fri, 27 Jul 2018 11:34:50 -0700 Subject: [PATCH 2/9] Add and configure ember-highcharts --- wherehows-web/ember-cli-build.js | 9 +++++++++ wherehows-web/package.json | 2 ++ wherehows-web/yarn.lock | 20 +++++++++++++++++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/wherehows-web/ember-cli-build.js b/wherehows-web/ember-cli-build.js index 32defadcfe..d5dbe140bf 100644 --- a/wherehows-web/ember-cli-build.js +++ b/wherehows-web/ember-cli-build.js @@ -21,6 +21,15 @@ module.exports = function(defaults) { includePolyfill: true }, + emberHighCharts: { + includedHighCharts: true, + // Note: Since we only need highcharts, excluding the other available modules in the addon + includeHighStock: false, + includeHighMaps: false, + includeHighChartsMore: false, + includeHighCharts3D: false + }, + storeConfigInMeta: false, SRI: { diff --git a/wherehows-web/package.json b/wherehows-web/package.json index e2848fb8ba..ba46d9b5a5 100644 --- a/wherehows-web/package.json +++ b/wherehows-web/package.json @@ -66,6 +66,7 @@ "ember-export-application-global": "^2.0.0", "ember-fetch": "^3.4.4", "ember-font-awesome": "^4.0.0-rc.2", + "ember-highcharts": "^1.0.0", "ember-inflector": "^2.2.0", "ember-load-initializers": "^1.0.0", "ember-math-helpers": "^2.4.0", @@ -87,6 +88,7 @@ "eslint-plugin-prettier": "^2.5.0", "eyeglass": "^1.3.0", "eyeglass-restyle": "^1.1.0", + "highcharts": "^6.1.1", "husky": "^0.14.3", "ivy-tabs": "^3.1.0", "lint-staged": "^7.1.0", diff --git a/wherehows-web/yarn.lock b/wherehows-web/yarn.lock index 265174f083..232b872676 100644 --- a/wherehows-web/yarn.lock +++ b/wherehows-web/yarn.lock @@ -1382,6 +1382,10 @@ bootstrap-sass@^3.0.0: version "3.3.7" resolved "https://registry.yarnpkg.com/bootstrap-sass/-/bootstrap-sass-3.3.7.tgz#6596c7ab40f6637393323ab0bc80d064fc630498" +bootstrap@3.3.7: + version "3.3.7" + resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-3.3.7.tgz#5a389394549f23330875a3b150656574f8a9eb71" + bower-config@^1.3.0: version "1.4.1" resolved "https://registry.yarnpkg.com/bower-config/-/bower-config-1.4.1.tgz#85fd9df367c2b8dbbd0caa4c5f2bad40cd84c2cc" @@ -1742,7 +1746,7 @@ broccoli-lint-eslint@^4.2.1: lodash.defaultsdeep "^4.6.0" md5-hex "^2.0.0" -broccoli-merge-trees@^1.0.0, broccoli-merge-trees@^1.1.0, broccoli-merge-trees@^1.1.1, broccoli-merge-trees@^1.1.4: +broccoli-merge-trees@^1.0.0, broccoli-merge-trees@^1.1.0, broccoli-merge-trees@^1.1.1, broccoli-merge-trees@^1.1.4, broccoli-merge-trees@^1.2.0: version "1.2.4" resolved "https://registry.yarnpkg.com/broccoli-merge-trees/-/broccoli-merge-trees-1.2.4.tgz#a001519bb5067f06589d91afa2942445a2d0fdb5" dependencies: @@ -3516,6 +3520,16 @@ ember-hash-helper-polyfill@^0.1.1: ember-cli-babel "^5.1.7" ember-cli-version-checker "^1.2.0" +ember-highcharts@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/ember-highcharts/-/ember-highcharts-1.0.0.tgz#d412af4d1f2f55e1cae0174c353852fd98b5bad9" + dependencies: + bootstrap "3.3.7" + broccoli-funnel "^2.0.1" + broccoli-merge-trees "^1.2.0" + ember-cli-babel "^6.6.0" + ember-cli-htmlbars "^2.0.1" + ember-ignore-children-helper@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/ember-ignore-children-helper/-/ember-ignore-children-helper-1.0.1.tgz#f7c4aa17afb9c5685e1d4dcdb61c7b138ca7cdc3" @@ -5170,6 +5184,10 @@ heimdalljs@^0.2.0, heimdalljs@^0.2.1, heimdalljs@^0.2.3, heimdalljs@^0.2.5: dependencies: rsvp "~3.2.1" +highcharts@^6.1.1: + version "6.1.1" + resolved "https://registry.yarnpkg.com/highcharts/-/highcharts-6.1.1.tgz#49dc34f5e963744ecd7eb87603b6cdaf8304c13a" + hoek@2.x.x: version "2.16.3" resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed" From efe00a695a269e82eb066c8a60003ab750c85eb3 Mon Sep 17 00:00:00 2001 From: cptran777 Date: Fri, 27 Jul 2018 13:27:28 -0700 Subject: [PATCH 3/9] Create health tab and components and flags on whether or not to show the contents --- .../app/controllers/Application.java | 3 ++ .../datasets/containers/dataset-health.ts | 36 +++++++++++++++++++ .../app/controllers/datasets/dataset.ts | 8 +++++ wherehows-web/app/routes/datasets/dataset.ts | 3 +- .../datasets/containers/dataset-health.hbs | 1 + .../app/templates/datasets/dataset.hbs | 12 +++++++ .../api/configurator/configurator.d.ts | 1 + .../containers/dataset-health-test.ts | 26 ++++++++++++++ 8 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 wherehows-web/app/components/datasets/containers/dataset-health.ts create mode 100644 wherehows-web/app/templates/components/datasets/containers/dataset-health.hbs create mode 100644 wherehows-web/tests/integration/components/datasets/containers/dataset-health-test.ts diff --git a/wherehows-frontend/app/controllers/Application.java b/wherehows-frontend/app/controllers/Application.java index 0aac478cd2..1921a877ab 100644 --- a/wherehows-frontend/app/controllers/Application.java +++ b/wherehows-frontend/app/controllers/Application.java @@ -66,6 +66,8 @@ public class Application extends Controller { private static final Boolean HTTPS_REDIRECT = Play.application().configuration().getBoolean("https.redirect", false); private static final Boolean WHZ_SHOW_LINEAGE = Play.application().configuration().getBoolean("linkedin.show.dataset.lineage", false); + private static final Boolean WHZ_SHOW_DS_HEALTH = + Play.application().configuration().getBoolean("linkedin.show.dataset.health", false); private static final String WHZ_WIKI_LINKS__GDRP_PII = Play.application().configuration().getString("linkedin.links.wiki.gdprPii", ""); @@ -199,6 +201,7 @@ public class Application extends Controller { config.put("appVersion", APP_VERSION); config.put("isInternal", IS_INTERNAL); config.put("shouldShowDatasetLineage", WHZ_SHOW_LINEAGE); + config.put("shouldShowDatasetHealth", WHZ_SHOW_DS_HEALTH); config.set("wikiLinks", wikiLinks()); config.set("JitAclAccessWhitelist", Json.toJson(StringUtils.split(JIT_ACL_WHITELIST, ','))); config.set("tracking", trackingInfo()); diff --git a/wherehows-web/app/components/datasets/containers/dataset-health.ts b/wherehows-web/app/components/datasets/containers/dataset-health.ts new file mode 100644 index 0000000000..4aa9177df7 --- /dev/null +++ b/wherehows-web/app/components/datasets/containers/dataset-health.ts @@ -0,0 +1,36 @@ +import Component from '@ember/component'; +import { get } from '@ember/object'; +import { task, TaskInstance } from 'ember-concurrency'; + +/** + * This is the container component for the dataset health tab. It should contain the health bar graphs and a table + * depicting the detailed health scores. Aside from fetching the data, it also handles click interactions between + * the graphs and the table in terms of filtering and displaying of data + */ +export default class DatasetHealthContainer extends Component { + /** + * The urn identifier for the dataset + * @type {string} + */ + urn: string; + + didInsertElement() { + get(this, 'getContainerDataTask').perform(); + } + + didUpdateAttrs() { + get(this, 'getContainerDataTask').perform(); + } + + /** + * An async parent task to group all data tasks for this container component + * @type {Task>, (a?: any) => TaskInstance>>>} + */ + getContainerDataTask = task(function*(this: DatasetHealthContainer): IterableIterator>> { + // Forms the tasks that are needed to fetch the data... to be determined + // const tasks = Object.values( + // getProperties(this, ['getDatasetOwnersTask', 'getSuggestedOwnersTask', 'getDatasetOwnerTypesTask']) + // ); + // yield* tasks.map(task => task.perform()); + }); +} diff --git a/wherehows-web/app/controllers/datasets/dataset.ts b/wherehows-web/app/controllers/datasets/dataset.ts index 28240a49e5..c5473b29a2 100644 --- a/wherehows-web/app/controllers/datasets/dataset.ts +++ b/wherehows-web/app/controllers/datasets/dataset.ts @@ -87,6 +87,14 @@ export default class DatasetController extends Controller { */ shouldShowDatasetLineage: boolean; + /** + * Flags the health feature for datasets, which is currently in the development stage so we should not + * have it appear in production + * @type {boolean} + * @memberof DatasetController + */ + shouldShowDatasetHealth: boolean; + /** * Flag indicating if the dataset contains personally identifiable information * @type {boolean} diff --git a/wherehows-web/app/routes/datasets/dataset.ts b/wherehows-web/app/routes/datasets/dataset.ts index d124299290..9d08472054 100644 --- a/wherehows-web/app/routes/datasets/dataset.ts +++ b/wherehows-web/app/routes/datasets/dataset.ts @@ -95,7 +95,8 @@ export default class DatasetRoute extends Route { setProperties(controller, { isInternal: !!getConfig('isInternal'), jitAclAccessWhitelist: getConfig('JitAclAccessWhitelist') || [], - shouldShowDatasetLineage: getConfig('shouldShowDatasetLineage') + shouldShowDatasetLineage: getConfig('shouldShowDatasetLineage'), + shouldShowDatasetHealth: getConfig('shouldShowDatasetHealth') }); } diff --git a/wherehows-web/app/templates/components/datasets/containers/dataset-health.hbs b/wherehows-web/app/templates/components/datasets/containers/dataset-health.hbs new file mode 100644 index 0000000000..809b2abd25 --- /dev/null +++ b/wherehows-web/app/templates/components/datasets/containers/dataset-health.hbs @@ -0,0 +1 @@ +Coming Soon! \ No newline at end of file diff --git a/wherehows-web/app/templates/datasets/dataset.hbs b/wherehows-web/app/templates/datasets/dataset.hbs index 575799f476..7f3fe77d13 100644 --- a/wherehows-web/app/templates/datasets/dataset.hbs +++ b/wherehows-web/app/templates/datasets/dataset.hbs @@ -105,6 +105,12 @@ {{/tablist.tab}} {{/if}} + {{#if shouldShowDatasetHealth}} + {{#tablist.tab tabIds.Health on-select=(action "tabSelectionChanged")}} + Health + {{/tablist.tab}} + {{/if}} + {{/tabs.tablist}} @@ -152,5 +158,11 @@ {{datasets/dataset-relationships urn=encodedUrn}} {{/tabs.tabpanel}} {{/if}} + + {{#if shouldShowDatasetHealth}} + {{#tabs.tabpanel tabIds.Health}} + {{datasets/containers/dataset-health urn=encodedUrn}} + {{/tabs.tabpanel}} + {{/if}} {{/ivy-tabs}} diff --git a/wherehows-web/app/typings/api/configurator/configurator.d.ts b/wherehows-web/app/typings/api/configurator/configurator.d.ts index 308f6d0de6..03b00e8542 100644 --- a/wherehows-web/app/typings/api/configurator/configurator.d.ts +++ b/wherehows-web/app/typings/api/configurator/configurator.d.ts @@ -9,6 +9,7 @@ interface IAppConfig { isInternal: boolean | void; JitAclAccessWhitelist: Array | void; shouldShowDatasetLineage: boolean; + shouldShowDatasetHealth: boolean; tracking: { isEnabled: boolean; trackers: { diff --git a/wherehows-web/tests/integration/components/datasets/containers/dataset-health-test.ts b/wherehows-web/tests/integration/components/datasets/containers/dataset-health-test.ts new file mode 100644 index 0000000000..693120a466 --- /dev/null +++ b/wherehows-web/tests/integration/components/datasets/containers/dataset-health-test.ts @@ -0,0 +1,26 @@ +import { module, test } from 'qunit'; +import { setupRenderingTest } from 'ember-qunit'; +import { render } from '@ember/test-helpers'; +import hbs from 'htmlbars-inline-precompile'; + +module('Integration | Component | datasets/containers/dataset-health', function(hooks) { + setupRenderingTest(hooks); + + test('it renders', async function(assert) { + // Set any properties with this.set('myProperty', 'value'); + // Handle any actions with this.set('myAction', function(val) { ... }); + + await render(hbs`{{datasets/containers/dataset-health}}`); + + assert.equal(this.element.textContent.trim(), ''); + + // Template block usage: + await render(hbs` + {{#datasets/containers/dataset-health}} + template block text + {{/datasets/containers/dataset-health}} + `); + + assert.equal(this.element.textContent.trim(), 'template block text'); + }); +}); From 767da24cfa66e61fd1d34d3f6cbd94468c0f8c3f Mon Sep 17 00:00:00 2001 From: cptran777 Date: Fri, 27 Jul 2018 13:46:30 -0700 Subject: [PATCH 4/9] Edit tests to match up with current state of container --- .../containers/dataset-health-test.js | 13 ++++++++++ .../containers/dataset-health-test.ts | 26 ------------------- 2 files changed, 13 insertions(+), 26 deletions(-) create mode 100644 wherehows-web/tests/integration/components/datasets/containers/dataset-health-test.js delete mode 100644 wherehows-web/tests/integration/components/datasets/containers/dataset-health-test.ts diff --git a/wherehows-web/tests/integration/components/datasets/containers/dataset-health-test.js b/wherehows-web/tests/integration/components/datasets/containers/dataset-health-test.js new file mode 100644 index 0000000000..4222cf34f3 --- /dev/null +++ b/wherehows-web/tests/integration/components/datasets/containers/dataset-health-test.js @@ -0,0 +1,13 @@ +import { module, test } from 'qunit'; +import { setupRenderingTest } from 'ember-qunit'; +import { render } from '@ember/test-helpers'; +import hbs from 'htmlbars-inline-precompile'; + +module('Integration | Component | datasets/containers/dataset-health', function(hooks) { + setupRenderingTest(hooks); + // TODO: More meaningful tests as we continue with development + test('it renders', async function(assert) { + await render(hbs`{{datasets/containers/dataset-health}}`); + assert.ok(this.element, 'Renders without errors'); + }); +}); diff --git a/wherehows-web/tests/integration/components/datasets/containers/dataset-health-test.ts b/wherehows-web/tests/integration/components/datasets/containers/dataset-health-test.ts deleted file mode 100644 index 693120a466..0000000000 --- a/wherehows-web/tests/integration/components/datasets/containers/dataset-health-test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { module, test } from 'qunit'; -import { setupRenderingTest } from 'ember-qunit'; -import { render } from '@ember/test-helpers'; -import hbs from 'htmlbars-inline-precompile'; - -module('Integration | Component | datasets/containers/dataset-health', function(hooks) { - setupRenderingTest(hooks); - - test('it renders', async function(assert) { - // Set any properties with this.set('myProperty', 'value'); - // Handle any actions with this.set('myAction', function(val) { ... }); - - await render(hbs`{{datasets/containers/dataset-health}}`); - - assert.equal(this.element.textContent.trim(), ''); - - // Template block usage: - await render(hbs` - {{#datasets/containers/dataset-health}} - template block text - {{/datasets/containers/dataset-health}} - `); - - assert.equal(this.element.textContent.trim(), 'template block text'); - }); -}); From 4c48b6352ccc7562ae38280d1a5758cacc2a985f Mon Sep 17 00:00:00 2001 From: cptran777 Date: Fri, 27 Jul 2018 14:51:04 -0700 Subject: [PATCH 5/9] code cleanup - delete commenteed out code --- .../app/components/datasets/containers/dataset-health.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/wherehows-web/app/components/datasets/containers/dataset-health.ts b/wherehows-web/app/components/datasets/containers/dataset-health.ts index 4aa9177df7..7f24e1e60e 100644 --- a/wherehows-web/app/components/datasets/containers/dataset-health.ts +++ b/wherehows-web/app/components/datasets/containers/dataset-health.ts @@ -27,10 +27,6 @@ export default class DatasetHealthContainer extends Component { * @type {Task>, (a?: any) => TaskInstance>>>} */ getContainerDataTask = task(function*(this: DatasetHealthContainer): IterableIterator>> { - // Forms the tasks that are needed to fetch the data... to be determined - // const tasks = Object.values( - // getProperties(this, ['getDatasetOwnersTask', 'getSuggestedOwnersTask', 'getDatasetOwnerTypesTask']) - // ); - // yield* tasks.map(task => task.perform()); + // Do something in the future }); } From 62a43c0fa3ccef47a832316d98c242bacf8175b4 Mon Sep 17 00:00:00 2001 From: Seyi Adebajo Date: Mon, 30 Jul 2018 11:20:00 -0700 Subject: [PATCH 6/9] implements configurable suggestion threshold for compliance entities --- .../dataset-compliance-field-tag.ts | 12 ++- .../dataset-compliance-rollup-row.ts | 25 +++++- .../app/components/dataset-compliance.ts | 78 ++++++++++++++----- .../datasets/containers/dataset-compliance.ts | 23 +++++- .../app/constants/dataset-compliance.ts | 34 ++++---- .../containers/dataset-compliance.hbs | 1 + .../-dataset-compliance-entities.hbs | 2 + .../api/configurator/configurator.d.ts | 2 + .../app/typings/app/dataset-compliance.d.ts | 3 + .../utils/datasets/compliance-suggestions.ts | 24 +++--- wherehows-web/app/utils/object.ts | 4 +- 11 files changed, 159 insertions(+), 49 deletions(-) diff --git a/wherehows-web/app/components/dataset-compliance-field-tag.ts b/wherehows-web/app/components/dataset-compliance-field-tag.ts index b6ab39bf49..92e6dd0e22 100644 --- a/wherehows-web/app/components/dataset-compliance-field-tag.ts +++ b/wherehows-web/app/components/dataset-compliance-field-tag.ts @@ -67,6 +67,13 @@ export default class DatasetComplianceFieldTag extends Component { */ parentHasSingleTag: boolean; + /** + * Confidence percentage number used to filter high quality suggestions versus lower quality + * @type {number} + * @memberof DatasetComplianceFieldTag + */ + suggestionConfidenceThreshold: number; + /** * Stores the value of error result if the valuePattern is invalid * @type {string} @@ -204,8 +211,11 @@ export default class DatasetComplianceFieldTag extends Component { this: DatasetComplianceFieldTag ): boolean { const tagWithoutSuggestion = omit(get(this, 'tag'), ['suggestion']); + const suggestionConfidenceThreshold = get(this, 'suggestionConfidenceThreshold'); - return tagNeedsReview(get(this, 'complianceDataTypes'))(tagWithoutSuggestion); + return tagNeedsReview(get(this, 'complianceDataTypes'), { checkSuggestions: true, suggestionConfidenceThreshold })( + tagWithoutSuggestion + ); }); /** diff --git a/wherehows-web/app/components/dataset-compliance-rollup-row.ts b/wherehows-web/app/components/dataset-compliance-rollup-row.ts index 844c55cd24..e942edf33f 100644 --- a/wherehows-web/app/components/dataset-compliance-rollup-row.ts +++ b/wherehows-web/app/components/dataset-compliance-rollup-row.ts @@ -65,9 +65,17 @@ export default class DatasetComplianceRollupRow extends Component.extend({ /** * Reference to the compliance data types * @type {Array} + * @memberof DatasetComplianceRollupRow */ complianceDataTypes: Array; + /** + * Confidence percentage number used to filter high quality suggestions versus lower quality + * @type {number} + * @memberof DatasetComplianceRollupRow + */ + suggestionConfidenceThreshold: number; + /** * Flag indicating the field has a readonly attribute * @type ComputedProperty @@ -83,9 +91,16 @@ export default class DatasetComplianceRollupRow extends Component.extend({ isReviewRequested = computed( `fieldChangeSet.@each.{${changeSetReviewableAttributeTriggers}}`, 'complianceDataTypes', + 'suggestionConfidenceThreshold', function(this: DatasetComplianceRollupRow): boolean { - const tags = get(this, 'fieldChangeSet'); - const { length } = fieldTagsRequiringReview(get(this, 'complianceDataTypes'))(get(this, 'identifierField'))(tags); + const { fieldChangeSet: tags, suggestionConfidenceThreshold } = getProperties(this, [ + 'fieldChangeSet', + 'suggestionConfidenceThreshold' + ]); + const { length } = fieldTagsRequiringReview(get(this, 'complianceDataTypes'), { + checkSuggestions: true, + suggestionConfidenceThreshold + })(get(this, 'identifierField'))(tags); return !!length || tagsHaveNoneAndNotNoneType(tags); } @@ -172,10 +187,12 @@ export default class DatasetComplianceRollupRow extends Component.extend({ * @type {(ComputedProperty<{ identifierType: ComplianceFieldIdValue; logicalType: string; confidence: number } | void>)} * @memberof DatasetComplianceRollupRow */ - suggestion = computed('fieldProps.suggestion', 'suggestionAuthority', function( + suggestion = computed('fieldProps.suggestion', 'suggestionAuthority', 'suggestionConfidenceThreshold', function( this: DatasetComplianceRollupRow ): ISuggestedFieldTypeValues | void { - return getTagSuggestions(getWithDefault(this, 'fieldProps', {})); + const fieldProps = getWithDefault(this, 'fieldProps', {}); + + return getTagSuggestions({ suggestionConfidenceThreshold: get(this, 'suggestionConfidenceThreshold') })(fieldProps); }); /** diff --git a/wherehows-web/app/components/dataset-compliance.ts b/wherehows-web/app/components/dataset-compliance.ts index efa57e50f1..c90257fd0a 100644 --- a/wherehows-web/app/components/dataset-compliance.ts +++ b/wherehows-web/app/components/dataset-compliance.ts @@ -35,7 +35,8 @@ import { singleTagsInChangeSet, tagsForIdentifierField, overrideTagReadonly, - editableTags + editableTags, + lowQualitySuggestionConfidenceThreshold } from 'wherehows-web/constants'; import { getTagsSuggestions } from 'wherehows-web/utils/datasets/compliance-suggestions'; import { arrayMap, compact, isListUnique, iterateArrayAsync } from 'wherehows-web/utils/array'; @@ -70,12 +71,12 @@ import { notificationDialogActionFactory } from 'wherehows-web/utils/notificatio import validateMetadataObject, { complianceEntitiesTaxonomy } from 'wherehows-web/utils/datasets/compliance/metadata-schema'; +import { typeOf } from '@ember/utils'; const { complianceDataException, complianceFieldNotUnique, missingTypes, - helpText, missingPurgePolicy, missingDatasetSecurityClassification } = compliancePolicyStrings; @@ -104,7 +105,6 @@ export default class DatasetCompliance extends Component { filterBy: string; sortDirection: string; searchTerm: string; - helpText = helpText; _hasBadData: boolean; platform: IDatasetView['platform']; isCompliancePolicyAvailable: boolean = false; @@ -112,7 +112,7 @@ export default class DatasetCompliance extends Component { complianceInfo: undefined | IComplianceInfo; /** - * Lists the compliance entities that are entered via the advanced edititing interface + * Lists the compliance entities that are entered via the advanced editing interface * @type {Pick} * @memberof DatasetCompliance */ @@ -135,9 +135,17 @@ export default class DatasetCompliance extends Component { /** * Flag indicating the current compliance policy edit-view mode * @type {boolean} + * @memberof DatasetCompliance */ showGuidedComplianceEditMode: boolean = true; + /** + * Confidence percentage number used to filter high quality suggestions versus lower quality + * @type {number} + * @memberof DatasetCompliance + */ + suggestionConfidenceThreshold: number; + /** * Formatted JSON string representing the compliance entities for this dataset * @type {ComputedProperty} @@ -146,7 +154,6 @@ export default class DatasetCompliance extends Component { this: DatasetCompliance ): string { const entityAttrs = ['identifierField', 'identifierType', 'logicalType', 'nonOwner', 'valuePattern', 'readonly']; - //@ts-ignore property access path using dot notation limitation const entityMap: ISchemaFieldsToPolicy = get(this, 'columnIdFieldsToCurrentPrivacyPolicy'); const entitiesWithModifiableKeys = arrayMap((tag: IComplianceEntityWithMetadata) => pick(tag, entityAttrs))( (>[]).concat(...Object.values(entityMap)) @@ -334,6 +341,8 @@ export default class DatasetCompliance extends Component { this.searchTerm || set(this, 'searchTerm', ''); this.schemaFieldNamesMappedToDataTypes || (this.schemaFieldNamesMappedToDataTypes = []); this.complianceDataTypes || (this.complianceDataTypes = []); + typeOf(this.suggestionConfidenceThreshold) === 'number' || + set(this, 'suggestionConfidenceThreshold', lowQualitySuggestionConfidenceThreshold); } /** @@ -739,16 +748,26 @@ export default class DatasetCompliance extends Component { 'columnIdFieldsToCurrentPrivacyPolicy', 'complianceDataTypes', 'identifierFieldToSuggestion', + 'suggestionConfidenceThreshold', function(this: DatasetCompliance): Array { // schemaFieldNamesMappedToDataTypes is a dependency for CP columnIdFieldsToCurrentPrivacyPolicy, so no need to dep on that directly const changeSet = mergeComplianceEntitiesWithSuggestions( get(this, 'columnIdFieldsToCurrentPrivacyPolicy'), get(this, 'identifierFieldToSuggestion') ); + const suggestionThreshold = get(this, 'suggestionConfidenceThreshold'); // pass current changeSet state to parent handlers - run(() => next(this, 'notifyHandlerOfSuggestions', changeSet)); - run(() => next(this, 'notifyHandlerOfFieldsRequiringReview', get(this, 'complianceDataTypes'), changeSet)); + run(() => next(this, 'notifyHandlerOfSuggestions', suggestionThreshold, changeSet)); + run(() => + next( + this, + 'notifyHandlerOfFieldsRequiringReview', + suggestionThreshold, + get(this, 'complianceDataTypes'), + changeSet + ) + ); return changeSet; } @@ -764,14 +783,16 @@ export default class DatasetCompliance extends Component { 'fieldReviewOption', 'compliancePolicyChangeSet', 'complianceDataTypes', + 'suggestionConfidenceThreshold', function(this: DatasetCompliance): Array { - const { compliancePolicyChangeSet: changeSet, complianceDataTypes } = getProperties(this, [ - 'compliancePolicyChangeSet', - 'complianceDataTypes' - ]); + const { + compliancePolicyChangeSet: changeSet, + complianceDataTypes, + suggestionConfidenceThreshold + } = getProperties(this, ['compliancePolicyChangeSet', 'complianceDataTypes', 'suggestionConfidenceThreshold']); return get(this, 'fieldReviewOption') === 'showReview' - ? tagsRequiringReview(complianceDataTypes)(changeSet) + ? tagsRequiringReview(complianceDataTypes, { checkSuggestions: true, suggestionConfidenceThreshold })(changeSet) : changeSet; } ); @@ -788,9 +809,10 @@ export default class DatasetCompliance extends Component { changeSetReviewWithoutSuggestionCheck = computed('changeSetReview', function( this: DatasetCompliance ): Array { - return tagsRequiringReview(get(this, 'complianceDataTypes'), { checkSuggestions: false })( - get(this, 'changeSetReview') - ); + return tagsRequiringReview(get(this, 'complianceDataTypes'), { + checkSuggestions: false, + suggestionConfidenceThreshold: 0 // irrelevant value set to 0 since checkSuggestions flag is false above + })(get(this, 'changeSetReview')); }); /** @@ -801,8 +823,17 @@ export default class DatasetCompliance extends Component { changeSetReview = computed( `compliancePolicyChangeSet.@each.{${changeSetReviewableAttributeTriggers}}`, 'complianceDataTypes', + 'suggestionConfidenceThreshold', function(this: DatasetCompliance): Array { - return tagsRequiringReview(get(this, 'complianceDataTypes'))(get(this, 'compliancePolicyChangeSet')); + const { suggestionConfidenceThreshold, compliancePolicyChangeSet } = getProperties(this, [ + 'suggestionConfidenceThreshold', + 'compliancePolicyChangeSet' + ]); + + return tagsRequiringReview(get(this, 'complianceDataTypes'), { + checkSuggestions: true, + suggestionConfidenceThreshold + })(compliancePolicyChangeSet); } ); @@ -866,19 +897,25 @@ export default class DatasetCompliance extends Component { /** * Invokes external action with flag indicating that at least 1 suggestion exists for a field in the changeSet + * @param {number} suggestionConfidenceThreshold confidence threshold for filtering out higher quality suggestions * @param {Array} changeSet */ - notifyHandlerOfSuggestions = (changeSet: Array): void => { - const hasChangeSetSuggestions = !!compact(getTagsSuggestions(changeSet)).length; + notifyHandlerOfSuggestions = ( + suggestionConfidenceThreshold: number, + changeSet: Array + ): void => { + const hasChangeSetSuggestions = !!compact(getTagsSuggestions({ suggestionConfidenceThreshold })(changeSet)).length; this.notifyOnChangeSetSuggestions(hasChangeSetSuggestions); }; /** * Invokes external action with flag indicating that a field in the tags requires user review + * @param {number} suggestionConfidenceThreshold confidence threshold for filtering out higher quality suggestions * @param {Array} complianceDataTypes * @param {Array} tags */ notifyHandlerOfFieldsRequiringReview = ( + suggestionConfidenceThreshold: number, complianceDataTypes: Array, tags: Array ): void => { @@ -886,7 +923,10 @@ export default class DatasetCompliance extends Component { assert('expected complianceDataTypes to be of type `array`', Array.isArray(complianceDataTypes)); assert('expected tags to be of type `array`', Array.isArray(tags)); - const hasChangeSetDrift = !!tagsRequiringReview(complianceDataTypes)(tags).length; + const hasChangeSetDrift = !!tagsRequiringReview(complianceDataTypes, { + checkSuggestions: true, + suggestionConfidenceThreshold + })(tags).length; this.notifyOnChangeSetRequiresReview(hasChangeSetDrift); }; diff --git a/wherehows-web/app/components/datasets/containers/dataset-compliance.ts b/wherehows-web/app/components/datasets/containers/dataset-compliance.ts index 2b9e450c02..775a0119e9 100644 --- a/wherehows-web/app/components/datasets/containers/dataset-compliance.ts +++ b/wherehows-web/app/components/datasets/containers/dataset-compliance.ts @@ -21,12 +21,20 @@ import { import { columnDataTypesAndFieldNames } from 'wherehows-web/utils/api/datasets/columns'; import { readDatasetSchemaByUrn } from 'wherehows-web/utils/api/datasets/schema'; import { readComplianceDataTypes } from 'wherehows-web/utils/api/list/compliance-datatypes'; -import { compliancePolicyStrings, removeReadonlyAttr, editableTags, SuggestionIntent } from 'wherehows-web/constants'; +import { + compliancePolicyStrings, + removeReadonlyAttr, + editableTags, + SuggestionIntent, + lowQualitySuggestionConfidenceThreshold +} from 'wherehows-web/constants'; import { iterateArrayAsync } from 'wherehows-web/utils/array'; import validateMetadataObject, { complianceEntitiesTaxonomy } from 'wherehows-web/utils/datasets/compliance/metadata-schema'; import { notificationDialogActionFactory } from 'wherehows-web/utils/notifications/notifications'; +import Configurator from 'wherehows-web/services/configurator'; +import { typeOf } from '@ember/utils'; /** * Type alias for the response when container data items are batched @@ -49,6 +57,7 @@ type BatchContainerDataResult = Pick< | 'complianceSuggestion' | 'schemaFieldNamesMappedToDataTypes' | 'schemaless' + | 'suggestionConfidenceThreshold' >; const { successUpdating, failedUpdating, successUploading, invalidPolicyData } = compliancePolicyStrings; @@ -128,6 +137,13 @@ export default class DatasetComplianceContainer extends Component { */ datasetName: string = ''; + /** + * Confidence percentage number used to filter high quality suggestions versus lower quality + * @type {number} + * @memberof DatasetComplianceContainer + */ + suggestionConfidenceThreshold: number = lowQualitySuggestionConfidenceThreshold; + /** * The urn identifier for the dataset * @type {string} @@ -177,11 +193,16 @@ export default class DatasetComplianceContainer extends Component { ]); const schemaFieldNamesMappedToDataTypes = await iterateArrayAsync(columnDataTypesAndFieldNames)(columns); const { containingPersonalData, fromUpstream } = complianceInfo; + let suggestionConfidenceThreshold = Configurator.getConfig('suggestionConfidenceThreshold'); + // convert to fractional percentage if valid number is present + typeOf(suggestionConfidenceThreshold) === 'number' && + (suggestionConfidenceThreshold = suggestionConfidenceThreshold / 100); this.notifyPiiStatus(!!containingPersonalData); this.onCompliancePolicyStateChange.call(this, { isNewComplianceInfo, fromUpstream: !!fromUpstream }); return setProperties(this, { + suggestionConfidenceThreshold, isNewComplianceInfo, complianceInfo, complianceDataTypes, diff --git a/wherehows-web/app/constants/dataset-compliance.ts b/wherehows-web/app/constants/dataset-compliance.ts index 47c7808c96..7d5c9e2607 100644 --- a/wherehows-web/app/constants/dataset-compliance.ts +++ b/wherehows-web/app/constants/dataset-compliance.ts @@ -40,11 +40,6 @@ const compliancePolicyStrings = { failedUpdating: 'An error occurred while saving.', successUploading: 'Metadata successfully updated! Please confirm and complete subsequent metadata information.', invalidPolicyData: 'Received policy in an unexpected format! Please check the provided attributes and try again.', - helpText: { - classification: - 'This security classification is from go/dht and should be good enough in most cases. ' + - 'You can optionally override it if required by house security.' - }, missingPurgePolicy: 'Please specify a Compliance Purge Policy', missingDatasetSecurityClassification: 'Please specify a security classification for this dataset.' }; @@ -183,10 +178,18 @@ const suggestedIdentifierTypesInList = (suggestion: ISuggestedFieldTypeValues | * @param {SchemaFieldToSuggestedValue} suggestion * @param {SuggestionIntent} suggestionAuthority * @param {ComplianceFieldIdValue | NonIdLogicalType | null} identifierType + * @param {number} suggestionConfidenceThreshold confidence threshold for filtering out higher quality suggestions * @return {boolean} */ -const tagSuggestionNeedsReview = ({ suggestion, suggestionAuthority, identifierType }: IComplianceChangeSet): boolean => - suggestion && suggestion.identifierType !== identifierType && isHighConfidenceSuggestion(suggestion) +const tagSuggestionNeedsReview = ({ + suggestion, + suggestionAuthority, + identifierType, + suggestionConfidenceThreshold +}: IComplianceChangeSet & { suggestionConfidenceThreshold: number }): boolean => + suggestion && + suggestion.identifierType !== identifierType && + isHighConfidenceSuggestion(suggestion, suggestionConfidenceThreshold) ? !suggestionAuthority : false; @@ -223,14 +226,14 @@ const tagValuePatternNeedsReview = ({ valuePattern }: IComplianceChangeSet): boo * checking steps * @return {(tag: IComplianceChangeSet) => boolean} */ -const tagNeedsReview = (complianceDataTypes: Array, options?: IComplianceTagReviewOptions) => +const tagNeedsReview = (complianceDataTypes: Array, options: IComplianceTagReviewOptions) => /** * Checks if a compliance tag needs to be reviewed against a set of rules * @param {IComplianceChangeSet} tag * @return {boolean} */ (tag: IComplianceChangeSet): boolean => { - const { checkSuggestions } = options || { checkSuggestions: true }; + const { checkSuggestions, suggestionConfidenceThreshold } = options; const { isDirty, privacyPolicyExists, identifierType, logicalType } = tag; let isReviewRequired = false; @@ -241,7 +244,7 @@ const tagNeedsReview = (complianceDataTypes: Array, options // Check that a hi confidence suggestion exists and the identifierType does not match the change set item if (checkSuggestions) { - isReviewRequired = isReviewRequired || tagSuggestionNeedsReview(tag); + isReviewRequired = isReviewRequired || tagSuggestionNeedsReview({ ...tag, suggestionConfidenceThreshold }); } // Ensure that tag has a logical type and nonOwner flag is set when tag is of id type @@ -391,17 +394,20 @@ const singleTagsInChangeSet = ( * @param {IComplianceTagReviewOptions} options * @return {(array: Array) => Array} */ -const tagsRequiringReview = (complianceDataTypes: Array, options?: IComplianceTagReviewOptions) => +const tagsRequiringReview = (complianceDataTypes: Array, options: IComplianceTagReviewOptions) => arrayFilter(tagNeedsReview(complianceDataTypes, options)); /** * Lists the tags for a specific identifier field that need to be reviewed * @param {Array} complianceDataTypes + * @param {IComplianceTagReviewOptions} options * @return {(identifierField: string) => (tags: Array) => Array} */ -const fieldTagsRequiringReview = (complianceDataTypes: Array) => (identifierField: string) => ( - tags: Array -) => tagsRequiringReview(complianceDataTypes)(tagsForIdentifierField(identifierField)(tags)); +const fieldTagsRequiringReview = ( + complianceDataTypes: Array, + options: IComplianceTagReviewOptions +) => (identifierField: string) => (tags: Array) => + tagsRequiringReview(complianceDataTypes, options)(tagsForIdentifierField(identifierField)(tags)); /** * Extracts a suggestion for a field from a suggestion map and merges a compliance entity with the suggestion diff --git a/wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs b/wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs index ee7b60c7ba..aa16de1c42 100644 --- a/wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs +++ b/wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs @@ -25,6 +25,7 @@ platform=platform complianceInfo=complianceInfo complianceSuggestion=complianceSuggestion + suggestionConfidenceThreshold=suggestionConfidenceThreshold isNewComplianceInfo=isNewComplianceInfo schemaFieldNamesMappedToDataTypes=schemaFieldNamesMappedToDataTypes complianceDataTypes=complianceDataTypes diff --git a/wherehows-web/app/templates/datasets/dataset-compliance/-dataset-compliance-entities.hbs b/wherehows-web/app/templates/datasets/dataset-compliance/-dataset-compliance-entities.hbs index 3e07075fdb..f089ef3a31 100644 --- a/wherehows-web/app/templates/datasets/dataset-compliance/-dataset-compliance-entities.hbs +++ b/wherehows-web/app/templates/datasets/dataset-compliance/-dataset-compliance-entities.hbs @@ -136,6 +136,7 @@ field=field isNewComplianceInfo=isNewComplianceInfo complianceDataTypes=complianceDataTypes + suggestionConfidenceThreshold=suggestionConfidenceThreshold onFieldDblClick=(action "onFieldDblClick") onFieldTagAdded=(action "onFieldTagAdded") onFieldTagRemoved=(action "onFieldTagRemoved") @@ -216,6 +217,7 @@ sourceTag=tag parentHasSingleTag=row.hasSingleTag tagDidChange=(action "tagPropertiesUpdated") + suggestionConfidenceThreshold=suggestionConfidenceThreshold complianceFieldIdDropdownOptions=complianceFieldIdDropdownOptions complianceDataTypes=complianceDataTypes as |tagRowComponent| }} diff --git a/wherehows-web/app/typings/api/configurator/configurator.d.ts b/wherehows-web/app/typings/api/configurator/configurator.d.ts index 03b00e8542..547b80f005 100644 --- a/wherehows-web/app/typings/api/configurator/configurator.d.ts +++ b/wherehows-web/app/typings/api/configurator/configurator.d.ts @@ -10,6 +10,8 @@ interface IAppConfig { JitAclAccessWhitelist: Array | void; shouldShowDatasetLineage: boolean; shouldShowDatasetHealth: boolean; + // confidence threshold for filtering out higher quality suggestions + suggestionConfidenceThreshold: number; tracking: { isEnabled: boolean; trackers: { diff --git a/wherehows-web/app/typings/app/dataset-compliance.d.ts b/wherehows-web/app/typings/app/dataset-compliance.d.ts index 59cf8dd49b..53e9aaa37f 100644 --- a/wherehows-web/app/typings/app/dataset-compliance.d.ts +++ b/wherehows-web/app/typings/app/dataset-compliance.d.ts @@ -23,7 +23,10 @@ interface IDatasetComplianceActions { * @interface IComplianceTagReviewOptions */ interface IComplianceTagReviewOptions { + // flag determines if suggested values are considered in tag(IComplianceChangeSet) review check checkSuggestions: boolean; + // confidence threshold for filtering out higher quality suggestions + suggestionConfidenceThreshold: number; } /** diff --git a/wherehows-web/app/utils/datasets/compliance-suggestions.ts b/wherehows-web/app/utils/datasets/compliance-suggestions.ts index a4ede90e58..6bd7e77ad5 100644 --- a/wherehows-web/app/utils/datasets/compliance-suggestions.ts +++ b/wherehows-web/app/utils/datasets/compliance-suggestions.ts @@ -1,4 +1,3 @@ -import { lowQualitySuggestionConfidenceThreshold } from 'wherehows-web/constants'; import { arrayMap } from 'wherehows-web/utils/array'; import { IComplianceChangeSet, ISuggestedFieldTypeValues } from 'wherehows-web/typings/app/dataset-compliance'; @@ -6,23 +5,28 @@ import { IComplianceChangeSet, ISuggestedFieldTypeValues } from 'wherehows-web/t * Takes a list of suggestions with confidence values, and if the confidence is greater than * a low confidence threshold * @param {number} confidenceLevel percentage indicating how confidence the system is in the suggested value + * @param {number} suggestionConfidenceThreshold threshold number to consider as a valid suggestion * @return {boolean} */ -const isHighConfidenceSuggestion = ({ confidenceLevel = 0 }: { confidenceLevel: number }): boolean => - confidenceLevel > lowQualitySuggestionConfidenceThreshold; +const isHighConfidenceSuggestion = ( + { confidenceLevel = 0 }: { confidenceLevel: number }, + suggestionConfidenceThreshold: number +): boolean => confidenceLevel > suggestionConfidenceThreshold; /** * Extracts the tag suggestion from an IComplianceChangeSet tag. * If a suggestionAuthority property exists on the tag, then the user has already either accepted or ignored * the suggestion for this tag. It's value should not be taken into account on re-renders, * in place, this substitutes an empty suggestion - * @param {IComplianceChangeSet} tag - * @return {{identifierType: IComplianceChangeSet.identifierType, logicalType: IComplianceChangeSet.logicalType, confidence: number} | void} + * @param {number} suggestionConfidenceThreshold confidence threshold for filtering out higher quality suggestions + * @return {(tag?: IComplianceChangeSet) => (ISuggestedFieldTypeValues | void)} */ -const getTagSuggestions = (tag: IComplianceChangeSet = {}): ISuggestedFieldTypeValues | void => { +const getTagSuggestions = ({ suggestionConfidenceThreshold }: { suggestionConfidenceThreshold: number }) => ( + tag: IComplianceChangeSet = {} +): ISuggestedFieldTypeValues | void => { const { suggestion } = tag; - if (suggestion && isHighConfidenceSuggestion(suggestion)) { + if (suggestion && isHighConfidenceSuggestion(suggestion, suggestionConfidenceThreshold)) { const { identifierType, logicalType, confidenceLevel: confidence } = suggestion; return { identifierType, logicalType, confidence: +(confidence * 100).toFixed(2) }; } @@ -30,8 +34,10 @@ const getTagSuggestions = (tag: IComplianceChangeSet = {}) /** * Gets the suggestions for a list of IComplianceChangeSet fields - * @type {(array: Array) => Array<{identifierType: ComplianceFieldIdValue | NonIdLogicalType | null; logicalType: IdLogicalType | null; confidence: number} | void>} + * @param {number} suggestionConfidenceThreshold + * @return {(array: Array) => Array} */ -const getTagsSuggestions = arrayMap(getTagSuggestions); +const getTagsSuggestions = ({ suggestionConfidenceThreshold }: { suggestionConfidenceThreshold: number }) => + arrayMap(getTagSuggestions({ suggestionConfidenceThreshold })); export { isHighConfidenceSuggestion, getTagSuggestions, getTagsSuggestions }; diff --git a/wherehows-web/app/utils/object.ts b/wherehows-web/app/utils/object.ts index 43964fde52..01ba2361a2 100644 --- a/wherehows-web/app/utils/object.ts +++ b/wherehows-web/app/utils/object.ts @@ -27,7 +27,9 @@ interface IPartialOrIdentityTypeFn { * @param {Array} [droppedKeys=[]] the list of attributes on T to be dropped * @returns {IPartialOrIdentityTypeFn} */ -const fleece = (droppedKeys: Array = []): IPartialOrIdentityTypeFn => o => { +const fleece = (droppedKeys: Array = []): IPartialOrIdentityTypeFn => ( + o: T +): Partial | T => { const partialResult = Object.assign({}, o); return droppedKeys.reduce((partial, key) => { From 00669afefbb5b75df35cdb84a245088fc81bcdaf Mon Sep 17 00:00:00 2001 From: Seyi Adebajo Date: Mon, 30 Jul 2018 11:21:02 -0700 Subject: [PATCH 7/9] adds mid-tier config value for suggestion threshold value --- wherehows-frontend/app/controllers/Application.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wherehows-frontend/app/controllers/Application.java b/wherehows-frontend/app/controllers/Application.java index 1921a877ab..e644cd5896 100644 --- a/wherehows-frontend/app/controllers/Application.java +++ b/wherehows-frontend/app/controllers/Application.java @@ -68,6 +68,8 @@ public class Application extends Controller { Play.application().configuration().getBoolean("linkedin.show.dataset.lineage", false); private static final Boolean WHZ_SHOW_DS_HEALTH = Play.application().configuration().getBoolean("linkedin.show.dataset.health", false); + private static final Boolean WHZ_SUGGESTION_CONFIDENCE_THRESHOLD = + Play.application().configuration().getString("linkedin.suggestion.confidence.threshold", "50"); private static final String WHZ_WIKI_LINKS__GDRP_PII = Play.application().configuration().getString("linkedin.links.wiki.gdprPii", ""); @@ -202,6 +204,7 @@ public class Application extends Controller { config.put("isInternal", IS_INTERNAL); config.put("shouldShowDatasetLineage", WHZ_SHOW_LINEAGE); config.put("shouldShowDatasetHealth", WHZ_SHOW_DS_HEALTH); + config.put("suggestionConfidenceThreshold", Integer.parseInt(WHZ_SUGGESTION_CONFIDENCE_THRESHOLD)); config.set("wikiLinks", wikiLinks()); config.set("JitAclAccessWhitelist", Json.toJson(StringUtils.split(JIT_ACL_WHITELIST, ','))); config.set("tracking", trackingInfo()); From 3b726e47e119b551ae4b8ac3fa5e9fd3e1584510 Mon Sep 17 00:00:00 2001 From: Seyi Adebajo Date: Mon, 30 Jul 2018 12:24:16 -0700 Subject: [PATCH 8/9] modifies tests to comply with changes to suggestion related functions api --- .../utils/datasets/compliance-suggestions.ts | 2 +- .../unit/constants/dataset-compliance-test.js | 18 ++++++++++++--- .../datasets/compliance-suggestions-test.js | 23 ++++++++++++++----- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/wherehows-web/app/utils/datasets/compliance-suggestions.ts b/wherehows-web/app/utils/datasets/compliance-suggestions.ts index 6bd7e77ad5..9ffbe4a02c 100644 --- a/wherehows-web/app/utils/datasets/compliance-suggestions.ts +++ b/wherehows-web/app/utils/datasets/compliance-suggestions.ts @@ -10,7 +10,7 @@ import { IComplianceChangeSet, ISuggestedFieldTypeValues } from 'wherehows-web/t */ const isHighConfidenceSuggestion = ( { confidenceLevel = 0 }: { confidenceLevel: number }, - suggestionConfidenceThreshold: number + suggestionConfidenceThreshold: number = 0 ): boolean => confidenceLevel > suggestionConfidenceThreshold; /** diff --git a/wherehows-web/tests/unit/constants/dataset-compliance-test.js b/wherehows-web/tests/unit/constants/dataset-compliance-test.js index c2d1fe0a19..1c0f5bb70d 100644 --- a/wherehows-web/tests/unit/constants/dataset-compliance-test.js +++ b/wherehows-web/tests/unit/constants/dataset-compliance-test.js @@ -8,7 +8,8 @@ import { PurgePolicy, initialComplianceObjectFactory, isRecentSuggestion, - tagNeedsReview + tagNeedsReview, + lowQualitySuggestionConfidenceThreshold } from 'wherehows-web/constants'; import complianceDataTypes from 'wherehows-web/mirage/fixtures/compliance-data-types'; import { mockTimeStamps } from 'wherehows-web/tests/helpers/datasets/compliance-policy/recent-suggestions-constants'; @@ -17,6 +18,11 @@ import { hdfsUrn } from 'wherehows-web/mirage/fixtures/urn'; module('Unit | Constants | dataset compliance'); +const complianceTagReviewOptions = { + checkSuggestions: false, + suggestionConfidenceThreshold: lowQualitySuggestionConfidenceThreshold +}; + test('initialComplianceObjectFactory', function(assert) { assert.expect(2); const mockUrn = hdfsUrn; @@ -55,14 +61,20 @@ test('isRecentSuggestion correctly determines if a suggestion is recent or not', test('tagNeedsReview exists', function(assert) { assert.ok(typeof tagNeedsReview === 'function', 'tagNeedsReview is a function'); - assert.ok(typeof tagNeedsReview([])({}) === 'boolean', 'tagNeedsReview returns a boolean'); + assert.ok( + typeof tagNeedsReview([], complianceTagReviewOptions)({}) === 'boolean', + 'tagNeedsReview returns a boolean' + ); }); test('tagNeedsReview correctly determines if a fieldChangeSet requires review', function(assert) { assert.expect(mockFieldChangeSets.length); mockFieldChangeSets.forEach(changeSet => - assert.ok(tagNeedsReview(complianceDataTypes)(changeSet) === changeSet.__requiresReview__, changeSet.__msg__) + assert.ok( + tagNeedsReview(complianceDataTypes, complianceTagReviewOptions)(changeSet) === changeSet.__requiresReview__, + changeSet.__msg__ + ) ); }); diff --git a/wherehows-web/tests/unit/utils/datasets/compliance-suggestions-test.js b/wherehows-web/tests/unit/utils/datasets/compliance-suggestions-test.js index bd3862f2a9..0cf7685106 100644 --- a/wherehows-web/tests/unit/utils/datasets/compliance-suggestions-test.js +++ b/wherehows-web/tests/unit/utils/datasets/compliance-suggestions-test.js @@ -8,21 +8,30 @@ test('isHighConfidenceSuggestion correctly determines the confidence of a sugges let result = isHighConfidenceSuggestion({}); assert.notOk(result, 'should be false if no arguments are supplied'); - result = isHighConfidenceSuggestion({ confidenceLevel: lowQualitySuggestionConfidenceThreshold + 1 }); + result = isHighConfidenceSuggestion( + { confidenceLevel: lowQualitySuggestionConfidenceThreshold + 1 }, + lowQualitySuggestionConfidenceThreshold + ); assert.ok( result, `should be true if the confidence value is greater than ${lowQualitySuggestionConfidenceThreshold}` ); - result = isHighConfidenceSuggestion({ confidenceLevel: lowQualitySuggestionConfidenceThreshold - 1 }); + result = isHighConfidenceSuggestion( + { confidenceLevel: lowQualitySuggestionConfidenceThreshold - 1 }, + lowQualitySuggestionConfidenceThreshold + ); assert.notOk( result, `should be false if the confidence value is less than ${lowQualitySuggestionConfidenceThreshold}` ); - result = isHighConfidenceSuggestion({ confidenceLevel: lowQualitySuggestionConfidenceThreshold }); + result = isHighConfidenceSuggestion( + { confidenceLevel: lowQualitySuggestionConfidenceThreshold }, + lowQualitySuggestionConfidenceThreshold + ); assert.notOk( result, @@ -42,15 +51,17 @@ test('getTagSuggestions correctly extracts suggestions from a compliance field', suggestionAuthority: SuggestionIntent.accept }; - let result = getTagSuggestions({}); + let result = getTagSuggestions({ suggestionConfidenceThreshold: lowQualitySuggestionConfidenceThreshold })({}); assert.ok(typeof result === 'undefined', 'expected undefined return when the argument is an empty object'); - result = getTagSuggestions(); + result = getTagSuggestions({ suggestionConfidenceThreshold: lowQualitySuggestionConfidenceThreshold })(); assert.ok(typeof result === 'undefined', 'expected undefined return when no argument is supplied'); - result = getTagSuggestions({ suggestion: changeSetField.suggestion }); + result = getTagSuggestions({ suggestionConfidenceThreshold: lowQualitySuggestionConfidenceThreshold })({ + suggestion: changeSetField.suggestion + }); assert.deepEqual( result, From b3c3321bd67df68a466433d0e37c0ae876f843c8 Mon Sep 17 00:00:00 2001 From: Seyi Adebajo Date: Mon, 30 Jul 2018 12:59:22 -0700 Subject: [PATCH 9/9] corrects type annotation for suggestionConfidenceThreshold --- wherehows-frontend/app/controllers/Application.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wherehows-frontend/app/controllers/Application.java b/wherehows-frontend/app/controllers/Application.java index e644cd5896..c646483a74 100644 --- a/wherehows-frontend/app/controllers/Application.java +++ b/wherehows-frontend/app/controllers/Application.java @@ -68,7 +68,7 @@ public class Application extends Controller { Play.application().configuration().getBoolean("linkedin.show.dataset.lineage", false); private static final Boolean WHZ_SHOW_DS_HEALTH = Play.application().configuration().getBoolean("linkedin.show.dataset.health", false); - private static final Boolean WHZ_SUGGESTION_CONFIDENCE_THRESHOLD = + private static final String WHZ_SUGGESTION_CONFIDENCE_THRESHOLD = Play.application().configuration().getString("linkedin.suggestion.confidence.threshold", "50"); private static final String WHZ_WIKI_LINKS__GDRP_PII =