yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-04 04:39:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/linkedin/WhereHows

Browse Source
This commit is contained in:
Jyoti Wadhwani 2018-07-30 14:36:51 -07:00
commit 7d6958dcb1
24 changed files with 327 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
wherehows-frontend/app/controllers/Application.java
View File

@ -66,6 +66,10 @@ public class Application extends Controller {
private static final Boolean HTTPS_REDIRECT = Play.application().configuration().getBoolean("https.redirect", false);
private static final Boolean WHZ_SHOW_LINEAGE =
Play.application().configuration().getBoolean("linkedin.show.dataset.lineage", false);
private static final Boolean WHZ_SHOW_DS_HEALTH =
Play.application().configuration().getBoolean("linkedin.show.dataset.health", false);
private static final String WHZ_SUGGESTION_CONFIDENCE_THRESHOLD =
Play.application().configuration().getString("linkedin.suggestion.confidence.threshold", "50");
private static final String WHZ_WIKI_LINKS__GDRP_PII =
Play.application().configuration().getString("linkedin.links.wiki.gdprPii", "");
@ -199,6 +203,8 @@ public class Application extends Controller {
config.put("appVersion", APP_VERSION);
config.put("isInternal", IS_INTERNAL);
config.put("shouldShowDatasetLineage", WHZ_SHOW_LINEAGE);
config.put("shouldShowDatasetHealth", WHZ_SHOW_DS_HEALTH);
config.put("suggestionConfidenceThreshold", Integer.parseInt(WHZ_SUGGESTION_CONFIDENCE_THRESHOLD));
config.set("wikiLinks", wikiLinks());
config.set("JitAclAccessWhitelist", Json.toJson(StringUtils.split(JIT_ACL_WHITELIST, ',')));
config.set("tracking", trackingInfo());

12
wherehows-web/app/components/dataset-compliance-field-tag.ts
View File

@ -67,6 +67,13 @@ export default class DatasetComplianceFieldTag extends Component {
*/
parentHasSingleTag: boolean;
/**
* Confidence percentage number used to filter high quality suggestions versus lower quality
* @type {number}
* @memberof DatasetComplianceFieldTag
*/
suggestionConfidenceThreshold: number;
/**
* Stores the value of error result if the valuePattern is invalid
* @type {string}
@ -204,8 +211,11 @@ export default class DatasetComplianceFieldTag extends Component {
this: DatasetComplianceFieldTag
): boolean {
const tagWithoutSuggestion = <IComplianceChangeSet>omit<IComplianceChangeSet>(get(this, 'tag'), ['suggestion']);
const suggestionConfidenceThreshold = get(this, 'suggestionConfidenceThreshold');
return tagNeedsReview(get(this, 'complianceDataTypes'))(tagWithoutSuggestion);
return tagNeedsReview(get(this, 'complianceDataTypes'), { checkSuggestions: true, suggestionConfidenceThreshold })(
tagWithoutSuggestion
);
});
/**

25
wherehows-web/app/components/dataset-compliance-rollup-row.ts
View File

@ -65,9 +65,17 @@ export default class DatasetComplianceRollupRow extends Component.extend({
/**
* Reference to the compliance data types
* @type {Array<IComplianceDataType>}
* @memberof DatasetComplianceRollupRow
*/
complianceDataTypes: Array<IComplianceDataType>;
/**
* Confidence percentage number used to filter high quality suggestions versus lower quality
* @type {number}
* @memberof DatasetComplianceRollupRow
*/
suggestionConfidenceThreshold: number;
/**
* Flag indicating the field has a readonly attribute
* @type ComputedProperty<boolean>
@ -83,9 +91,16 @@ export default class DatasetComplianceRollupRow extends Component.extend({
isReviewRequested = computed(
`fieldChangeSet.@each.{${changeSetReviewableAttributeTriggers}}`,
'complianceDataTypes',
'suggestionConfidenceThreshold',
function(this: DatasetComplianceRollupRow): boolean {
const tags = get(this, 'fieldChangeSet');
const { length } = fieldTagsRequiringReview(get(this, 'complianceDataTypes'))(get(this, 'identifierField'))(tags);
const { fieldChangeSet: tags, suggestionConfidenceThreshold } = getProperties(this, [
'fieldChangeSet',
'suggestionConfidenceThreshold'
]);
const { length } = fieldTagsRequiringReview(get(this, 'complianceDataTypes'), {
checkSuggestions: true,
suggestionConfidenceThreshold
})(get(this, 'identifierField'))(tags);
return !!length || tagsHaveNoneAndNotNoneType(tags);
}
@ -172,10 +187,12 @@ export default class DatasetComplianceRollupRow extends Component.extend({
* @type {(ComputedProperty<{ identifierType: ComplianceFieldIdValue; logicalType: string; confidence: number } | void>)}
* @memberof DatasetComplianceRollupRow
*/
suggestion = computed('fieldProps.suggestion', 'suggestionAuthority', function(
suggestion = computed('fieldProps.suggestion', 'suggestionAuthority', 'suggestionConfidenceThreshold', function(
this: DatasetComplianceRollupRow
): ISuggestedFieldTypeValues | void {
return getTagSuggestions(getWithDefault(this, 'fieldProps', <IComplianceChangeSet>{}));
const fieldProps = getWithDefault(this, 'fieldProps', <IComplianceChangeSet>{});
return getTagSuggestions({ suggestionConfidenceThreshold: get(this, 'suggestionConfidenceThreshold') })(fieldProps);
});
/**

78
wherehows-web/app/components/dataset-compliance.ts
View File

@ -35,7 +35,8 @@ import {
singleTagsInChangeSet,
tagsForIdentifierField,
overrideTagReadonly,
editableTags
editableTags,
lowQualitySuggestionConfidenceThreshold
} from 'wherehows-web/constants';
import { getTagsSuggestions } from 'wherehows-web/utils/datasets/compliance-suggestions';
import { arrayMap, compact, isListUnique, iterateArrayAsync } from 'wherehows-web/utils/array';
@ -70,12 +71,12 @@ import { notificationDialogActionFactory } from 'wherehows-web/utils/notificatio
import validateMetadataObject, {
complianceEntitiesTaxonomy
} from 'wherehows-web/utils/datasets/compliance/metadata-schema';
import { typeOf } from '@ember/utils';
const {
complianceDataException,
complianceFieldNotUnique,
missingTypes,
helpText,
missingPurgePolicy,
missingDatasetSecurityClassification
} = compliancePolicyStrings;
@ -104,7 +105,6 @@ export default class DatasetCompliance extends Component {
filterBy: string;
sortDirection: string;
searchTerm: string;
helpText = helpText;
_hasBadData: boolean;
platform: IDatasetView['platform'];
isCompliancePolicyAvailable: boolean = false;
@ -112,7 +112,7 @@ export default class DatasetCompliance extends Component {
complianceInfo: undefined | IComplianceInfo;
/**
* Lists the compliance entities that are entered via the advanced edititing interface
* Lists the compliance entities that are entered via the advanced editing interface
* @type {Pick<IComplianceInfo, 'complianceEntities'>}
* @memberof DatasetCompliance
*/
@ -135,9 +135,17 @@ export default class DatasetCompliance extends Component {
/**
* Flag indicating the current compliance policy edit-view mode
* @type {boolean}
* @memberof DatasetCompliance
*/
showGuidedComplianceEditMode: boolean = true;
/**
* Confidence percentage number used to filter high quality suggestions versus lower quality
* @type {number}
* @memberof DatasetCompliance
*/
suggestionConfidenceThreshold: number;
/**
* Formatted JSON string representing the compliance entities for this dataset
* @type {ComputedProperty<string>}
@ -146,7 +154,6 @@ export default class DatasetCompliance extends Component {
this: DatasetCompliance
): string {
const entityAttrs = ['identifierField', 'identifierType', 'logicalType', 'nonOwner', 'valuePattern', 'readonly'];
//@ts-ignore property access path using dot notation limitation
const entityMap: ISchemaFieldsToPolicy = get(this, 'columnIdFieldsToCurrentPrivacyPolicy');
const entitiesWithModifiableKeys = arrayMap((tag: IComplianceEntityWithMetadata) => pick(tag, entityAttrs))(
(<Array<IComplianceEntityWithMetadata>>[]).concat(...Object.values(entityMap))
@ -334,6 +341,8 @@ export default class DatasetCompliance extends Component {
this.searchTerm || set(this, 'searchTerm', '');
this.schemaFieldNamesMappedToDataTypes || (this.schemaFieldNamesMappedToDataTypes = []);
this.complianceDataTypes || (this.complianceDataTypes = []);
typeOf(this.suggestionConfidenceThreshold) === 'number' ||
set(this, 'suggestionConfidenceThreshold', lowQualitySuggestionConfidenceThreshold);
}
/**
@ -739,16 +748,26 @@ export default class DatasetCompliance extends Component {
'columnIdFieldsToCurrentPrivacyPolicy',
'complianceDataTypes',
'identifierFieldToSuggestion',
'suggestionConfidenceThreshold',
function(this: DatasetCompliance): Array<IComplianceChangeSet> {
// schemaFieldNamesMappedToDataTypes is a dependency for CP columnIdFieldsToCurrentPrivacyPolicy, so no need to dep on that directly
const changeSet = mergeComplianceEntitiesWithSuggestions(
get(this, 'columnIdFieldsToCurrentPrivacyPolicy'),
get(this, 'identifierFieldToSuggestion')
);
const suggestionThreshold = get(this, 'suggestionConfidenceThreshold');
// pass current changeSet state to parent handlers
run(() => next(this, 'notifyHandlerOfSuggestions', changeSet));
run(() => next(this, 'notifyHandlerOfFieldsRequiringReview', get(this, 'complianceDataTypes'), changeSet));
run(() => next(this, 'notifyHandlerOfSuggestions', suggestionThreshold, changeSet));
run(() =>
next(
this,
'notifyHandlerOfFieldsRequiringReview',
suggestionThreshold,
get(this, 'complianceDataTypes'),
changeSet
)
);
return changeSet;
}
@ -764,14 +783,16 @@ export default class DatasetCompliance extends Component {
'fieldReviewOption',
'compliancePolicyChangeSet',
'complianceDataTypes',
'suggestionConfidenceThreshold',
function(this: DatasetCompliance): Array<IComplianceChangeSet> {
const { compliancePolicyChangeSet: changeSet, complianceDataTypes } = getProperties(this, [
'compliancePolicyChangeSet',
'complianceDataTypes'
]);
const {
compliancePolicyChangeSet: changeSet,
complianceDataTypes,
suggestionConfidenceThreshold
} = getProperties(this, ['compliancePolicyChangeSet', 'complianceDataTypes', 'suggestionConfidenceThreshold']);
return get(this, 'fieldReviewOption') === 'showReview'
? tagsRequiringReview(complianceDataTypes)(changeSet)
? tagsRequiringReview(complianceDataTypes, { checkSuggestions: true, suggestionConfidenceThreshold })(changeSet)
: changeSet;
}
);
@ -788,9 +809,10 @@ export default class DatasetCompliance extends Component {
changeSetReviewWithoutSuggestionCheck = computed('changeSetReview', function(
this: DatasetCompliance
): Array<IComplianceChangeSet> {
return tagsRequiringReview(get(this, 'complianceDataTypes'), { checkSuggestions: false })(
get(this, 'changeSetReview')
);
return tagsRequiringReview(get(this, 'complianceDataTypes'), {
checkSuggestions: false,
suggestionConfidenceThreshold: 0 // irrelevant value set to 0 since checkSuggestions flag is false above
})(get(this, 'changeSetReview'));
});
/**
@ -801,8 +823,17 @@ export default class DatasetCompliance extends Component {
changeSetReview = computed(
`compliancePolicyChangeSet.@each.{${changeSetReviewableAttributeTriggers}}`,
'complianceDataTypes',
'suggestionConfidenceThreshold',
function(this: DatasetCompliance): Array<IComplianceChangeSet> {
return tagsRequiringReview(get(this, 'complianceDataTypes'))(get(this, 'compliancePolicyChangeSet'));
const { suggestionConfidenceThreshold, compliancePolicyChangeSet } = getProperties(this, [
'suggestionConfidenceThreshold',
'compliancePolicyChangeSet'
]);
return tagsRequiringReview(get(this, 'complianceDataTypes'), {
checkSuggestions: true,
suggestionConfidenceThreshold
})(compliancePolicyChangeSet);
}
);
@ -866,19 +897,25 @@ export default class DatasetCompliance extends Component {
/**
* Invokes external action with flag indicating that at least 1 suggestion exists for a field in the changeSet
* @param {number} suggestionConfidenceThreshold confidence threshold for filtering out higher quality suggestions
* @param {Array<IComplianceChangeSet>} changeSet
*/
notifyHandlerOfSuggestions = (changeSet: Array<IComplianceChangeSet>): void => {
const hasChangeSetSuggestions = !!compact(getTagsSuggestions(changeSet)).length;
notifyHandlerOfSuggestions = (
suggestionConfidenceThreshold: number,
changeSet: Array<IComplianceChangeSet>
): void => {
const hasChangeSetSuggestions = !!compact(getTagsSuggestions({ suggestionConfidenceThreshold })(changeSet)).length;
this.notifyOnChangeSetSuggestions(hasChangeSetSuggestions);
};
/**
* Invokes external action with flag indicating that a field in the tags requires user review
* @param {number} suggestionConfidenceThreshold confidence threshold for filtering out higher quality suggestions
* @param {Array<IComplianceDataType>} complianceDataTypes
* @param {Array<IComplianceChangeSet>} tags
*/
notifyHandlerOfFieldsRequiringReview = (
suggestionConfidenceThreshold: number,
complianceDataTypes: Array<IComplianceDataType>,
tags: Array<IComplianceChangeSet>
): void => {
@ -886,7 +923,10 @@ export default class DatasetCompliance extends Component {
assert('expected complianceDataTypes to be of type `array`', Array.isArray(complianceDataTypes));
assert('expected tags to be of type `array`', Array.isArray(tags));
const hasChangeSetDrift = !!tagsRequiringReview(complianceDataTypes)(tags).length;
const hasChangeSetDrift = !!tagsRequiringReview(complianceDataTypes, {
checkSuggestions: true,
suggestionConfidenceThreshold
})(tags).length;
this.notifyOnChangeSetRequiresReview(hasChangeSetDrift);
};

23
wherehows-web/app/components/datasets/containers/dataset-compliance.ts
View File

@ -21,12 +21,20 @@ import {
import { columnDataTypesAndFieldNames } from 'wherehows-web/utils/api/datasets/columns';
import { readDatasetSchemaByUrn } from 'wherehows-web/utils/api/datasets/schema';
import { readComplianceDataTypes } from 'wherehows-web/utils/api/list/compliance-datatypes';
import { compliancePolicyStrings, removeReadonlyAttr, editableTags, SuggestionIntent } from 'wherehows-web/constants';
import {
compliancePolicyStrings,
removeReadonlyAttr,
editableTags,
SuggestionIntent,
lowQualitySuggestionConfidenceThreshold
} from 'wherehows-web/constants';
import { iterateArrayAsync } from 'wherehows-web/utils/array';
import validateMetadataObject, {
complianceEntitiesTaxonomy
} from 'wherehows-web/utils/datasets/compliance/metadata-schema';
import { notificationDialogActionFactory } from 'wherehows-web/utils/notifications/notifications';
import Configurator from 'wherehows-web/services/configurator';
import { typeOf } from '@ember/utils';
/**
* Type alias for the response when container data items are batched
@ -49,6 +57,7 @@ type BatchContainerDataResult = Pick<
| 'complianceSuggestion'
| 'schemaFieldNamesMappedToDataTypes'
| 'schemaless'
| 'suggestionConfidenceThreshold'
>;
const { successUpdating, failedUpdating, successUploading, invalidPolicyData } = compliancePolicyStrings;
@ -128,6 +137,13 @@ export default class DatasetComplianceContainer extends Component {
*/
datasetName: string = '';
/**
* Confidence percentage number used to filter high quality suggestions versus lower quality
* @type {number}
* @memberof DatasetComplianceContainer
*/
suggestionConfidenceThreshold: number = lowQualitySuggestionConfidenceThreshold;
/**
* The urn identifier for the dataset
* @type {string}
@ -177,11 +193,16 @@ export default class DatasetComplianceContainer extends Component {
]);
const schemaFieldNamesMappedToDataTypes = await iterateArrayAsync(columnDataTypesAndFieldNames)(columns);
const { containingPersonalData, fromUpstream } = complianceInfo;
let suggestionConfidenceThreshold = Configurator.getConfig('suggestionConfidenceThreshold');
// convert to fractional percentage if valid number is present
typeOf(suggestionConfidenceThreshold) === 'number' &&
(suggestionConfidenceThreshold = suggestionConfidenceThreshold / 100);
this.notifyPiiStatus(!!containingPersonalData);
this.onCompliancePolicyStateChange.call(this, { isNewComplianceInfo, fromUpstream: !!fromUpstream });
return setProperties(this, {
suggestionConfidenceThreshold,
isNewComplianceInfo,
complianceInfo,
complianceDataTypes,

32
wherehows-web/app/components/datasets/containers/dataset-health.ts Normal file
View File

@ -0,0 +1,32 @@
import Component from '@ember/component';
import { get } from '@ember/object';
import { task, TaskInstance } from 'ember-concurrency';
/**
* This is the container component for the dataset health tab. It should contain the health bar graphs and a table
* depicting the detailed health scores. Aside from fetching the data, it also handles click interactions between
* the graphs and the table in terms of filtering and displaying of data
*/
export default class DatasetHealthContainer extends Component {
/**
* The urn identifier for the dataset
* @type {string}
*/
urn: string;
didInsertElement() {
get(this, 'getContainerDataTask').perform();
}
didUpdateAttrs() {
get(this, 'getContainerDataTask').perform();
}
/**
* An async parent task to group all data tasks for this container component
* @type {Task<TaskInstance<Promise<any>>, (a?: any) => TaskInstance<TaskInstance<Promise<any>>>>}
*/
getContainerDataTask = task(function*(this: DatasetHealthContainer): IterableIterator<TaskInstance<Promise<any>>> {
// Do something in the future
});
}

34
wherehows-web/app/constants/dataset-compliance.ts
View File

@ -40,11 +40,6 @@ const compliancePolicyStrings = {
failedUpdating: 'An error occurred while saving.',
successUploading: 'Metadata successfully updated! Please confirm and complete subsequent metadata information.',
invalidPolicyData: 'Received policy in an unexpected format! Please check the provided attributes and try again.',
helpText: {
classification:
'This security classification is from go/dht and should be good enough in most cases. ' +
'You can optionally override it if required by house security.'
},
missingPurgePolicy: 'Please specify a Compliance Purge Policy',
missingDatasetSecurityClassification: 'Please specify a security classification for this dataset.'
};
@ -183,10 +178,18 @@ const suggestedIdentifierTypesInList = (suggestion: ISuggestedFieldTypeValues |
* @param {SchemaFieldToSuggestedValue} suggestion
* @param {SuggestionIntent} suggestionAuthority
* @param {ComplianceFieldIdValue | NonIdLogicalType | null} identifierType
* @param {number} suggestionConfidenceThreshold confidence threshold for filtering out higher quality suggestions
* @return {boolean}
*/
const tagSuggestionNeedsReview = ({ suggestion, suggestionAuthority, identifierType }: IComplianceChangeSet): boolean =>
suggestion && suggestion.identifierType !== identifierType && isHighConfidenceSuggestion(suggestion)
const tagSuggestionNeedsReview = ({
suggestion,
suggestionAuthority,
identifierType,
suggestionConfidenceThreshold
}: IComplianceChangeSet & { suggestionConfidenceThreshold: number }): boolean =>
suggestion &&
suggestion.identifierType !== identifierType &&
isHighConfidenceSuggestion(suggestion, suggestionConfidenceThreshold)
? !suggestionAuthority
: false;
@ -223,14 +226,14 @@ const tagValuePatternNeedsReview = ({ valuePattern }: IComplianceChangeSet): boo
* checking steps
* @return {(tag: IComplianceChangeSet) => boolean}
*/
const tagNeedsReview = (complianceDataTypes: Array<IComplianceDataType>, options?: IComplianceTagReviewOptions) =>
const tagNeedsReview = (complianceDataTypes: Array<IComplianceDataType>, options: IComplianceTagReviewOptions) =>
/**
* Checks if a compliance tag needs to be reviewed against a set of rules
* @param {IComplianceChangeSet} tag
* @return {boolean}
*/
(tag: IComplianceChangeSet): boolean => {
const { checkSuggestions } = options || { checkSuggestions: true };
const { checkSuggestions, suggestionConfidenceThreshold } = options;
const { isDirty, privacyPolicyExists, identifierType, logicalType } = tag;
let isReviewRequired = false;
@ -241,7 +244,7 @@ const tagNeedsReview = (complianceDataTypes: Array<IComplianceDataType>, options
// Check that a hi confidence suggestion exists and the identifierType does not match the change set item
if (checkSuggestions) {
isReviewRequired = isReviewRequired || tagSuggestionNeedsReview(tag);
isReviewRequired = isReviewRequired || tagSuggestionNeedsReview({ ...tag, suggestionConfidenceThreshold });
}
// Ensure that tag has a logical type and nonOwner flag is set when tag is of id type
@ -391,17 +394,20 @@ const singleTagsInChangeSet = (
* @param {IComplianceTagReviewOptions} options
* @return {(array: Array<IComplianceChangeSet>) => Array<IComplianceChangeSet>}
*/
const tagsRequiringReview = (complianceDataTypes: Array<IComplianceDataType>, options?: IComplianceTagReviewOptions) =>
const tagsRequiringReview = (complianceDataTypes: Array<IComplianceDataType>, options: IComplianceTagReviewOptions) =>
arrayFilter<IComplianceChangeSet>(tagNeedsReview(complianceDataTypes, options));
/**
* Lists the tags for a specific identifier field that need to be reviewed
* @param {Array<IComplianceDataType>} complianceDataTypes
* @param {IComplianceTagReviewOptions} options
* @return {(identifierField: string) => (tags: Array<IComplianceChangeSet>) => Array<IComplianceChangeSet>}
*/
const fieldTagsRequiringReview = (complianceDataTypes: Array<IComplianceDataType>) => (identifierField: string) => (
tags: Array<IComplianceChangeSet>
) => tagsRequiringReview(complianceDataTypes)(tagsForIdentifierField(identifierField)(tags));
const fieldTagsRequiringReview = (
complianceDataTypes: Array<IComplianceDataType>,
options: IComplianceTagReviewOptions
) => (identifierField: string) => (tags: Array<IComplianceChangeSet>) =>
tagsRequiringReview(complianceDataTypes, options)(tagsForIdentifierField(identifierField)(tags));
/**
* Extracts a suggestion for a field from a suggestion map and merges a compliance entity with the suggestion

8
wherehows-web/app/controllers/datasets/dataset.ts
View File

@ -87,6 +87,14 @@ export default class DatasetController extends Controller {
*/
shouldShowDatasetLineage: boolean;
/**
* Flags the health feature for datasets, which is currently in the development stage so we should not
* have it appear in production
* @type {boolean}
* @memberof DatasetController
*/
shouldShowDatasetHealth: boolean;
/**
* Flag indicating if the dataset contains personally identifiable information
* @type {boolean}

3
wherehows-web/app/routes/datasets/dataset.ts
View File

@ -95,7 +95,8 @@ export default class DatasetRoute extends Route {
setProperties(controller, {
isInternal: !!getConfig('isInternal'),
jitAclAccessWhitelist: getConfig('JitAclAccessWhitelist') || [],
shouldShowDatasetLineage: getConfig('shouldShowDatasetLineage')
shouldShowDatasetLineage: getConfig('shouldShowDatasetLineage'),
shouldShowDatasetHealth: getConfig('shouldShowDatasetHealth')
});
}

1
wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs
View File

@ -25,6 +25,7 @@
platform=platform
complianceInfo=complianceInfo
complianceSuggestion=complianceSuggestion
suggestionConfidenceThreshold=suggestionConfidenceThreshold
isNewComplianceInfo=isNewComplianceInfo
schemaFieldNamesMappedToDataTypes=schemaFieldNamesMappedToDataTypes
complianceDataTypes=complianceDataTypes

1
wherehows-web/app/templates/components/datasets/containers/dataset-health.hbs Normal file
View File

@ -0,0 +1 @@
Coming Soon!

2
wherehows-web/app/templates/datasets/dataset-compliance/-dataset-compliance-entities.hbs
View File

@ -136,6 +136,7 @@
field=field
isNewComplianceInfo=isNewComplianceInfo
complianceDataTypes=complianceDataTypes
suggestionConfidenceThreshold=suggestionConfidenceThreshold
onFieldDblClick=(action "onFieldDblClick")
onFieldTagAdded=(action "onFieldTagAdded")
onFieldTagRemoved=(action "onFieldTagRemoved")
@ -216,6 +217,7 @@
sourceTag=tag
parentHasSingleTag=row.hasSingleTag
tagDidChange=(action "tagPropertiesUpdated")
suggestionConfidenceThreshold=suggestionConfidenceThreshold
complianceFieldIdDropdownOptions=complianceFieldIdDropdownOptions
complianceDataTypes=complianceDataTypes as |tagRowComponent|
}}

12
wherehows-web/app/templates/datasets/dataset.hbs
View File

@ -105,6 +105,12 @@
{{/tablist.tab}}
{{/if}}
{{#if shouldShowDatasetHealth}}
{{#tablist.tab tabIds.Health on-select=(action "tabSelectionChanged")}}
Health
{{/tablist.tab}}
{{/if}}
{{/tabs.tablist}}
</div>
</div>
@ -152,5 +158,11 @@
{{datasets/dataset-relationships urn=encodedUrn}}
{{/tabs.tabpanel}}
{{/if}}
{{#if shouldShowDatasetHealth}}
{{#tabs.tabpanel tabIds.Health}}
{{datasets/containers/dataset-health urn=encodedUrn}}
{{/tabs.tabpanel}}
{{/if}}
</div>
{{/ivy-tabs}}

3
wherehows-web/app/typings/api/configurator/configurator.d.ts vendored
View File

@ -9,6 +9,9 @@ interface IAppConfig {
isInternal: boolean | void;
JitAclAccessWhitelist: Array<DatasetPlatform> | void;
shouldShowDatasetLineage: boolean;
shouldShowDatasetHealth: boolean;
// confidence threshold for filtering out higher quality suggestions
suggestionConfidenceThreshold: number;
tracking: {
isEnabled: boolean;
trackers: {

3
wherehows-web/app/typings/app/dataset-compliance.d.ts vendored
View File

@ -23,7 +23,10 @@ interface IDatasetComplianceActions {
* @interface IComplianceTagReviewOptions
*/
interface IComplianceTagReviewOptions {
// flag determines if suggested values are considered in tag(IComplianceChangeSet) review check
checkSuggestions: boolean;
// confidence threshold for filtering out higher quality suggestions
suggestionConfidenceThreshold: number;
}
/**

45
wherehows-web/app/utils/array.ts
View File

@ -1,4 +1,4 @@
import { identity } from 'wherehows-web/utils/helpers/functions';
import { identity, not } from 'wherehows-web/utils/helpers/functions';
/**
* Composable function that will in turn consume an item from a list an emit a result of equal or same type
@ -22,37 +22,53 @@ const take = <T>(n: number = 0) => (list: Array<T>): Array<T> => Array.prototype
/**
* Convenience utility takes a type-safe mapping function, and returns a list mapping function
* @param {(param: T) => U} mappingFunction maps a single type T to type U
* @param {(param: T) => U} predicate maps a single type T to type U
* @return {(array: Array<T>) => Array<U>}
*/
const arrayMap = <T, U>(mappingFunction: (param: T) => U): ((array: Array<T>) => Array<U>) => (array = []) =>
array.map(mappingFunction);
const arrayMap = <T, U>(predicate: (param: T) => U): ((array: Array<T>) => Array<U>) => (array = []) =>
array.map(predicate);
/**
* Partitions an array into a tuple containing elements that meet the predicate in the zeroth index,
* and excluded elements in the next
* `iterate-first data-last` function
* @template T type of source element list
* @template U subtype of T in first partition
* @param {(param: T) => param is U} predicate is a type guard function
* @returns {((array: Array<T>) => [Array<U>, Array<Exclude<T, U>>])}
*/
const arrayPartition = <T, U extends T>(
predicate: (param: T) => param is U
): ((array: Array<T>) => [Array<U>, Array<Exclude<T, U>>]) => (array = []) => [
array.filter(predicate),
array.filter<Exclude<T, U>>((v: T): v is Exclude<T, U> => not(predicate)(v))
];
/**
* Convenience utility takes a type-safe filter function, and returns a list filtering function
* @param {(param: T) => boolean} filtrationFunction
* @param {(param: T) => boolean} predicate
* @return {(array: Array<T>) => Array<T>}
*/
const arrayFilter = <T>(filtrationFunction: (param: T) => boolean): ((array: Array<T>) => Array<T>) => (array = []) =>
array.filter(filtrationFunction);
const arrayFilter = <T>(predicate: (param: T) => boolean): ((array: Array<T>) => Array<T>) => (array = []) =>
array.filter(predicate);
/**
* Type safe utility `iterate-first data-last` function for array every
* @template T
* @param {(param: T) => boolean} filter
* @param {(param: T) => boolean} predicate
* @returns {((array: Array<T>) => boolean)}
*/
const arrayEvery = <T>(filter: (param: T) => boolean): ((array: Array<T>) => boolean) => (array = []) =>
array.every(filter);
const arrayEvery = <T>(predicate: (param: T) => boolean): ((array: Array<T>) => boolean) => (array = []) =>
array.every(predicate);
/**
* Type safe utility `iterate-first data-last` function for array some
* @template T
* @param {(param: T) => boolean} filter
* @param {(param: T) => boolean} predicate
* @return {(array: Array<T>) => boolean}
*/
const arraySome = <T>(filter: (param: T) => boolean): ((array: Array<T>) => boolean) => (array = []) =>
array.some(filter);
const arraySome = <T>(predicate: (param: T) => boolean): ((array: Array<T>) => boolean) => (array = []) =>
array.some(predicate);
/**
* Composable reducer abstraction, curries a reducing iteratee and returns a reducing function that takes a list
@ -228,9 +244,10 @@ export { Many, Iteratee };
export {
take,
arrayMap,
arrayPipe,
arrayFilter,
arrayReduce,
arrayPipe,
arrayPartition,
isListUnique,
compact,
arrayEvery,

24
wherehows-web/app/utils/datasets/compliance-suggestions.ts
View File

@ -1,4 +1,3 @@
import { lowQualitySuggestionConfidenceThreshold } from 'wherehows-web/constants';
import { arrayMap } from 'wherehows-web/utils/array';
import { IComplianceChangeSet, ISuggestedFieldTypeValues } from 'wherehows-web/typings/app/dataset-compliance';
@ -6,23 +5,28 @@ import { IComplianceChangeSet, ISuggestedFieldTypeValues } from 'wherehows-web/t
* Takes a list of suggestions with confidence values, and if the confidence is greater than
* a low confidence threshold
* @param {number} confidenceLevel percentage indicating how confidence the system is in the suggested value
* @param {number} suggestionConfidenceThreshold threshold number to consider as a valid suggestion
* @return {boolean}
*/
const isHighConfidenceSuggestion = ({ confidenceLevel = 0 }: { confidenceLevel: number }): boolean =>
confidenceLevel > lowQualitySuggestionConfidenceThreshold;
const isHighConfidenceSuggestion = (
{ confidenceLevel = 0 }: { confidenceLevel: number },
suggestionConfidenceThreshold: number = 0
): boolean => confidenceLevel > suggestionConfidenceThreshold;
/**
* Extracts the tag suggestion from an IComplianceChangeSet tag.
* If a suggestionAuthority property exists on the tag, then the user has already either accepted or ignored
* the suggestion for this tag. It's value should not be taken into account on re-renders,
* in place, this substitutes an empty suggestion
* @param {IComplianceChangeSet} tag
* @return {{identifierType: IComplianceChangeSet.identifierType, logicalType: IComplianceChangeSet.logicalType, confidence: number} | void}
* @param {number} suggestionConfidenceThreshold confidence threshold for filtering out higher quality suggestions
* @return {(tag?: IComplianceChangeSet) => (ISuggestedFieldTypeValues | void)}
*/
const getTagSuggestions = (tag: IComplianceChangeSet = <IComplianceChangeSet>{}): ISuggestedFieldTypeValues | void => {
const getTagSuggestions = ({ suggestionConfidenceThreshold }: { suggestionConfidenceThreshold: number }) => (
tag: IComplianceChangeSet = <IComplianceChangeSet>{}
): ISuggestedFieldTypeValues | void => {
const { suggestion } = tag;
if (suggestion && isHighConfidenceSuggestion(suggestion)) {
if (suggestion && isHighConfidenceSuggestion(suggestion, suggestionConfidenceThreshold)) {
const { identifierType, logicalType, confidenceLevel: confidence } = suggestion;
return { identifierType, logicalType, confidence: +(confidence * 100).toFixed(2) };
}
@ -30,8 +34,10 @@ const getTagSuggestions = (tag: IComplianceChangeSet = <IComplianceChangeSet>{})
/**
* Gets the suggestions for a list of IComplianceChangeSet fields
* @type {(array: Array<IComplianceChangeSet>) => Array<{identifierType: ComplianceFieldIdValue | NonIdLogicalType | null; logicalType: IdLogicalType | null; confidence: number} | void>}
* @param {number} suggestionConfidenceThreshold
* @return {(array: Array<IComplianceChangeSet>) => Array<ISuggestedFieldTypeValues | void>}
*/
const getTagsSuggestions = arrayMap(getTagSuggestions);
const getTagsSuggestions = ({ suggestionConfidenceThreshold }: { suggestionConfidenceThreshold: number }) =>
arrayMap(getTagSuggestions({ suggestionConfidenceThreshold }));
export { isHighConfidenceSuggestion, getTagSuggestions, getTagsSuggestions };

4
wherehows-web/app/utils/object.ts
View File

@ -27,7 +27,9 @@ interface IPartialOrIdentityTypeFn<T> {
* @param {Array<K>} [droppedKeys=[]] the list of attributes on T to be dropped
* @returns {IPartialOrIdentityTypeFn<T>}
*/
const fleece = <T, K extends keyof T>(droppedKeys: Array<K> = []): IPartialOrIdentityTypeFn<T> => o => {
const fleece = <T, K extends keyof T = keyof T>(droppedKeys: Array<K> = []): IPartialOrIdentityTypeFn<T> => (
o: T
): Partial<T> | T => {
const partialResult = Object.assign({}, o);
return droppedKeys.reduce((partial, key) => {

9
wherehows-web/ember-cli-build.js
View File

@ -21,6 +21,15 @@ module.exports = function(defaults) {
includePolyfill: true
},
emberHighCharts: {
includedHighCharts: true,
// Note: Since we only need highcharts, excluding the other available modules in the addon
includeHighStock: false,
includeHighMaps: false,
includeHighChartsMore: false,
includeHighCharts3D: false
},
storeConfigInMeta: false,
SRI: {

2
wherehows-web/package.json
View File

@ -66,6 +66,7 @@
"ember-export-application-global": "^2.0.0",
"ember-fetch": "^3.4.4",
"ember-font-awesome": "^4.0.0-rc.2",
"ember-highcharts": "^1.0.0",
"ember-inflector": "^2.2.0",
"ember-load-initializers": "^1.0.0",
"ember-math-helpers": "^2.4.0",
@ -87,6 +88,7 @@
"eslint-plugin-prettier": "^2.5.0",
"eyeglass": "^1.3.0",
"eyeglass-restyle": "^1.1.0",
"highcharts": "^6.1.1",
"husky": "^0.14.3",
"ivy-tabs": "^3.1.0",
"lint-staged": "^7.1.0",

13
wherehows-web/tests/integration/components/datasets/containers/dataset-health-test.js Normal file
View File

@ -0,0 +1,13 @@
import { module, test } from 'qunit';
import { setupRenderingTest } from 'ember-qunit';
import { render } from '@ember/test-helpers';
import hbs from 'htmlbars-inline-precompile';
module('Integration | Component | datasets/containers/dataset-health', function(hooks) {
setupRenderingTest(hooks);
// TODO: More meaningful tests as we continue with development
test('it renders', async function(assert) {
await render(hbs`{{datasets/containers/dataset-health}}`);
assert.ok(this.element, 'Renders without errors');
});
});

18
wherehows-web/tests/unit/constants/dataset-compliance-test.js
View File

@ -8,7 +8,8 @@ import {
PurgePolicy,
initialComplianceObjectFactory,
isRecentSuggestion,
tagNeedsReview
tagNeedsReview,
lowQualitySuggestionConfidenceThreshold
} from 'wherehows-web/constants';
import complianceDataTypes from 'wherehows-web/mirage/fixtures/compliance-data-types';
import { mockTimeStamps } from 'wherehows-web/tests/helpers/datasets/compliance-policy/recent-suggestions-constants';
@ -17,6 +18,11 @@ import { hdfsUrn } from 'wherehows-web/mirage/fixtures/urn';
module('Unit | Constants | dataset compliance');
const complianceTagReviewOptions = {
checkSuggestions: false,
suggestionConfidenceThreshold: lowQualitySuggestionConfidenceThreshold
};
test('initialComplianceObjectFactory', function(assert) {
assert.expect(2);
const mockUrn = hdfsUrn;
@ -55,14 +61,20 @@ test('isRecentSuggestion correctly determines if a suggestion is recent or not',
test('tagNeedsReview exists', function(assert) {
assert.ok(typeof tagNeedsReview === 'function', 'tagNeedsReview is a function');
assert.ok(typeof tagNeedsReview([])({}) === 'boolean', 'tagNeedsReview returns a boolean');
assert.ok(
typeof tagNeedsReview([], complianceTagReviewOptions)({}) === 'boolean',
'tagNeedsReview returns a boolean'
);
});
test('tagNeedsReview correctly determines if a fieldChangeSet requires review', function(assert) {
assert.expect(mockFieldChangeSets.length);
mockFieldChangeSets.forEach(changeSet =>
assert.ok(tagNeedsReview(complianceDataTypes)(changeSet) === changeSet.__requiresReview__, changeSet.__msg__)
assert.ok(
tagNeedsReview(complianceDataTypes, complianceTagReviewOptions)(changeSet) === changeSet.__requiresReview__,
changeSet.__msg__
)
);
});

23
wherehows-web/tests/unit/utils/datasets/compliance-suggestions-test.js
View File

@ -8,21 +8,30 @@ test('isHighConfidenceSuggestion correctly determines the confidence of a sugges
let result = isHighConfidenceSuggestion({});
assert.notOk(result, 'should be false if no arguments are supplied');
result = isHighConfidenceSuggestion({ confidenceLevel: lowQualitySuggestionConfidenceThreshold + 1 });
result = isHighConfidenceSuggestion(
{ confidenceLevel: lowQualitySuggestionConfidenceThreshold + 1 },
lowQualitySuggestionConfidenceThreshold
);
assert.ok(
result,
`should be true if the confidence value is greater than ${lowQualitySuggestionConfidenceThreshold}`
);
result = isHighConfidenceSuggestion({ confidenceLevel: lowQualitySuggestionConfidenceThreshold - 1 });
result = isHighConfidenceSuggestion(
{ confidenceLevel: lowQualitySuggestionConfidenceThreshold - 1 },
lowQualitySuggestionConfidenceThreshold
);
assert.notOk(
result,
`should be false if the confidence value is less than ${lowQualitySuggestionConfidenceThreshold}`
);
result = isHighConfidenceSuggestion({ confidenceLevel: lowQualitySuggestionConfidenceThreshold });
result = isHighConfidenceSuggestion(
{ confidenceLevel: lowQualitySuggestionConfidenceThreshold },
lowQualitySuggestionConfidenceThreshold
);
assert.notOk(
result,
@ -42,15 +51,17 @@ test('getTagSuggestions correctly extracts suggestions from a compliance field',
suggestionAuthority: SuggestionIntent.accept
};
let result = getTagSuggestions({});
let result = getTagSuggestions({ suggestionConfidenceThreshold: lowQualitySuggestionConfidenceThreshold })({});
assert.ok(typeof result === 'undefined', 'expected undefined return when the argument is an empty object');
result = getTagSuggestions();
result = getTagSuggestions({ suggestionConfidenceThreshold: lowQualitySuggestionConfidenceThreshold })();
assert.ok(typeof result === 'undefined', 'expected undefined return when no argument is supplied');
result = getTagSuggestions({ suggestion: changeSetField.suggestion });
result = getTagSuggestions({ suggestionConfidenceThreshold: lowQualitySuggestionConfidenceThreshold })({
suggestion: changeSetField.suggestion
});
assert.deepEqual(
result,

20
wherehows-web/yarn.lock
View File

@ -1382,6 +1382,10 @@ bootstrap-sass@^3.0.0:
version "3.3.7"
resolved "https://registry.yarnpkg.com/bootstrap-sass/-/bootstrap-sass-3.3.7.tgz#6596c7ab40f6637393323ab0bc80d064fc630498"
bootstrap@3.3.7:
version "3.3.7"
resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-3.3.7.tgz#5a389394549f23330875a3b150656574f8a9eb71"
bower-config@^1.3.0:
version "1.4.1"
resolved "https://registry.yarnpkg.com/bower-config/-/bower-config-1.4.1.tgz#85fd9df367c2b8dbbd0caa4c5f2bad40cd84c2cc"
@ -1742,7 +1746,7 @@ broccoli-lint-eslint@^4.2.1:
lodash.defaultsdeep "^4.6.0"
md5-hex "^2.0.0"
broccoli-merge-trees@^1.0.0, broccoli-merge-trees@^1.1.0, broccoli-merge-trees@^1.1.1, broccoli-merge-trees@^1.1.4:
broccoli-merge-trees@^1.0.0, broccoli-merge-trees@^1.1.0, broccoli-merge-trees@^1.1.1, broccoli-merge-trees@^1.1.4, broccoli-merge-trees@^1.2.0:
version "1.2.4"
resolved "https://registry.yarnpkg.com/broccoli-merge-trees/-/broccoli-merge-trees-1.2.4.tgz#a001519bb5067f06589d91afa2942445a2d0fdb5"
dependencies:
@ -3516,6 +3520,16 @@ ember-hash-helper-polyfill@^0.1.1:
ember-cli-babel "^5.1.7"
ember-cli-version-checker "^1.2.0"
ember-highcharts@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/ember-highcharts/-/ember-highcharts-1.0.0.tgz#d412af4d1f2f55e1cae0174c353852fd98b5bad9"
dependencies:
bootstrap "3.3.7"
broccoli-funnel "^2.0.1"
broccoli-merge-trees "^1.2.0"
ember-cli-babel "^6.6.0"
ember-cli-htmlbars "^2.0.1"
ember-ignore-children-helper@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/ember-ignore-children-helper/-/ember-ignore-children-helper-1.0.1.tgz#f7c4aa17afb9c5685e1d4dcdb61c7b138ca7cdc3"
@ -5170,6 +5184,10 @@ heimdalljs@^0.2.0, heimdalljs@^0.2.1, heimdalljs@^0.2.3, heimdalljs@^0.2.5:
dependencies:
rsvp "~3.2.1"
highcharts@^6.1.1:
version "6.1.1"
resolved "https://registry.yarnpkg.com/highcharts/-/highcharts-6.1.1.tgz#49dc34f5e963744ecd7eb87603b6cdaf8304c13a"
hoek@2.x.x:
version "2.16.3"
resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed"