From dc0641d955fb453fa931c80be6bb8eb1b72c36ae Mon Sep 17 00:00:00 2001 From: cptran777 Date: Mon, 24 Sep 2018 16:28:14 -0700 Subject: [PATCH 1/3] use upstream lineage to determine show parents logic vs resolved compliance using fromUpstream --- .../datasets/containers/dataset-compliance.ts | 21 +++++++++++++++---- .../datasets/containers/upstream-dataset.ts | 12 ++++++----- .../containers/dataset-compliance.hbs | 5 ++++- .../components/datasets/upstream-dataset.hbs | 4 ++-- 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/wherehows-web/app/components/datasets/containers/dataset-compliance.ts b/wherehows-web/app/components/datasets/containers/dataset-compliance.ts index 98e70ce0b6..fdb285fbb0 100644 --- a/wherehows-web/app/components/datasets/containers/dataset-compliance.ts +++ b/wherehows-web/app/components/datasets/containers/dataset-compliance.ts @@ -44,6 +44,8 @@ import { containerDataSource } from 'wherehows-web/utils/components/containers/d import { saveDatasetRetentionByUrn } from 'wherehows-web/utils/api/datasets/retention'; import { extractRetentionFromComplianceInfo } from 'wherehows-web/utils/datasets/retention'; import { IDatasetRetention } from 'wherehows-web/typings/api/datasets/retention'; +import { readUpstreamDatasetsByUrn } from 'wherehows-web/utils/api/datasets/lineage'; +import { LineageList } from 'wherehows-web/typings/api/datasets/relationships'; /** * Type alias for the response when container data items are batched @@ -53,7 +55,8 @@ type BatchComplianceResponse = [ Array, IComplianceSuggestion, IDatasetSchema, - IDatasetExportPolicy | null + IDatasetExportPolicy | null, + LineageList ]; /** @@ -126,6 +129,13 @@ export default class DatasetComplianceContainer extends Component { */ complianceInfo: IComplianceInfo | void; + /** + * List object containing a list of lineage objects that include upstream dataests and lineage + * metainformation + * @type {LineageList} + */ + upstreams: LineageList = []; + /** * Object containing the fields for the export policy for this dataset * @type {IDatasetExportPolicy} @@ -190,13 +200,15 @@ export default class DatasetComplianceContainer extends Component { complianceDataTypes, complianceSuggestion, { columns, schemaless }, - exportPolicy + exportPolicy, + upstreams ]: BatchComplianceResponse = await Promise.all([ readDatasetComplianceByUrn(urn), readComplianceDataTypes(), readDatasetComplianceSuggestionByUrn(urn), readDatasetSchemaByUrn(urn), - readDatasetExportPolicyByUrn(urn) + readDatasetExportPolicyByUrn(urn), + readUpstreamDatasetsByUrn(urn) ]); const schemaFieldNamesMappedToDataTypes = await iterateArrayAsync(columnDataTypesAndFieldNames)(columns); const { containingPersonalData, fromUpstream } = complianceInfo; @@ -218,7 +230,8 @@ export default class DatasetComplianceContainer extends Component { complianceSuggestion, schemaFieldNamesMappedToDataTypes, schemaless, - exportPolicy + exportPolicy, + upstreams }); } diff --git a/wherehows-web/app/components/datasets/containers/upstream-dataset.ts b/wherehows-web/app/components/datasets/containers/upstream-dataset.ts index 844550e335..b1ad707ed7 100644 --- a/wherehows-web/app/components/datasets/containers/upstream-dataset.ts +++ b/wherehows-web/app/components/datasets/containers/upstream-dataset.ts @@ -3,7 +3,6 @@ import { task, TaskInstance } from 'ember-concurrency'; import { get, set, computed, setProperties, getProperties } from '@ember/object'; import { IDatasetView } from 'wherehows-web/typings/api/datasets/dataset'; import { assert } from '@ember/debug'; -import { readUpstreamDatasetsByUrn } from 'wherehows-web/utils/api/datasets/lineage'; import { IUpstreamWithComplianceMetadata } from 'wherehows-web/typings/app/datasets/lineage'; import { datasetsWithComplianceMetadata } from 'wherehows-web/constants/datasets/lineage'; import { arraySome } from 'wherehows-web/utils/array'; @@ -18,6 +17,7 @@ import Notifications, { NotificationEvent } from 'wherehows-web/services/notific import { IComplianceInfo } from 'wherehows-web/typings/api/datasets/compliance'; import { service } from '@ember-decorators/service'; import { LineageList } from 'wherehows-web/typings/api/datasets/relationships'; +import { readUpstreamDatasetsByUrn } from 'wherehows-web/utils/api/datasets/lineage'; /** * Aliases the yieldable values for the container task @@ -68,10 +68,10 @@ export default class UpstreamDatasetContainer extends Component { supportedPurgePolicies: Array = []; /** - * The list of upstream datasets for the related urn + * The list of upstream datasets for the related urn, passed in from parent container * @type {LineageList} */ - upstreamDatasets: LineageList = []; + upstreamLineage: LineageList; /** * List of metadata properties for upstream datasets @@ -135,7 +135,7 @@ export default class UpstreamDatasetContainer extends Component { */ getUpstreamDatasetsTask = task(function*(this: UpstreamDatasetContainer): IterableIterator> { const upstreamDatasets: LineageList = yield readUpstreamDatasetsByUrn(get(this, 'urn')); - return set(this, 'upstreamDatasets', upstreamDatasets); + return set(this, 'upstreamLineage', upstreamDatasets); }); /** @@ -146,7 +146,9 @@ export default class UpstreamDatasetContainer extends Component { getUpstreamMetadataTask = task(function*( this: UpstreamDatasetContainer ): IterableIterator> | Promise>> { - const upstreamLineage: LineageList = yield get(this, 'getUpstreamDatasetsTask').perform(); + // Fallback logic, if we have lineage then use that otherwise get upstreams again + const upstreamLineage = + get(this, 'upstreamLineage') || yield get(this, 'getUpstreamDatasetsTask').perform(); const upstreamMetadataPromises = datasetsWithComplianceMetadata(upstreamLineage.map(lineage => lineage.dataset)); const upstreamsMetadata: Array = yield Promise.all(upstreamMetadataPromises); diff --git a/wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs b/wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs index 5879aaccdd..57688d042e 100644 --- a/wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs +++ b/wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs @@ -10,11 +10,14 @@ {{else}} - {{#if complianceInfo.fromUpstream}} + {{!-- {{#if complianceInfo.fromUpstream}} --}} + {{#if upstreams.length}} {{datasets/containers/upstream-dataset urn=urn platform=platform + upstreamLineage=upstreams + hasResolvedCompliance=complianceInfo.fromUpstream upstreamComplianceType=complianceInfo.complianceType }} diff --git a/wherehows-web/app/templates/components/datasets/upstream-dataset.hbs b/wherehows-web/app/templates/components/datasets/upstream-dataset.hbs index 17d42e8842..4c1b87d58a 100644 --- a/wherehows-web/app/templates/components/datasets/upstream-dataset.hbs +++ b/wherehows-web/app/templates/components/datasets/upstream-dataset.hbs @@ -13,10 +13,10 @@
{{tooltip-on-element - text=(if parent.hasCompliance "Compliance completed for parent dataset" "Please complete the Compliance on this parent dataset") + text=(if hasResolvedCompliance "Compliance completed for parent dataset" "Please complete the Compliance on this parent dataset") }} - {{#if parent.hasCompliance}} + {{#if hasResolvedCompliance}} {{fa-icon "check" class="upstream-dataset__compliance-status__complete"}} {{else}} {{fa-icon "exclamation" class="upstream-dataset__compliance-status__incomplete"}} From 15fa9d6ca4333135885cfc23cd25e1453c1064e7 Mon Sep 17 00:00:00 2001 From: cptran777 Date: Mon, 24 Sep 2018 16:30:23 -0700 Subject: [PATCH 2/3] Minor cleanup for upstream lineage stuff --- .../app/components/datasets/containers/upstream-dataset.ts | 4 ++-- .../components/datasets/containers/dataset-compliance.hbs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/wherehows-web/app/components/datasets/containers/upstream-dataset.ts b/wherehows-web/app/components/datasets/containers/upstream-dataset.ts index b1ad707ed7..d25075c3cc 100644 --- a/wherehows-web/app/components/datasets/containers/upstream-dataset.ts +++ b/wherehows-web/app/components/datasets/containers/upstream-dataset.ts @@ -147,8 +147,8 @@ export default class UpstreamDatasetContainer extends Component { this: UpstreamDatasetContainer ): IterableIterator> | Promise>> { // Fallback logic, if we have lineage then use that otherwise get upstreams again - const upstreamLineage = - get(this, 'upstreamLineage') || yield get(this, 'getUpstreamDatasetsTask').perform(); + // prettier-ignore + const upstreamLineage = get(this, 'upstreamLineage') || (yield get(this, 'getUpstreamDatasetsTask').perform()); const upstreamMetadataPromises = datasetsWithComplianceMetadata(upstreamLineage.map(lineage => lineage.dataset)); const upstreamsMetadata: Array = yield Promise.all(upstreamMetadataPromises); diff --git a/wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs b/wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs index 57688d042e..5bcad4822e 100644 --- a/wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs +++ b/wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs @@ -10,7 +10,6 @@ {{else}} - {{!-- {{#if complianceInfo.fromUpstream}} --}} {{#if upstreams.length}} {{datasets/containers/upstream-dataset From 83511572176bf291211a0ae63b634931f35207f6 Mon Sep 17 00:00:00 2001 From: cptran777 Date: Tue, 25 Sep 2018 14:14:03 -0700 Subject: [PATCH 3/3] Minor addition to get upstreams task --- .../app/components/datasets/containers/upstream-dataset.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wherehows-web/app/components/datasets/containers/upstream-dataset.ts b/wherehows-web/app/components/datasets/containers/upstream-dataset.ts index d25075c3cc..9157da41da 100644 --- a/wherehows-web/app/components/datasets/containers/upstream-dataset.ts +++ b/wherehows-web/app/components/datasets/containers/upstream-dataset.ts @@ -136,7 +136,7 @@ export default class UpstreamDatasetContainer extends Component { getUpstreamDatasetsTask = task(function*(this: UpstreamDatasetContainer): IterableIterator> { const upstreamDatasets: LineageList = yield readUpstreamDatasetsByUrn(get(this, 'urn')); return set(this, 'upstreamLineage', upstreamDatasets); - }); + }).restartable(); /** * Task to get and set upstream metadata for upstream datasets