Merge pull request #1408 from cptran777/upstream-lineage

Use upstream lineage instead of resolved compliance

wherehows-web/app/components/datasets/containers/dataset-compliance.ts

@@ -44,6 +44,8 @@ import { containerDataSource } from 'wherehows-web/utils/components/containers/d
 import { saveDatasetRetentionByUrn } from 'wherehows-web/utils/api/datasets/retention';
 import { extractRetentionFromComplianceInfo } from 'wherehows-web/utils/datasets/retention';
 import { IDatasetRetention } from 'wherehows-web/typings/api/datasets/retention';
+import { readUpstreamDatasetsByUrn } from 'wherehows-web/utils/api/datasets/lineage';
+import { LineageList } from 'wherehows-web/typings/api/datasets/relationships';
 
 /**
  * Type alias for the response when container data items are batched
@@ -53,7 +55,8 @@ type BatchComplianceResponse = [
   Array<IComplianceDataType>,
   IComplianceSuggestion,
   IDatasetSchema,
-  IDatasetExportPolicy | null
+  IDatasetExportPolicy | null,
+  LineageList
 ];
 
 /**
@@ -126,6 +129,13 @@ export default class DatasetComplianceContainer extends Component {
    */
   complianceInfo: IComplianceInfo | void;
 
+  /**
+   * List object containing a list of lineage objects that include upstream dataests and lineage
+   * metainformation
+   * @type {LineageList}
+   */
+  upstreams: LineageList = [];
+
   /**
    * Object containing the fields for the export policy for this dataset
    * @type {IDatasetExportPolicy}
@@ -190,13 +200,15 @@ export default class DatasetComplianceContainer extends Component {
       complianceDataTypes,
       complianceSuggestion,
       { columns, schemaless },
-      exportPolicy
+      exportPolicy,
+      upstreams
     ]: BatchComplianceResponse = await Promise.all([
       readDatasetComplianceByUrn(urn),
       readComplianceDataTypes(),
       readDatasetComplianceSuggestionByUrn(urn),
       readDatasetSchemaByUrn(urn),
-      readDatasetExportPolicyByUrn(urn)
+      readDatasetExportPolicyByUrn(urn),
+      readUpstreamDatasetsByUrn(urn)
     ]);
     const schemaFieldNamesMappedToDataTypes = await iterateArrayAsync(columnDataTypesAndFieldNames)(columns);
     const { containingPersonalData, fromUpstream } = complianceInfo;
@@ -218,7 +230,8 @@ export default class DatasetComplianceContainer extends Component {
       complianceSuggestion,
       schemaFieldNamesMappedToDataTypes,
       schemaless,
-      exportPolicy
+      exportPolicy,
+      upstreams
     });
   }
 

wherehows-web/app/components/datasets/containers/upstream-dataset.ts

@@ -3,7 +3,6 @@ import { task, TaskInstance } from 'ember-concurrency';
 import { get, set, computed, setProperties, getProperties } from '@ember/object';
 import { IDatasetView } from 'wherehows-web/typings/api/datasets/dataset';
 import { assert } from '@ember/debug';
-import { readUpstreamDatasetsByUrn } from 'wherehows-web/utils/api/datasets/lineage';
 import { IUpstreamWithComplianceMetadata } from 'wherehows-web/typings/app/datasets/lineage';
 import { datasetsWithComplianceMetadata } from 'wherehows-web/constants/datasets/lineage';
 import { arraySome } from 'wherehows-web/utils/array';
@@ -18,6 +17,7 @@ import Notifications, { NotificationEvent } from 'wherehows-web/services/notific
 import { IComplianceInfo } from 'wherehows-web/typings/api/datasets/compliance';
 import { service } from '@ember-decorators/service';
 import { LineageList } from 'wherehows-web/typings/api/datasets/relationships';
+import { readUpstreamDatasetsByUrn } from 'wherehows-web/utils/api/datasets/lineage';
 
 /**
  * Aliases the yieldable values for the container task
@@ -68,10 +68,10 @@ export default class UpstreamDatasetContainer extends Component {
   supportedPurgePolicies: Array<PurgePolicy> = [];
 
   /**
-   * The list of upstream datasets for the related urn
+   * The list of upstream datasets for the related urn, passed in from parent container
    * @type {LineageList}
    */
-  upstreamDatasets: LineageList = [];
+  upstreamLineage: LineageList;
 
   /**
    * List of metadata properties for upstream datasets
@@ -135,8 +135,8 @@ export default class UpstreamDatasetContainer extends Component {
    */
   getUpstreamDatasetsTask = task(function*(this: UpstreamDatasetContainer): IterableIterator<Promise<LineageList>> {
     const upstreamDatasets: LineageList = yield readUpstreamDatasetsByUrn(get(this, 'urn'));
-    return set(this, 'upstreamDatasets', upstreamDatasets);
+    return set(this, 'upstreamLineage', upstreamDatasets);
-  });
+  }).restartable();
 
   /**
    * Task to get and set upstream metadata for upstream datasets
@@ -146,7 +146,9 @@ export default class UpstreamDatasetContainer extends Component {
   getUpstreamMetadataTask = task(function*(
     this: UpstreamDatasetContainer
   ): IterableIterator<TaskInstance<Promise<LineageList>> | Promise<Array<IUpstreamWithComplianceMetadata>>> {
-    const upstreamLineage: LineageList = yield get(this, 'getUpstreamDatasetsTask').perform();
+    // Fallback logic, if we have lineage then use that otherwise get upstreams again
+    // prettier-ignore
+    const upstreamLineage = get(this, 'upstreamLineage') || <LineageList>(yield get(this, 'getUpstreamDatasetsTask').perform());
     const upstreamMetadataPromises = datasetsWithComplianceMetadata(upstreamLineage.map(lineage => lineage.dataset));
     const upstreamsMetadata: Array<IUpstreamWithComplianceMetadata> = yield Promise.all(upstreamMetadataPromises);
 

wherehows-web/app/templates/components/datasets/containers/dataset-compliance.hbs

@@ -10,11 +10,13 @@
 
 {{else}}
 
-  {{#if complianceInfo.fromUpstream}}
+  {{#if upstreams.length}}
 
     {{datasets/containers/upstream-dataset
       urn=urn
       platform=platform
+      upstreamLineage=upstreams
+      hasResolvedCompliance=complianceInfo.fromUpstream
       upstreamComplianceType=complianceInfo.complianceType
     }}
 

wherehows-web/app/templates/components/datasets/upstream-dataset.hbs

@@ -13,10 +13,10 @@
   <div class="upstream-dataset">
     <span class="upstream-dataset__compliance-status">
       {{tooltip-on-element
-        text=(if parent.hasCompliance "Compliance completed for parent dataset" "Please complete the Compliance on this parent dataset")
+        text=(if hasResolvedCompliance "Compliance completed for parent dataset" "Please complete the Compliance on this parent dataset")
       }}
 
-      {{#if parent.hasCompliance}}
+      {{#if hasResolvedCompliance}}
         {{fa-icon "check" class="upstream-dataset__compliance-status__complete"}}
       {{else}}
         {{fa-icon "exclamation" class="upstream-dataset__compliance-status__incomplete"}}