Merge pull request #1390 from cptran777/handle-new-relationships-api

Handle new relationships api

wherehows-dao/src/main/java/wherehows/models/view/LineageView.java

@@ -31,3 +31,4 @@ public class LineageView {
 
   private String modified;
 }
+

wherehows-web/app/components/datasets/containers/dataset-lineage-downstreams.ts

@@ -3,7 +3,7 @@ import { get, set } from '@ember/object';
 import { task } from 'ember-concurrency';
 import { readDownstreamDatasetsByUrn } from 'wherehows-web/utils/api/datasets/lineage';
 import { assert } from '@ember/debug';
-import { Relationships } from 'wherehows-web/typings/api/datasets/relationships';
+import { LineageList } from 'wherehows-web/typings/api/datasets/relationships';
 
 export default class DatasetLineageDownstreamsContainer extends Component {
   /**
@@ -15,10 +15,10 @@ export default class DatasetLineageDownstreamsContainer extends Component {
 
   /**
    * List of downstreams datasets for this urn
-   * @type {Relationships}
+   * @type {LineageList}
    * @memberof DatasetLineageDownstreamsContainer
    */
-  downstreams: Relationships = [];
+  downstreams: LineageList = [];
 
   /**
    * Creates an instance of DatasetLineageDownstreamsContainer.
@@ -46,8 +46,8 @@ export default class DatasetLineageDownstreamsContainer extends Component {
    */
   getDatasetDownstreamsTask = task(function*(
     this: DatasetLineageDownstreamsContainer
-  ): IterableIterator<Promise<Relationships>> {
+  ): IterableIterator<Promise<LineageList>> {
-    let downstreams: Relationships = [];
+    let downstreams: LineageList = [];
 
     try {
       downstreams = yield readDownstreamDatasetsByUrn(get(this, 'urn'));

wherehows-web/app/components/datasets/containers/dataset-lineage-upstreams.ts

@@ -3,7 +3,7 @@ import { get, set } from '@ember/object';
 import { task } from 'ember-concurrency';
 import { readUpstreamDatasetsByUrn } from 'wherehows-web/utils/api/datasets/lineage';
 import { assert } from '@ember/debug';
-import { Relationships } from 'wherehows-web/typings/api/datasets/relationships';
+import { LineageList } from 'wherehows-web/typings/api/datasets/relationships';
 
 export default class DatasetLineageUpstreamsContainer extends Component {
   /**
@@ -15,10 +15,10 @@ export default class DatasetLineageUpstreamsContainer extends Component {
 
   /**
    * List of upstream datasets for this urn
-   * @type {Relationships}
+   * @type {LineageList}
    * @memberof DatasetLineageUpstreamsContainer
    */
-  upstreams: Relationships = [];
+  upstreams: LineageList = [];
 
   /**
    * Creates an instance of DatasetLineageUpstreamsContainer.
@@ -46,8 +46,8 @@ export default class DatasetLineageUpstreamsContainer extends Component {
    */
   getDatasetUpstreamsTask = task(function*(
     this: DatasetLineageUpstreamsContainer
-  ): IterableIterator<Promise<Relationships>> {
+  ): IterableIterator<Promise<LineageList>> {
-    let upstreams: Relationships = [];
+    let upstreams: LineageList = [];
 
     try {
       upstreams = yield readUpstreamDatasetsByUrn(get(this, 'urn'));

wherehows-web/app/components/datasets/containers/upstream-dataset.ts

@@ -17,13 +17,14 @@ import { retentionObjectFactory } from 'wherehows-web/constants/datasets/retenti
 import Notifications, { NotificationEvent } from 'wherehows-web/services/notifications';
 import { IComplianceInfo } from 'wherehows-web/typings/api/datasets/compliance';
 import { service } from '@ember-decorators/service';
+import { LineageList } from 'wherehows-web/typings/api/datasets/relationships';
 
 /**
  * Aliases the yieldable values for the container task
  * @alias {IterableIterator<TaskInstance<TaskInstance<Promise<IDatasetView[]>> | Promise<IUpstreamWithComplianceMetadata[]>> | TaskInstance<Promise<IDataPlatform[]>> | TaskInstance<Promise<IGetDatasetRetentionResponse | null>>>}
  */
 type ContainerYieldableResult = IterableIterator<
-  | TaskInstance<TaskInstance<Promise<IDatasetView[]>> | Promise<IUpstreamWithComplianceMetadata[]>>
+  | TaskInstance<TaskInstance<Promise<LineageList>> | Promise<IUpstreamWithComplianceMetadata[]>>
   | TaskInstance<Promise<IDataPlatform[]>>
   | TaskInstance<Promise<IGetDatasetRetentionResponse | null>>
 >;
@@ -68,9 +69,9 @@ export default class UpstreamDatasetContainer extends Component {
 
   /**
    * The list of upstream datasets for the related urn
-   * @type {Array<IDatasetView>}
+   * @type {LineageList}
    */
-  upstreamDatasets: Array<IDatasetView> = [];
+  upstreamDatasets: LineageList = [];
 
   /**
    * List of metadata properties for upstream datasets
@@ -132,10 +133,8 @@ export default class UpstreamDatasetContainer extends Component {
    * @type {Task<Promise<Array<IDatasetView>>>, (a?: {} | undefined) => TaskInstance<Promise<Array<IDatasetView>>>>}
    * @memberof UpstreamDatasetContainer
    */
-  getUpstreamDatasetsTask = task(function*(
+  getUpstreamDatasetsTask = task(function*(this: UpstreamDatasetContainer): IterableIterator<Promise<LineageList>> {
-    this: UpstreamDatasetContainer
+    const upstreamDatasets: LineageList = yield readUpstreamDatasetsByUrn(get(this, 'urn'));
-  ): IterableIterator<Promise<Array<IDatasetView>>> {
-    const upstreamDatasets: Array<IDatasetView> = yield readUpstreamDatasetsByUrn(get(this, 'urn'));
     return set(this, 'upstreamDatasets', upstreamDatasets);
   });
 
@@ -146,9 +145,9 @@ export default class UpstreamDatasetContainer extends Component {
    */
   getUpstreamMetadataTask = task(function*(
     this: UpstreamDatasetContainer
-  ): IterableIterator<TaskInstance<Promise<Array<IDatasetView>>> | Promise<Array<IUpstreamWithComplianceMetadata>>> {
+  ): IterableIterator<TaskInstance<Promise<LineageList>> | Promise<Array<IUpstreamWithComplianceMetadata>>> {
-    const upstreamDatasets: Array<IDatasetView> = yield get(this, 'getUpstreamDatasetsTask').perform();
+    const upstreamLineage: LineageList = yield get(this, 'getUpstreamDatasetsTask').perform();
-    const upstreamMetadataPromises = datasetsWithComplianceMetadata(upstreamDatasets);
+    const upstreamMetadataPromises = datasetsWithComplianceMetadata(upstreamLineage.map(lineage => lineage.dataset));
     const upstreamsMetadata: Array<IUpstreamWithComplianceMetadata> = yield Promise.all(upstreamMetadataPromises);
 
     set(this, 'upstreamsMetadata', upstreamsMetadata);

wherehows-web/app/components/datasets/dataset-relationship-table.ts

@@ -1,15 +1,14 @@
 import Component from '@ember/component';
 import { action } from '@ember-decorators/object';
-import { Relationships, RelationshipType } from 'wherehows-web/typings/api/datasets/relationships';
+import { RelationshipType, IDatasetLineage, LineageList } from 'wherehows-web/typings/api/datasets/relationships';
 import { computed, get, getProperties, set } from '@ember/object';
-import { IDatasetView } from 'wherehows-web/typings/api/datasets/dataset';
 import { arrayMap, arrayPipe, arrayReduce } from 'wherehows-web/utils/array';
 import ComputedProperty from '@ember/object/computed';
 import {
   allRelationshipType,
   dedupeType,
-  filterRelationshipsByType,
+  takeNLineageItems,
-  takeNRelationships
+  filterLineageByType
 } from 'wherehows-web/utils/datasets/lineage';
 
 export default class DatasetRelationshipTable extends Component {
@@ -22,10 +21,10 @@ export default class DatasetRelationshipTable extends Component {
 
   /**
    * List of  dataset relationships
-   * @type {Relationships}
+   * @type {IDatasetLineage}
    * @memberof DatasetRelationshipTable
    */
-  relationships: Relationships;
+  relationships: LineageList;
 
   /**
    * References the currently selected relationship type, used to filter out relationships
@@ -47,18 +46,18 @@ export default class DatasetRelationshipTable extends Component {
 
     // set default values for required props
     this.selectedRelationshipType || set(this, 'selectedRelationshipType', allRelationshipType);
-    Array.isArray(this.relationships) || set(this, 'relationships', []);
+    Array.isArray(this.relationships) || set(this, 'relationships', <LineageList>[]);
   }
 
-  filteredRelationshipsByType: ComputedProperty<Relationships> = computed(
+  filteredRelationshipsByType: ComputedProperty<LineageList> = computed(
     'selectedRelationshipType',
     'relationships.[]',
-    function(this: DatasetRelationshipTable): Relationships {
+    function(this: DatasetRelationshipTable): LineageList {
       const {
         selectedRelationshipType: { value },
         relationships
       } = getProperties(this, ['selectedRelationshipType', 'relationships']);
-      return filterRelationshipsByType(value)(relationships);
+      return filterLineageByType(value)(relationships);
     }
   );
 
@@ -68,17 +67,17 @@ export default class DatasetRelationshipTable extends Component {
    * @type {ComputedProperty<Relationships>}
    * @memberof DatasetRelationshipTable
    */
-  filteredRelationships: ComputedProperty<Relationships> = computed(
+  filteredRelationships: ComputedProperty<LineageList> = computed(
     'showAllRelationships',
     'filteredRelationshipsByType',
-    function(this: DatasetRelationshipTable): Relationships {
+    function(this: DatasetRelationshipTable): LineageList {
       const { filteredRelationshipsByType, showAllRelationships, truncatedLength: n } = getProperties(this, [
         'filteredRelationshipsByType',
         'showAllRelationships',
         'truncatedLength'
       ]);
 
-      return takeNRelationships(showAllRelationships, n)(filteredRelationshipsByType);
+      return takeNLineageItems(showAllRelationships, n)(filteredRelationshipsByType);
     }
   );
 
@@ -120,9 +119,9 @@ export default class DatasetRelationshipTable extends Component {
     this: DatasetRelationshipTable
   ): Array<RelationshipType> {
     const relationships = get(this, 'relationships');
-    const typeOption = ({ nativeType }: IDatasetView): RelationshipType => ({
+    const typeOption = ({ type }: IDatasetLineage): RelationshipType => ({
-      label: nativeType,
+      label: type,
-      value: nativeType
+      value: type
     });
 
     return [allRelationshipType, ...arrayPipe(arrayMap(typeOption), arrayReduce(dedupeType, []))(relationships)];

wherehows-web/app/templates/components/datasets/dataset-relationship-table.hbs

@@ -16,23 +16,23 @@
   {{/relationTable.head}}
 
   {{#relationTable.body as |body|}}
-    {{#each relationTable.data as |dataset|}}
+    {{#each relationTable.data as |lineage|}}
-      {{#body.row field=dataset as |row|}}
+      {{#body.row field=lineage as |row|}}
         {{#row.cell}}
-          {{#link-to 'datasets.dataset' 'urn' (query-params urn=dataset.uri)}}
+          {{#link-to 'datasets.dataset' 'urn' (query-params urn=lineage.dataset.uri)}}
             <strong>
-              {{dataset.nativeName}}
+              {{lineage.dataset.nativeName}}
             </strong>
           {{/link-to}}
         {{/row.cell}}
         {{#row.cell}}
-          {{titleize dataset.nativeType}}
+          {{titleize lineage.type}}
         {{/row.cell}}
         {{#row.cell}}
-          \{{Actor URN}}
+          {{lineage.actor}}
         {{/row.cell}}
         {{#row.cell}}
-          {{moment-calendar dataset.modifiedTime sameElse="MMM Do YYYY, h:mm a"}}
+          {{moment-calendar lineage.dataset.modifiedTime sameElse="MMM Do YYYY, h:mm a"}}
         {{/row.cell}}
       {{/body.row}}
     {{/each}}

wherehows-web/app/typings/api/datasets/relationships.d.ts

@@ -1,13 +1,6 @@
 import { IDatasetView } from 'wherehows-web/typings/api/datasets/dataset';
 import { IDropDownOption } from 'wherehows-web/typings/app/dataset-compliance';
 
-/**
- * Alias for a list of IDatasetView instances
- * @alias
- * @type Relationships
- */
-type Relationships = Array<IDatasetView>;
-
 /**
  * Alias for a drop-down option based on an IDatasetView nativeType
  * @alias
@@ -23,7 +16,8 @@ interface IDatasetLineage {
   dataset: IDatasetView;
   type: string;
   actor: string;
-  modified: string | undefined;
 }
 
-export { Relationships, RelationshipType, IDatasetLineage };
+type LineageList = Array<IDatasetLineage>;
+
+export { RelationshipType, IDatasetLineage, LineageList };

wherehows-web/app/utils/api/datasets/lineage.ts

@@ -1,7 +1,7 @@
 import { getJSON, returnDefaultIfNotFound } from 'wherehows-web/utils/api/fetcher';
 import { datasetUrlByUrn } from 'wherehows-web/utils/api/datasets/shared';
-import { IDatasetView } from 'wherehows-web/typings/api/datasets/dataset';
 import { encodeUrn } from 'wherehows-web/utils/validators/urn';
+import { LineageList } from 'wherehows-web/typings/api/datasets/relationships';
 
 /**
  * Constructs the url for a datasets upstreams
@@ -22,15 +22,15 @@ const datasetDownstreamUrlByUrn = (urn: string): string => `${datasetUrlByUrn(ur
  * @param {string} urn urn for the child dataset
  * @return {Promise<Array<IDatasetView>>}
  */
-const readUpstreamDatasetsByUrn = (urn: string): Promise<Array<IDatasetView>> =>
+const readUpstreamDatasetsByUrn = (urn: string): Promise<LineageList> =>
-  returnDefaultIfNotFound(getJSON<Array<IDatasetView>>({ url: datasetUpstreamUrlByUrn(encodeUrn(urn)) }), []);
+  returnDefaultIfNotFound(getJSON<LineageList>({ url: datasetUpstreamUrlByUrn(encodeUrn(urn)) }), []);
 
 /**
  * Requests the downstream datasets for the dataset identified by urn
  * @param {string} urn string urn for the dataset
  * @return {Promise<Array<IDatasetView>>}
  */
-const readDownstreamDatasetsByUrn = (urn: string): Promise<Array<IDatasetView>> =>
+const readDownstreamDatasetsByUrn = (urn: string): Promise<LineageList> =>
-  returnDefaultIfNotFound(getJSON<Array<IDatasetView>>({ url: datasetDownstreamUrlByUrn(encodeUrn(urn)) }), []);
+  returnDefaultIfNotFound(getJSON<LineageList>({ url: datasetDownstreamUrlByUrn(encodeUrn(urn)) }), []);
 
 export { readUpstreamDatasetsByUrn, readDownstreamDatasetsByUrn };

wherehows-web/app/utils/datasets/lineage.ts

@@ -1,5 +1,4 @@
-import { Relationships, RelationshipType } from 'wherehows-web/typings/api/datasets/relationships';
+import { RelationshipType, IDatasetLineage, LineageList } from 'wherehows-web/typings/api/datasets/relationships';
-import { IDatasetView } from 'wherehows-web/typings/api/datasets/dataset';
 import { arrayFilter, take } from 'wherehows-web/utils/array';
 
 /**
@@ -10,20 +9,19 @@ import { arrayFilter, take } from 'wherehows-web/utils/array';
 const allRelationshipType: RelationshipType = { label: 'All Types', value: '' };
 
 /**
- * Creates a filter function, the will filter an instance of an IDatasetView on it's nativeType
+ * Creates a filter function and will filter an instance of an IDatasetLineage based on its type property
- * property
+ * @param {IDatasetLineage.type} filter
- * @param {IDatasetView.nativeType} [filter='']
  */
-const nativeTypeFilter = (filter: IDatasetView['nativeType'] = '') => ({ nativeType }: IDatasetView): boolean =>
+const lineageTypeFilter = (filter: IDatasetLineage['type'] = '') => ({ type }: IDatasetLineage): boolean =>
-  filter ? nativeType === filter : true;
+  filter ? type === filter : true;
 
 /**
- * Filters a list of Relationships on the nativeType attribute
+ * Filters a list of dataset lineage objects on the type attribute
  * @param {string} filter
- * @return {(array: Array<IDatasetView>) => Array<IDatasetView>}
+ * @return {(array: LineageList) => LineageList}
  */
-const filterRelationshipsByType = (filter: string = ''): ((array: Relationships) => Relationships) =>
+const filterLineageByType = (filter: string = ''): ((array: LineageList) => LineageList) =>
-  arrayFilter(nativeTypeFilter(filter));
+  arrayFilter(lineageTypeFilter(filter));
 
 /**
  * Dedupes a list of RelationshipType objects
@@ -43,7 +41,7 @@ const dedupeType = (set: Array<RelationshipType>, relationshipType: Relationship
  * @param {boolean} shouldShowAll flag to determine if all relationships should be shown
  * @param {number} [n=10]
  */
-const takeNRelationships = (shouldShowAll: boolean, n: number = 10) => (relationships: Relationships): Relationships =>
+const takeNLineageItems = (shouldShowAll: boolean, n: number = 10) => (relationships: LineageList): LineageList =>
-  shouldShowAll ? relationships : take<IDatasetView>(n)(relationships);
+  shouldShowAll ? relationships : take<IDatasetLineage>(n)(relationships);
 
-export { allRelationshipType, filterRelationshipsByType, dedupeType, takeNRelationships };
+export { allRelationshipType, dedupeType, takeNLineageItems, filterLineageByType };

wherehows-web/mirage/helpers/dataset-downstreams.ts

@@ -1,7 +1,8 @@
-import { IFunctionRouteHandler } from 'wherehows-web/typings/ember-cli-mirage';
+const getDatasetDownstreams = ({ datasetViews }: { datasetViews: any }) =>
-
+  datasetViews.all().models.map((datasetView: any) => ({
-const getDatasetDownstreams = function(this: IFunctionRouteHandler, { datasetViews }: { datasetViews: any }) {
+    dataset: datasetView,
-  return this.serialize(datasetViews.all());
+    actor: 'corpuser:lskywalker',
-};
+    type: 'FAKE-TYPE'
+  }));
 
 export { getDatasetDownstreams };

wherehows-web/mirage/helpers/dataset-upstreams.ts

@@ -1,3 +1,8 @@
-const getDatasetUpstreams = ({ datasetViews }: { datasetViews: any }) => datasetViews.all();
+const getDatasetUpstreams = ({ datasetViews }: { datasetViews: any }) =>
+  datasetViews.all().models.map((datasetView: any) => ({
+    dataset: datasetView,
+    actor: 'corpuser:lskywalker',
+    type: 'FAKE-TYPE'
+  }));
 
 export { getDatasetUpstreams };

wherehows-web/tests/integration/components/dataset-authors-test.js

@@ -38,7 +38,6 @@ module('Integration | Component | dataset authors', function(hooks) {
     this.set('ownerTypes', ownerTypes);
     this.set('saveOwnerChanges', noop);
     await render(hbs`{{dataset-authors owners=owners ownerTypes=ownerTypes save=(action saveOwnerChanges)}}`);
-
     await click('.remove-dataset-author');
 
     assert.equal(this.get('owners').length, 0);

wherehows-web/tests/integration/components/datasets/containers/upstream-dataset-test.js

@@ -4,6 +4,7 @@ import { render, waitUntil, find, findAll, click } from '@ember/test-helpers';
 import hbs from 'htmlbars-inline-precompile';
 import { DatasetPlatform, PurgePolicy } from 'wherehows-web/constants';
 import { hdfsUrn } from 'wherehows-web/mirage/fixtures/urn';
+import defaultScenario from 'wherehows-web/mirage/scenarios/default';
 
 const upstreamElement = '.upstream-dataset';
 const downstreamPolicyEditButton = '#downstream-purge-edit';
@@ -15,7 +16,7 @@ module('Integration | Component | datasets/containers/upstream dataset', functio
   test('it renders', async function(assert) {
     assert.expect(1);
     const { server } = this;
-    const { nativeName, platform, uri } = server.create('datasetView');
+    const { nativeName, platform, uri } = server.create('dataset-view');
 
     this.set('urn', uri);
     this.set('platform', platform);

wherehows-web/tests/integration/components/datasets/upstream-dataset-test.js

@@ -6,7 +6,6 @@ import { hdfsUrn, nonHdfsUrn } from 'wherehows-web/mirage/fixtures/urn';
 
 module('Integration | Component | datasets/upstream dataset', function(hooks) {
   setupRenderingTest(hooks);
-
   test('it renders', async function(assert) {
     await render(hbs`{{datasets/upstream-dataset}}`);
 
@@ -15,7 +14,7 @@ module('Integration | Component | datasets/upstream dataset', function(hooks) {
 
   test('it renders upstream dataset properties', async function(assert) {
     const upstreamLink = '.upstream-dataset__upstream-link';
-    const upstreamsMetadata = [{}, {}];
+    const upstreamsMetadata = [{ dataset: {} }, { dataset: {} }];
     this.set('upstreamsMetadata', upstreamsMetadata);
     await render(hbs`{{datasets/upstream-dataset upstreamsMetadata=upstreamsMetadata}}`);