UI : New Data profiler Layout. (#3184)

* UI : New Data profiler Layout.

* Add tests column in data profiler

* Add percentage graph

* Add data quality column

* Adding table level tests

* Minor change

* Change mock data for feed

* Addressing review comments
This commit is contained in:
Sachin Chaurasiya 2022-03-06 22:16:21 +05:30 committed by GitHub
parent a6e98b3fdb
commit 40b64d2d3e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 375 additions and 226 deletions

View File

@ -16,7 +16,7 @@ import { Table } from 'Models';
import { ColumnTestType } from '../enums/columnTest.enum';
import { CreateTableTest } from '../generated/api/tests/createTableTest';
import { TableTestType } from '../generated/tests/tableTest';
import { CreateColumnTest } from '../interface/dataQuality.interface';
import { ColumnTest } from '../interface/dataQuality.interface';
import { getURLWithQueryFields } from '../utils/APIUtils';
import APIClient from './index';
@ -138,7 +138,7 @@ export const deleteTableTestCase = (
);
};
export const addColumnTestCase = (tableId: string, data: CreateColumnTest) => {
export const addColumnTestCase = (tableId: string, data: ColumnTest) => {
const configOptions = {
headers: { 'Content-type': 'application/json' },
};

View File

@ -16,7 +16,7 @@ import { CreateTableTest } from '../../generated/api/tests/createTableTest';
import { Table } from '../../generated/entity/data/table';
import { TableTest } from '../../generated/tests/tableTest';
import {
CreateColumnTest,
ColumnTest,
TableTestDataType,
} from '../../interface/dataQuality.interface';
import ColumnTestForm from './Forms/ColumnTestForm';
@ -28,7 +28,7 @@ type Props = {
columnOptions: Table['columns'];
tableTestCase: TableTest[];
handleAddTableTestCase: (data: CreateTableTest) => void;
handleAddColumnTestCase: (data: CreateColumnTest) => void;
handleAddColumnTestCase: (data: ColumnTest) => void;
onFormCancel: () => void;
};
@ -53,7 +53,7 @@ const AddDataQualityTest = ({
) : (
<ColumnTestForm
column={columnOptions}
data={data as CreateColumnTest}
data={data as ColumnTest}
handleAddColumnTestCase={handleAddColumnTestCase}
onFormCancel={onFormCancel}
/>

View File

@ -19,7 +19,7 @@ import { ColumnTestType } from '../../../enums/columnTest.enum';
import { TestCaseExecutionFrequency } from '../../../generated/api/tests/createTableTest';
import { Table } from '../../../generated/entity/data/table';
import {
CreateColumnTest,
ColumnTest,
ModifiedTableColumn,
} from '../../../interface/dataQuality.interface';
import {
@ -33,9 +33,9 @@ import { Button } from '../../buttons/Button/Button';
import MarkdownWithPreview from '../../common/editor/MarkdownWithPreview';
type Props = {
data: CreateColumnTest;
data: ColumnTest;
column: ModifiedTableColumn[];
handleAddColumnTestCase: (data: CreateColumnTest) => void;
handleAddColumnTestCase: (data: ColumnTest) => void;
onFormCancel: () => void;
};
@ -118,7 +118,7 @@ const ColumnTestForm = ({
const selectedColumn = column.find((d) => d.name === name);
const existingTests =
selectedColumn?.columnTests?.map(
(d: CreateColumnTest) => d.testCase.columnTestType
(d: ColumnTest) => d.testCase.columnTestType
) || [];
if (existingTests.length) {
const newTest = Object.values(ColumnTestType).filter(
@ -222,7 +222,7 @@ const ColumnTestForm = ({
const handleSave = () => {
if (validateForm()) {
const columnTestObj: CreateColumnTest = {
const columnTestObj: ColumnTest = {
columnName: columnName,
description: markdownRef.current?.getEditorContent() || undefined,
executionFrequency: frequency,

View File

@ -17,7 +17,7 @@ import { CreateTableTest } from '../../generated/api/tests/createTableTest';
import { Table } from '../../generated/entity/data/table';
import { TableTest, TableTestType } from '../../generated/tests/tableTest';
import {
CreateColumnTest,
ColumnTest,
DatasetTestModeType,
ModifiedTableColumn,
TableTestDataType,
@ -27,7 +27,7 @@ import DataQualityTest from '../DataQualityTest/DataQualityTest';
type Props = {
handleAddTableTestCase: (data: CreateTableTest) => void;
handleAddColumnTestCase: (data: CreateColumnTest) => void;
handleAddColumnTestCase: (data: ColumnTest) => void;
columnOptions: Table['columns'];
testMode: DatasetTestModeType;
handleTestModeChange: (mode: DatasetTestModeType) => void;
@ -93,7 +93,7 @@ const DataQualityTab = ({
setActiveData(undefined);
};
const onColumnTestSave = (data: CreateColumnTest) => {
const onColumnTestSave = (data: ColumnTest) => {
handleAddColumnTestCase(data);
setActiveData(undefined);
};

View File

@ -36,7 +36,11 @@ import {
import { getEntityFeedLink } from '../../utils/EntityUtils';
import { getDefaultValue } from '../../utils/FeedElementUtils';
import { getEntityFieldThreadCounts } from '../../utils/FeedUtils';
import { getTagsWithoutTier, getUsagePercentile } from '../../utils/TableUtils';
import {
getTableTestsValue,
getTagsWithoutTier,
getUsagePercentile,
} from '../../utils/TableUtils';
import ActivityFeedList from '../ActivityFeed/ActivityFeedList/ActivityFeedList';
import ActivityThreadPanel from '../ActivityFeed/ActivityThreadPanel/ActivityThreadPanel';
import Description from '../common/description/Description';
@ -111,6 +115,7 @@ const DatasetDetails: React.FC<DatasetDetailsProps> = ({
handleShowTestForm,
handleRemoveTableTest,
handleRemoveColumnTest,
qualityTestFormHandler,
}: DatasetDetailsProps) => {
const { isAuthDisabled } = useAuth();
const [isEdit, setIsEdit] = useState(false);
@ -350,6 +355,7 @@ const DatasetDetails: React.FC<DatasetDetailsProps> = ({
''
),
},
{ key: 'Tests', value: getTableTestsValue(tableTestCase) },
];
const onDescriptionEdit = (): void => {
@ -616,7 +622,10 @@ const DatasetDetails: React.FC<DatasetDetailsProps> = ({
columns={columns.map((col) => ({
constraint: col.constraint as string,
colName: col.name,
colType: col.dataTypeDisplay as string,
colTests: col.columnTests,
}))}
qualityTestFormHandler={qualityTestFormHandler}
tableProfiles={tableProfile}
/>
</div>

View File

@ -34,8 +34,9 @@ import { TableTest, TableTestType } from '../../generated/tests/tableTest';
import { EntityLineage } from '../../generated/type/entityLineage';
import { TagLabel } from '../../generated/type/tagLabel';
import {
CreateColumnTest,
ColumnTest,
DatasetTestModeType,
ModifiedTableColumn,
} from '../../interface/dataQuality.interface';
import { TitleBreadcrumbProps } from '../common/title-breadcrumb/title-breadcrumb.interface';
import { Edge, EdgeData } from '../EntityLineage/EntityLineage.interface';
@ -60,7 +61,7 @@ export interface DatasetDetailsProps {
description: string;
tableProfile: Table['tableProfile'];
tableQueries: Table['tableQueries'];
columns: Table['columns'];
columns: ModifiedTableColumn[];
tier: TagLabel;
sampleData: TableData;
entityLineage: EntityLineage;
@ -78,6 +79,10 @@ export interface DatasetDetailsProps {
testMode: DatasetTestModeType;
tableTestCase: TableTest[];
showTestForm: boolean;
qualityTestFormHandler: (
tabValue: number,
testMode: DatasetTestModeType
) => void;
handleShowTestForm: (value: boolean) => void;
handleTestModeChange: (mode: DatasetTestModeType) => void;
createThread: (data: CreateThread) => void;
@ -94,7 +99,7 @@ export interface DatasetDetailsProps {
entityLineageHandler: (lineage: EntityLineage) => void;
postFeedHandler: (value: string, id: string) => void;
handleAddTableTestCase: (data: CreateTableTest) => void;
handleAddColumnTestCase: (data: CreateColumnTest) => void;
handleAddColumnTestCase: (data: ColumnTest) => void;
handleRemoveTableTest: (testType: TableTestType) => void;
handleRemoveColumnTest: (
columnName: string,

View File

@ -94,6 +94,7 @@ const DatasetDetailsProps = {
handleRemoveTableTest: jest.fn(),
handleRemoveColumnTest: jest.fn(),
handleTestModeChange: jest.fn(),
qualityTestFormHandler: jest.fn(),
};
jest.mock('../ManageTab/ManageTab.component', () => {
return jest.fn().mockReturnValue(<p>ManageTab</p>);

View File

@ -26,7 +26,12 @@ import {
Table,
} from '../../generated/entity/data/table';
import { Operation } from '../../generated/entity/policies/accessControl/rule';
import { TestCaseStatus } from '../../generated/tests/tableTest';
import { LabelType, State, TagLabel } from '../../generated/type/tagLabel';
import {
ColumnTest,
ModifiedTableColumn,
} from '../../interface/dataQuality.interface';
import {
getHtmlForNonAdminAction,
getPartialNameFromFQN,
@ -50,7 +55,7 @@ import Tags from '../tags/tags';
type Props = {
owner: Table['owner'];
tableColumns: Table['columns'];
tableColumns: ModifiedTableColumn[];
joins: Array<ColumnJoins>;
searchText?: string;
columnName: string;
@ -58,7 +63,7 @@ type Props = {
isReadOnly?: boolean;
entityFqn?: string;
entityFieldThreads?: EntityFieldThreads[];
onUpdate?: (columns: Table['columns']) => void;
onUpdate?: (columns: ModifiedTableColumn[]) => void;
onThreadLinkSelect?: (value: string) => void;
onEntityFieldSelect?: (value: string) => void;
};
@ -86,6 +91,10 @@ const EntityTable = ({
Header: 'Type',
accessor: 'dataTypeDisplay',
},
{
Header: 'Data Quality',
accessor: 'columnTests',
},
{
Header: 'Description',
accessor: 'description',
@ -98,7 +107,9 @@ const EntityTable = ({
[]
);
const [searchedColumns, setSearchedColumns] = useState<Table['columns']>([]);
const [searchedColumns, setSearchedColumns] = useState<ModifiedTableColumn[]>(
[]
);
const data = React.useMemo(
() => makeData(searchedColumns),
@ -158,7 +169,7 @@ const EntityTable = ({
};
const updateColumnDescription = (
tableCols: Table['columns'],
tableCols: ModifiedTableColumn[],
changedColName: string,
description: string
) => {
@ -167,7 +178,7 @@ const EntityTable = ({
col.description = description;
} else {
updateColumnDescription(
col?.children as Table['columns'],
col?.children as ModifiedTableColumn[],
changedColName,
description
);
@ -176,7 +187,7 @@ const EntityTable = ({
};
const updateColumnTags = (
tableCols: Table['columns'],
tableCols: ModifiedTableColumn[],
changedColName: string,
newColumnTags: Array<string>
) => {
@ -204,7 +215,7 @@ const EntityTable = ({
col.tags = getUpdatedTags(col);
} else {
updateColumnTags(
col?.children as Table['columns'],
col?.children as ModifiedTableColumn[],
changedColName,
newColumnTags
);
@ -309,7 +320,8 @@ const EntityTable = ({
{headerGroup.headers.map((column: any, index: number) => (
<th
className={classNames('tableHead-cell', {
'tw-w-60': column.id === 'tags',
'tw-w-60':
column.id === 'tags' || column.id === 'columnTests',
})}
key={index}
{...column.getHeaderProps()}>
@ -332,6 +344,22 @@ const EntityTable = ({
{...row.getRowProps()}>
{/* eslint-disable-next-line */}
{row.cells.map((cell: any, index: number) => {
const columnTests =
cell.column.id === 'columnTests'
? ((cell.value ?? []) as ColumnTest[])
: ([] as ColumnTest[]);
const columnTestLength = columnTests.length;
const failingTests = columnTests.filter((test) =>
test.results?.some(
(t) => t.testCaseStatus === TestCaseStatus.Failed
)
);
const passingTests = columnTests.filter((test) =>
test.results?.some(
(t) => t.testCaseStatus === TestCaseStatus.Success
)
);
return (
<td
className={classNames(
@ -354,6 +382,34 @@ const EntityTable = ({
</span>
) : null}
{cell.column.id === 'columnTests' && (
<Fragment>
{columnTestLength ? (
<Fragment>
{failingTests.length ? (
<div className="tw-flex">
<p className="tw-mr-2">
<i className="fas fa-times tw-text-status-failed" />
</p>
<p>
{`${failingTests.length}/${columnTestLength} tests failing`}
</p>
</div>
) : (
<div className="tw-flex">
<div className="tw-mr-2">
<i className="fas fa-check-square tw-text-status-success" />
</div>
<p>{`${passingTests.length} tests`}</p>
</div>
)}
</Fragment>
) : (
'--'
)}
</Fragment>
)}
{cell.column.id === 'dataTypeDisplay' && (
<>
{isReadOnly ? (

View File

@ -12,31 +12,73 @@
*/
import classNames from 'classnames';
import React, { Fragment, useState } from 'react';
import React, { FC, Fragment } from 'react';
import { Link } from 'react-router-dom';
import { TITLE_FOR_NON_ADMIN_ACTION } from '../../constants/constants';
import { Table, TableProfile } from '../../generated/entity/data/table';
import { useAuth } from '../../hooks/authHooks';
import {
ColumnTest,
DatasetTestModeType,
} from '../../interface/dataQuality.interface';
import { getConstraintIcon } from '../../utils/TableUtils';
import TableProfilerGraph from './TableProfilerGraph.component';
import { Button } from '../buttons/Button/Button';
import NonAdminAction from '../common/non-admin-action/NonAdminAction';
import PopOver from '../common/popover/PopOver';
import RichTextEditorPreviewer from '../common/rich-text-editor/RichTextEditorPreviewer';
type Props = {
tableProfiles: Table['tableProfile'];
columns: Array<{ constraint: string; colName: string }>;
columns: Array<{
constraint: string;
colName: string;
colType: string;
colTests?: ColumnTest[];
}>;
qualityTestFormHandler: (
tabValue: number,
testMode: DatasetTestModeType
) => void;
};
type ProfilerGraphData = Array<{
date: Date;
value: number;
}>;
const PercentageGraph = ({
percentage,
title,
}: {
percentage: number;
title: string;
}) => {
return (
<PopOver
position="top"
title={`${percentage}% ${title}`}
trigger="mouseenter">
<div className="tw-border tw-border-primary tw-h-8 tw-w-20">
<div
className="tw-bg-primary tw-opacity-40 tw-h-full"
style={{ width: `${percentage}%` }}
/>
</div>
</PopOver>
);
};
const TableProfiler = ({ tableProfiles, columns }: Props) => {
const [expandedColumn, setExpandedColumn] = useState<{
name: string;
isExpanded: boolean;
}>({
name: '',
isExpanded: false,
});
const excludedMetrics = [
'profilDate',
'name',
'nullCount',
'nullProportion',
'uniqueCount',
'uniqueProportion',
'rows',
];
const TableProfiler: FC<Props> = ({
tableProfiles,
columns,
qualityTestFormHandler,
}) => {
const { isAuthDisabled, isAdminUser } = useAuth();
const modifiedData = tableProfiles?.map((tableProfile: TableProfile) => ({
rows: tableProfile.rowCount,
profileDate: tableProfile.profileDate,
@ -49,19 +91,27 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => {
(colProfile) => colProfile.name === column.colName
);
return { profilDate: md.profileDate, ...currentColumn, rows: md.rows };
return {
profilDate: md.profileDate,
...currentColumn,
rows: md.rows,
};
});
return {
name: column,
columnMetrics: Object.entries(data?.[0] ?? {}).map((d) => ({
key: d[0],
value: d[1],
})),
columnTests: column.colTests,
data,
min: data?.length ? data[0].min ?? 0 : 0,
max: data?.length ? data[0].max ?? 0 : 0,
type: column.colType,
};
});
return (
<>
<div className="tw-table-responsive tw-overflow-x-auto">
{tableProfiles?.length ? (
<table
className="tw-w-full"
@ -70,11 +120,13 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => {
<thead>
<tr className="tableHead-row">
<th className="tableHead-cell">Column Name</th>
<th className="tableHead-cell">Distinct Ratio (%)</th>
<th className="tableHead-cell">Null Ratio (%)</th>
<th className="tableHead-cell">Min</th>
<th className="tableHead-cell">Max</th>
<th className="tableHead-cell">Standard Deviation</th>
<th className="tableHead-cell">Type</th>
<th className="tableHead-cell">Null</th>
<th className="tableHead-cell">Unique</th>
<th className="tableHead-cell">Distinct</th>
<th className="tableHead-cell">Metrics</th>
<th className="tableHead-cell">Tests</th>
<th className="tableHead-cell" />
</tr>
</thead>
{columnSpecificData.map((col, colIndex) => {
@ -88,27 +140,6 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => {
className="tw-relative tableBody-cell"
data-testid="tableBody-cell">
<div className="tw-flex">
<span
className="tw-mr-2 tw-cursor-pointer"
onClick={() =>
setExpandedColumn((prevState) => ({
name: col.name.colName,
isExpanded:
prevState.name === col.name.colName
? !prevState.isExpanded
: true,
}))
}>
{expandedColumn.name === col.name.colName ? (
expandedColumn.isExpanded ? (
<i className="fas fa-caret-down" />
) : (
<i className="fas fa-caret-right" />
)
) : (
<i className="fas fa-caret-right" />
)}
</span>
{col.name.constraint && (
<span className="tw-mr-3 tw--ml-2">
{getConstraintIcon(
@ -120,84 +151,140 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => {
<span>{col.name.colName}</span>
</div>
</td>
<td className="tw-relative tableBody-cell profiler-graph">
<TableProfilerGraph
data={
col.data
?.map((d) => ({
date: d.profilDate,
value: d.uniqueProportion ?? 0,
}))
.reverse() as ProfilerGraphData
<td
className="tw-relative tableBody-cell"
data-testid="tableBody-cell">
<div className="tw-flex">
{col.name.colType.length > 25 ? (
<span>
<PopOver
html={
<div className="tw-break-words">
<span>{col.name.colType.toLowerCase()}</span>
</div>
}
position="bottom"
theme="light"
trigger="click">
<div className="tw-cursor-pointer tw-underline tw-inline-block">
<RichTextEditorPreviewer
markdown={`${col.name.colType
.slice(0, 20)
.toLowerCase()}...`}
/>
</td>
<td className="tw-relative tableBody-cell profiler-graph">
<TableProfilerGraph
data={
col.data
?.map((d) => ({
date: d.profilDate,
value: d.nullProportion ?? 0,
}))
.reverse() as ProfilerGraphData
}
/>
</td>
<td className="tw-relative tableBody-cell">{col.min}</td>
<td className="tw-relative tableBody-cell">{col.max}</td>
<td className="tw-relative tableBody-cell profiler-graph">
<TableProfilerGraph
data={
col.data
?.map((d) => ({
date: d.profilDate,
value: d.stddev ?? 0,
}))
.reverse() as ProfilerGraphData
}
/>
</td>
</tr>
</tbody>
{expandedColumn.name === col.name.colName &&
expandedColumn.isExpanded && (
<tbody>
{col.data?.map((colData, index) => (
<tr
className={classNames(
'tableBody-row tw-border-0 tw-border-l tw-border-r',
{
'tw-border-b':
columnSpecificData.length - 1 === colIndex &&
col.data?.length === index + 1,
}
)}
key={index}>
<td className="tw-relative tableBody-cell">
<span className="tw-pl-6">
{colData.profilDate}
</div>
</PopOver>
</span>
) : (
col.name.colType.toLowerCase()
)}
</div>
</td>
<td className="tw-relative tableBody-cell">
{colData.uniqueProportion ?? 0}
<td className="tw-relative tableBody-cell profiler-graph">
<PercentageGraph
percentage={(col.data?.[0]?.nullProportion ?? 0) * 100}
title="null value"
/>
</td>
<td className="tw-relative tableBody-cell">
{colData.nullProportion ?? 0}
<td className="tw-relative tableBody-cell profiler-graph">
<PercentageGraph
percentage={
(col.data?.[0]?.uniqueProportion ?? 0) * 100
}
title="unique value"
/>
</td>
<td className="tw-relative tableBody-cell">
{colData.min ?? 0}
<td className="tw-relative tableBody-cell profiler-graph">
<PercentageGraph
percentage={
((col.data?.[0]?.distinctCount ?? 0) /
(col.data?.[0]?.rows ?? 0)) *
100
}
title="distinct value"
/>
</td>
<td className="tw-relative tableBody-cell">
{colData.max ?? 0}
<td
className="tw-relative tableBody-cell"
data-testid="tableBody-cell">
<div className="tw-border tw-border-main tw-rounded tw-p-2 tw-min-h-32 tw-max-h-44 tw-overflow-y-auto">
{col.columnMetrics
.filter((m) => !excludedMetrics.includes(m.key))
.map((m, i) => (
<p className="tw-mb-1 tw-flex" key={i}>
<span className="tw-mx-1">{m.key}</span>
<span className="tw-mx-1">-</span>
<span className="tw-mx-1">{m.value}</span>
</p>
))}
</div>
</td>
<td className="tw-relative tableBody-cell">
{colData.stddev ?? 0}
<td
className="tw-relative tableBody-cell"
colSpan={2}
data-testid="tableBody-cell">
<div className="tw-flex tw-justify-between">
{col.columnTests ? (
<div className="tw-border tw-border-main tw-rounded tw-p-2 tw-min-h-32 tw-max-h-44 tw-overflow-y-auto tw-flex-1">
{col.columnTests.map((m, i) => (
<div className="tw-flex tw-mb-2" key={i}>
<p className="tw-mr-2">
{m.results?.every(
(result) =>
result.testCaseStatus === 'Success'
) ? (
<i className="fas fa-check-square tw-text-status-success" />
) : (
<i className="fas fa-times tw-text-status-failed" />
)}
</p>
<div>
<span className="tw-mx-1 tw-font-medium">
{m.testCase.columnTestType}
</span>
<div className="tw-mx-1">
{Object.entries(
m.testCase.config ?? {}
).map((config, i) => (
<p className="tw-mx-1" key={i}>
<span>{config[0]}:</span>
<span>{config[1] ?? 'null'}</span>
</p>
))}
</div>
</div>
</div>
))}
</div>
) : (
`No tests available`
)}
<div className="tw-self-center tw-ml-5">
<NonAdminAction
position="bottom"
title={TITLE_FOR_NON_ADMIN_ACTION}>
<Button
className={classNames(
'tw-px-2 tw-py-0.5 tw-rounded tw-border-grey-muted',
{
'tw-opacity-40':
!isAdminUser && !isAuthDisabled,
}
)}
size="custom"
type="button"
variant="outlined"
onClick={() =>
qualityTestFormHandler(6, 'column')
}>
Add Test
</Button>
</NonAdminAction>
</div>
</div>
</td>
</tr>
))}
</tbody>
)}
</Fragment>
);
})}
@ -218,7 +305,7 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => {
</Link>
</div>
)}
</>
</div>
);
};

View File

@ -15,86 +15,18 @@
export const mockFeedData = [
{
index: 'table_search_index',
id: 'd336d794-6fce-4094-a064-7ceb3b208dab',
name: 'dim_address',
description:
'This dimension table contains the billing and shipping addresses of customers. You can join this table with the sales table to generate lists of the billing and shipping addresses. Customers can enter their addresses more than once, so the same address can appear in more than one row in this table. This table contains one row per customer address.',
fullyQualifiedName: 'bigquery_gcp.shopify.dim_address',
tableType: 'Regular',
tags: [
'PersonalData.Personal',
'PII.Sensitive',
'PII.NonSensitive',
'PII.Sensitive',
'PersonalData.Personal',
'User.FacePhoto',
],
service: 'bigquery_gcp',
serviceType: 'BigQuery',
database: 'shopify',
entityType: 'table',
changeDescriptions: [
{
id: '2b133a6d-8562-4220-a997-eaaa0376cad3',
href: 'http://localhost:8585/api/v1/feed/2b133a6d-8562-4220-a997-eaaa0376cad3',
threadTs: 1646577496489,
about: '<#E/table/bigquery_gcp.shopify.dim_staff/description>',
entityId: '37e50e09-d5b6-4609-821c-1ed84d92dd57',
createdBy: 'aaron_johnson0',
updatedAt: 1646577496489,
updatedBy: 'anonymous',
updatedAt: 1640239422867,
fieldsAdded: [],
fieldsUpdated: [],
fieldsDeleted: [],
},
{
updatedBy: 'anonymous',
fieldsAdded: [
{
newValue:
'[{"tagFQN":"PersonalData.Personal","labelType":"Manual","state":"Confirmed"}]',
name: 'columns.address_id.tags',
},
],
fieldsUpdated: [],
fieldsDeleted: [],
updatedAt: 1640247571788,
},
{
updatedBy: 'anonymous',
fieldsAdded: [
{
newValue:
'[{"tagFQN":"PII.Sensitive","labelType":"Manual","state":"Confirmed"}]',
name: 'columns.shop_id.tags',
},
],
fieldsUpdated: [],
fieldsDeleted: [],
updatedAt: 1640248828684,
},
{
updatedBy: 'anonymous',
fieldsAdded: [
{
newValue:
'[{"tagFQN":"PII.NonSensitive","labelType":"Manual","state":"Confirmed"}]',
name: 'columns.first_name.tags',
},
],
fieldsUpdated: [],
fieldsDeleted: [],
updatedAt: 1640248877069,
},
{
updatedBy: 'anonymous',
fieldsAdded: [
{
newValue:
'[{"tagFQN":"PII.Sensitive","description":"PII which if lost, compromised, or disclosed without authorization, could result in substantial harm, embarrassment, inconvenience, or unfairness to an individual.","labelType":"Derived","state":"Confirmed"},{"tagFQN":"PersonalData.Personal","description":"Data that can be used to directly or indirectly identify a person.","labelType":"Derived","state":"Confirmed"},{"tagFQN":"User.FacePhoto","labelType":"Manual","state":"Confirmed"}]',
name: 'columns.last_name.tags',
},
],
fieldsUpdated: [],
fieldsDeleted: [],
updatedAt: 1640248886960,
},
],
resolved: false,
message: 'Can you add a description?',
postsCount: 0,
posts: [],
},
];

View File

@ -628,6 +628,10 @@ export interface ColumnProfile {
* Variance of a column.
*/
variance?: number;
minLength?: number;
maxLength?: number;
}
export interface HistogramObject {

View File

@ -30,12 +30,19 @@ export interface TestCaseConfigType {
missingValueMatch?: string;
}
export interface CreateColumnTest {
export interface Result {
executionTime: number;
testCaseStatus: string;
result: string;
}
export interface ColumnTest {
id?: string;
columnName: string;
description?: string;
executionFrequency?: TestCaseExecutionFrequency;
owner?: EntityReference;
results?: Result[];
testCase: {
columnTestType: ColumnTestType;
config?: TestCaseConfigType;
@ -45,7 +52,7 @@ export interface CreateColumnTest {
export type DatasetTestModeType = 'table' | 'column';
export interface ModifiedTableColumn extends Column {
columnTests?: CreateColumnTest[];
columnTests?: ColumnTest[];
}
export interface TableTestDataType {

View File

@ -77,7 +77,7 @@ import { EntityLineage } from '../../generated/type/entityLineage';
import { TagLabel } from '../../generated/type/tagLabel';
import useToastContext from '../../hooks/useToastContext';
import {
CreateColumnTest,
ColumnTest,
DatasetTestModeType,
ModifiedTableColumn,
} from '../../interface/dataQuality.interface';
@ -118,7 +118,7 @@ const DatasetDetailsPage: FunctionComponent = () => {
TitleBreadcrumbProps['titleLinks']
>([]);
const [description, setDescription] = useState('');
const [columns, setColumns] = useState<Table['columns']>([]);
const [columns, setColumns] = useState<ModifiedTableColumn[]>([]);
const [sampleData, setSampleData] = useState<TableData>({
columns: [],
rows: [],
@ -191,6 +191,15 @@ const DatasetDetailsPage: FunctionComponent = () => {
}
};
const qualityTestFormHandler = (
tabValue: number,
testMode: DatasetTestModeType
) => {
activeTabHandler(tabValue);
setTestMode(testMode);
setShowTestForm(true);
};
const getLineageData = () => {
setIsLineageLoading(true);
getLineageByFQN(tableFQN, EntityType.TABLE)
@ -647,7 +656,7 @@ const DatasetDetailsPage: FunctionComponent = () => {
});
};
const handleAddColumnTestCase = (data: CreateColumnTest) => {
const handleAddColumnTestCase = (data: ColumnTest) => {
addColumnTestCase(tableDetails.id, data)
.then((res: AxiosResponse) => {
const columnTestRes = res.data.columns.find(
@ -786,6 +795,7 @@ const DatasetDetailsPage: FunctionComponent = () => {
loadNodeHandler={loadNodeHandler}
owner={owner as Table['owner'] & { displayName: string }}
postFeedHandler={postFeedHandler}
qualityTestFormHandler={qualityTestFormHandler}
removeLineageHandler={removeLineageHandler}
sampleData={sampleData}
setActiveTabHandler={activeTabHandler}

View File

@ -127,7 +127,7 @@ const TourPage = () => {
pipelineCount: 8,
}}
error=""
feedData={mockFeedData as unknown as MyDataProps['feedData']}
feedData={mockFeedData as MyDataProps['feedData']}
feedFilter={FeedFilter.ALL}
feedFilterHandler={() => {
setMyDataSearchResult(mockData);
@ -185,7 +185,7 @@ const TourPage = () => {
entityLineage={mockDatasetData.entityLineage}
entityLineageHandler={handleCountChange}
entityName={mockDatasetData.entityName}
entityThread={[]}
entityThread={mockFeedData}
feedCount={0}
followTableHandler={handleCountChange}
followers={mockDatasetData.followers}
@ -205,6 +205,7 @@ const TourPage = () => {
loadNodeHandler={handleCountChange}
owner={undefined as unknown as DatasetOwner}
postFeedHandler={handleCountChange}
qualityTestFormHandler={handleCountChange}
removeLineageHandler={handleCountChange}
sampleData={mockDatasetData.sampleData}
setActiveTabHandler={(tab) => setdatasetActiveTab(tab)}

View File

@ -14,7 +14,7 @@
import classNames from 'classnames';
import { upperCase } from 'lodash';
import { EntityTags, TableDetail } from 'Models';
import React from 'react';
import React, { Fragment } from 'react';
import AppState from '../AppState';
import PopOver from '../components/common/popover/PopOver';
import {
@ -27,8 +27,10 @@ import {
import { EntityType } from '../enums/entity.enum';
import { SearchIndex } from '../enums/search.enum';
import { ConstraintTypes } from '../enums/table.enum';
import { Column, DataType, Table } from '../generated/entity/data/table';
import { Column, DataType } from '../generated/entity/data/table';
import { TableTest, TestCaseStatus } from '../generated/tests/tableTest';
import { TagLabel } from '../generated/type/tagLabel';
import { ModifiedTableColumn } from '../interface/dataQuality.interface';
import { ordinalize } from './StringsUtils';
import SVGIcons from './SvgUtils';
@ -253,7 +255,7 @@ export const makeRow = (column: Column) => {
};
export const makeData = (
columns: Table['columns'] = []
columns: ModifiedTableColumn[] = []
): Array<Column & { subRows: Column[] | undefined }> => {
const data = columns.map((column) => ({
...makeRow(column),
@ -290,3 +292,38 @@ export const getDataTypeString = (dataType: string): string => {
return dataType;
}
};
export const getTableTestsValue = (tableTestCase: TableTest[]) => {
const tableTestLength = tableTestCase.length;
const failingTests = tableTestCase.filter((test) =>
test.results?.some((t) => t.testCaseStatus === TestCaseStatus.Failed)
);
const passingTests = tableTestCase.filter((test) =>
test.results?.some((t) => t.testCaseStatus === TestCaseStatus.Success)
);
return (
<Fragment>
{tableTestLength ? (
<Fragment>
{failingTests.length ? (
<div className="tw-flex">
<p className="tw-mr-2">
<i className="fas fa-times tw-text-status-failed" />
</p>
<p>{`${failingTests.length}/${tableTestLength} tests failing`}</p>
</div>
) : (
<div className="tw-flex">
<div className="tw-mr-2">
<i className="fas fa-check-square tw-text-status-success" />
</div>
<p>{`${passingTests.length} tests`}</p>
</div>
)}
</Fragment>
) : null}
</Fragment>
);
};