mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-10-05 13:56:41 +00:00
UI : New Data profiler Layout. (#3184)
* UI : New Data profiler Layout. * Add tests column in data profiler * Add percentage graph * Add data quality column * Adding table level tests * Minor change * Change mock data for feed * Addressing review comments
This commit is contained in:
parent
a6e98b3fdb
commit
40b64d2d3e
@ -16,7 +16,7 @@ import { Table } from 'Models';
|
||||
import { ColumnTestType } from '../enums/columnTest.enum';
|
||||
import { CreateTableTest } from '../generated/api/tests/createTableTest';
|
||||
import { TableTestType } from '../generated/tests/tableTest';
|
||||
import { CreateColumnTest } from '../interface/dataQuality.interface';
|
||||
import { ColumnTest } from '../interface/dataQuality.interface';
|
||||
import { getURLWithQueryFields } from '../utils/APIUtils';
|
||||
import APIClient from './index';
|
||||
|
||||
@ -138,7 +138,7 @@ export const deleteTableTestCase = (
|
||||
);
|
||||
};
|
||||
|
||||
export const addColumnTestCase = (tableId: string, data: CreateColumnTest) => {
|
||||
export const addColumnTestCase = (tableId: string, data: ColumnTest) => {
|
||||
const configOptions = {
|
||||
headers: { 'Content-type': 'application/json' },
|
||||
};
|
||||
|
@ -16,7 +16,7 @@ import { CreateTableTest } from '../../generated/api/tests/createTableTest';
|
||||
import { Table } from '../../generated/entity/data/table';
|
||||
import { TableTest } from '../../generated/tests/tableTest';
|
||||
import {
|
||||
CreateColumnTest,
|
||||
ColumnTest,
|
||||
TableTestDataType,
|
||||
} from '../../interface/dataQuality.interface';
|
||||
import ColumnTestForm from './Forms/ColumnTestForm';
|
||||
@ -28,7 +28,7 @@ type Props = {
|
||||
columnOptions: Table['columns'];
|
||||
tableTestCase: TableTest[];
|
||||
handleAddTableTestCase: (data: CreateTableTest) => void;
|
||||
handleAddColumnTestCase: (data: CreateColumnTest) => void;
|
||||
handleAddColumnTestCase: (data: ColumnTest) => void;
|
||||
onFormCancel: () => void;
|
||||
};
|
||||
|
||||
@ -53,7 +53,7 @@ const AddDataQualityTest = ({
|
||||
) : (
|
||||
<ColumnTestForm
|
||||
column={columnOptions}
|
||||
data={data as CreateColumnTest}
|
||||
data={data as ColumnTest}
|
||||
handleAddColumnTestCase={handleAddColumnTestCase}
|
||||
onFormCancel={onFormCancel}
|
||||
/>
|
||||
|
@ -19,7 +19,7 @@ import { ColumnTestType } from '../../../enums/columnTest.enum';
|
||||
import { TestCaseExecutionFrequency } from '../../../generated/api/tests/createTableTest';
|
||||
import { Table } from '../../../generated/entity/data/table';
|
||||
import {
|
||||
CreateColumnTest,
|
||||
ColumnTest,
|
||||
ModifiedTableColumn,
|
||||
} from '../../../interface/dataQuality.interface';
|
||||
import {
|
||||
@ -33,9 +33,9 @@ import { Button } from '../../buttons/Button/Button';
|
||||
import MarkdownWithPreview from '../../common/editor/MarkdownWithPreview';
|
||||
|
||||
type Props = {
|
||||
data: CreateColumnTest;
|
||||
data: ColumnTest;
|
||||
column: ModifiedTableColumn[];
|
||||
handleAddColumnTestCase: (data: CreateColumnTest) => void;
|
||||
handleAddColumnTestCase: (data: ColumnTest) => void;
|
||||
onFormCancel: () => void;
|
||||
};
|
||||
|
||||
@ -118,7 +118,7 @@ const ColumnTestForm = ({
|
||||
const selectedColumn = column.find((d) => d.name === name);
|
||||
const existingTests =
|
||||
selectedColumn?.columnTests?.map(
|
||||
(d: CreateColumnTest) => d.testCase.columnTestType
|
||||
(d: ColumnTest) => d.testCase.columnTestType
|
||||
) || [];
|
||||
if (existingTests.length) {
|
||||
const newTest = Object.values(ColumnTestType).filter(
|
||||
@ -222,7 +222,7 @@ const ColumnTestForm = ({
|
||||
|
||||
const handleSave = () => {
|
||||
if (validateForm()) {
|
||||
const columnTestObj: CreateColumnTest = {
|
||||
const columnTestObj: ColumnTest = {
|
||||
columnName: columnName,
|
||||
description: markdownRef.current?.getEditorContent() || undefined,
|
||||
executionFrequency: frequency,
|
||||
|
@ -17,7 +17,7 @@ import { CreateTableTest } from '../../generated/api/tests/createTableTest';
|
||||
import { Table } from '../../generated/entity/data/table';
|
||||
import { TableTest, TableTestType } from '../../generated/tests/tableTest';
|
||||
import {
|
||||
CreateColumnTest,
|
||||
ColumnTest,
|
||||
DatasetTestModeType,
|
||||
ModifiedTableColumn,
|
||||
TableTestDataType,
|
||||
@ -27,7 +27,7 @@ import DataQualityTest from '../DataQualityTest/DataQualityTest';
|
||||
|
||||
type Props = {
|
||||
handleAddTableTestCase: (data: CreateTableTest) => void;
|
||||
handleAddColumnTestCase: (data: CreateColumnTest) => void;
|
||||
handleAddColumnTestCase: (data: ColumnTest) => void;
|
||||
columnOptions: Table['columns'];
|
||||
testMode: DatasetTestModeType;
|
||||
handleTestModeChange: (mode: DatasetTestModeType) => void;
|
||||
@ -93,7 +93,7 @@ const DataQualityTab = ({
|
||||
setActiveData(undefined);
|
||||
};
|
||||
|
||||
const onColumnTestSave = (data: CreateColumnTest) => {
|
||||
const onColumnTestSave = (data: ColumnTest) => {
|
||||
handleAddColumnTestCase(data);
|
||||
setActiveData(undefined);
|
||||
};
|
||||
|
@ -36,7 +36,11 @@ import {
|
||||
import { getEntityFeedLink } from '../../utils/EntityUtils';
|
||||
import { getDefaultValue } from '../../utils/FeedElementUtils';
|
||||
import { getEntityFieldThreadCounts } from '../../utils/FeedUtils';
|
||||
import { getTagsWithoutTier, getUsagePercentile } from '../../utils/TableUtils';
|
||||
import {
|
||||
getTableTestsValue,
|
||||
getTagsWithoutTier,
|
||||
getUsagePercentile,
|
||||
} from '../../utils/TableUtils';
|
||||
import ActivityFeedList from '../ActivityFeed/ActivityFeedList/ActivityFeedList';
|
||||
import ActivityThreadPanel from '../ActivityFeed/ActivityThreadPanel/ActivityThreadPanel';
|
||||
import Description from '../common/description/Description';
|
||||
@ -111,6 +115,7 @@ const DatasetDetails: React.FC<DatasetDetailsProps> = ({
|
||||
handleShowTestForm,
|
||||
handleRemoveTableTest,
|
||||
handleRemoveColumnTest,
|
||||
qualityTestFormHandler,
|
||||
}: DatasetDetailsProps) => {
|
||||
const { isAuthDisabled } = useAuth();
|
||||
const [isEdit, setIsEdit] = useState(false);
|
||||
@ -350,6 +355,7 @@ const DatasetDetails: React.FC<DatasetDetailsProps> = ({
|
||||
''
|
||||
),
|
||||
},
|
||||
{ key: 'Tests', value: getTableTestsValue(tableTestCase) },
|
||||
];
|
||||
|
||||
const onDescriptionEdit = (): void => {
|
||||
@ -616,7 +622,10 @@ const DatasetDetails: React.FC<DatasetDetailsProps> = ({
|
||||
columns={columns.map((col) => ({
|
||||
constraint: col.constraint as string,
|
||||
colName: col.name,
|
||||
colType: col.dataTypeDisplay as string,
|
||||
colTests: col.columnTests,
|
||||
}))}
|
||||
qualityTestFormHandler={qualityTestFormHandler}
|
||||
tableProfiles={tableProfile}
|
||||
/>
|
||||
</div>
|
||||
|
@ -34,8 +34,9 @@ import { TableTest, TableTestType } from '../../generated/tests/tableTest';
|
||||
import { EntityLineage } from '../../generated/type/entityLineage';
|
||||
import { TagLabel } from '../../generated/type/tagLabel';
|
||||
import {
|
||||
CreateColumnTest,
|
||||
ColumnTest,
|
||||
DatasetTestModeType,
|
||||
ModifiedTableColumn,
|
||||
} from '../../interface/dataQuality.interface';
|
||||
import { TitleBreadcrumbProps } from '../common/title-breadcrumb/title-breadcrumb.interface';
|
||||
import { Edge, EdgeData } from '../EntityLineage/EntityLineage.interface';
|
||||
@ -60,7 +61,7 @@ export interface DatasetDetailsProps {
|
||||
description: string;
|
||||
tableProfile: Table['tableProfile'];
|
||||
tableQueries: Table['tableQueries'];
|
||||
columns: Table['columns'];
|
||||
columns: ModifiedTableColumn[];
|
||||
tier: TagLabel;
|
||||
sampleData: TableData;
|
||||
entityLineage: EntityLineage;
|
||||
@ -78,6 +79,10 @@ export interface DatasetDetailsProps {
|
||||
testMode: DatasetTestModeType;
|
||||
tableTestCase: TableTest[];
|
||||
showTestForm: boolean;
|
||||
qualityTestFormHandler: (
|
||||
tabValue: number,
|
||||
testMode: DatasetTestModeType
|
||||
) => void;
|
||||
handleShowTestForm: (value: boolean) => void;
|
||||
handleTestModeChange: (mode: DatasetTestModeType) => void;
|
||||
createThread: (data: CreateThread) => void;
|
||||
@ -94,7 +99,7 @@ export interface DatasetDetailsProps {
|
||||
entityLineageHandler: (lineage: EntityLineage) => void;
|
||||
postFeedHandler: (value: string, id: string) => void;
|
||||
handleAddTableTestCase: (data: CreateTableTest) => void;
|
||||
handleAddColumnTestCase: (data: CreateColumnTest) => void;
|
||||
handleAddColumnTestCase: (data: ColumnTest) => void;
|
||||
handleRemoveTableTest: (testType: TableTestType) => void;
|
||||
handleRemoveColumnTest: (
|
||||
columnName: string,
|
||||
|
@ -94,6 +94,7 @@ const DatasetDetailsProps = {
|
||||
handleRemoveTableTest: jest.fn(),
|
||||
handleRemoveColumnTest: jest.fn(),
|
||||
handleTestModeChange: jest.fn(),
|
||||
qualityTestFormHandler: jest.fn(),
|
||||
};
|
||||
jest.mock('../ManageTab/ManageTab.component', () => {
|
||||
return jest.fn().mockReturnValue(<p>ManageTab</p>);
|
||||
|
@ -26,7 +26,12 @@ import {
|
||||
Table,
|
||||
} from '../../generated/entity/data/table';
|
||||
import { Operation } from '../../generated/entity/policies/accessControl/rule';
|
||||
import { TestCaseStatus } from '../../generated/tests/tableTest';
|
||||
import { LabelType, State, TagLabel } from '../../generated/type/tagLabel';
|
||||
import {
|
||||
ColumnTest,
|
||||
ModifiedTableColumn,
|
||||
} from '../../interface/dataQuality.interface';
|
||||
import {
|
||||
getHtmlForNonAdminAction,
|
||||
getPartialNameFromFQN,
|
||||
@ -50,7 +55,7 @@ import Tags from '../tags/tags';
|
||||
|
||||
type Props = {
|
||||
owner: Table['owner'];
|
||||
tableColumns: Table['columns'];
|
||||
tableColumns: ModifiedTableColumn[];
|
||||
joins: Array<ColumnJoins>;
|
||||
searchText?: string;
|
||||
columnName: string;
|
||||
@ -58,7 +63,7 @@ type Props = {
|
||||
isReadOnly?: boolean;
|
||||
entityFqn?: string;
|
||||
entityFieldThreads?: EntityFieldThreads[];
|
||||
onUpdate?: (columns: Table['columns']) => void;
|
||||
onUpdate?: (columns: ModifiedTableColumn[]) => void;
|
||||
onThreadLinkSelect?: (value: string) => void;
|
||||
onEntityFieldSelect?: (value: string) => void;
|
||||
};
|
||||
@ -86,6 +91,10 @@ const EntityTable = ({
|
||||
Header: 'Type',
|
||||
accessor: 'dataTypeDisplay',
|
||||
},
|
||||
{
|
||||
Header: 'Data Quality',
|
||||
accessor: 'columnTests',
|
||||
},
|
||||
{
|
||||
Header: 'Description',
|
||||
accessor: 'description',
|
||||
@ -98,7 +107,9 @@ const EntityTable = ({
|
||||
[]
|
||||
);
|
||||
|
||||
const [searchedColumns, setSearchedColumns] = useState<Table['columns']>([]);
|
||||
const [searchedColumns, setSearchedColumns] = useState<ModifiedTableColumn[]>(
|
||||
[]
|
||||
);
|
||||
|
||||
const data = React.useMemo(
|
||||
() => makeData(searchedColumns),
|
||||
@ -158,7 +169,7 @@ const EntityTable = ({
|
||||
};
|
||||
|
||||
const updateColumnDescription = (
|
||||
tableCols: Table['columns'],
|
||||
tableCols: ModifiedTableColumn[],
|
||||
changedColName: string,
|
||||
description: string
|
||||
) => {
|
||||
@ -167,7 +178,7 @@ const EntityTable = ({
|
||||
col.description = description;
|
||||
} else {
|
||||
updateColumnDescription(
|
||||
col?.children as Table['columns'],
|
||||
col?.children as ModifiedTableColumn[],
|
||||
changedColName,
|
||||
description
|
||||
);
|
||||
@ -176,7 +187,7 @@ const EntityTable = ({
|
||||
};
|
||||
|
||||
const updateColumnTags = (
|
||||
tableCols: Table['columns'],
|
||||
tableCols: ModifiedTableColumn[],
|
||||
changedColName: string,
|
||||
newColumnTags: Array<string>
|
||||
) => {
|
||||
@ -204,7 +215,7 @@ const EntityTable = ({
|
||||
col.tags = getUpdatedTags(col);
|
||||
} else {
|
||||
updateColumnTags(
|
||||
col?.children as Table['columns'],
|
||||
col?.children as ModifiedTableColumn[],
|
||||
changedColName,
|
||||
newColumnTags
|
||||
);
|
||||
@ -309,7 +320,8 @@ const EntityTable = ({
|
||||
{headerGroup.headers.map((column: any, index: number) => (
|
||||
<th
|
||||
className={classNames('tableHead-cell', {
|
||||
'tw-w-60': column.id === 'tags',
|
||||
'tw-w-60':
|
||||
column.id === 'tags' || column.id === 'columnTests',
|
||||
})}
|
||||
key={index}
|
||||
{...column.getHeaderProps()}>
|
||||
@ -332,6 +344,22 @@ const EntityTable = ({
|
||||
{...row.getRowProps()}>
|
||||
{/* eslint-disable-next-line */}
|
||||
{row.cells.map((cell: any, index: number) => {
|
||||
const columnTests =
|
||||
cell.column.id === 'columnTests'
|
||||
? ((cell.value ?? []) as ColumnTest[])
|
||||
: ([] as ColumnTest[]);
|
||||
const columnTestLength = columnTests.length;
|
||||
const failingTests = columnTests.filter((test) =>
|
||||
test.results?.some(
|
||||
(t) => t.testCaseStatus === TestCaseStatus.Failed
|
||||
)
|
||||
);
|
||||
const passingTests = columnTests.filter((test) =>
|
||||
test.results?.some(
|
||||
(t) => t.testCaseStatus === TestCaseStatus.Success
|
||||
)
|
||||
);
|
||||
|
||||
return (
|
||||
<td
|
||||
className={classNames(
|
||||
@ -354,6 +382,34 @@ const EntityTable = ({
|
||||
</span>
|
||||
) : null}
|
||||
|
||||
{cell.column.id === 'columnTests' && (
|
||||
<Fragment>
|
||||
{columnTestLength ? (
|
||||
<Fragment>
|
||||
{failingTests.length ? (
|
||||
<div className="tw-flex">
|
||||
<p className="tw-mr-2">
|
||||
<i className="fas fa-times tw-text-status-failed" />
|
||||
</p>
|
||||
<p>
|
||||
{`${failingTests.length}/${columnTestLength} tests failing`}
|
||||
</p>
|
||||
</div>
|
||||
) : (
|
||||
<div className="tw-flex">
|
||||
<div className="tw-mr-2">
|
||||
<i className="fas fa-check-square tw-text-status-success" />
|
||||
</div>
|
||||
<p>{`${passingTests.length} tests`}</p>
|
||||
</div>
|
||||
)}
|
||||
</Fragment>
|
||||
) : (
|
||||
'--'
|
||||
)}
|
||||
</Fragment>
|
||||
)}
|
||||
|
||||
{cell.column.id === 'dataTypeDisplay' && (
|
||||
<>
|
||||
{isReadOnly ? (
|
||||
|
@ -12,31 +12,73 @@
|
||||
*/
|
||||
|
||||
import classNames from 'classnames';
|
||||
import React, { Fragment, useState } from 'react';
|
||||
import React, { FC, Fragment } from 'react';
|
||||
import { Link } from 'react-router-dom';
|
||||
import { TITLE_FOR_NON_ADMIN_ACTION } from '../../constants/constants';
|
||||
import { Table, TableProfile } from '../../generated/entity/data/table';
|
||||
import { useAuth } from '../../hooks/authHooks';
|
||||
import {
|
||||
ColumnTest,
|
||||
DatasetTestModeType,
|
||||
} from '../../interface/dataQuality.interface';
|
||||
import { getConstraintIcon } from '../../utils/TableUtils';
|
||||
import TableProfilerGraph from './TableProfilerGraph.component';
|
||||
import { Button } from '../buttons/Button/Button';
|
||||
import NonAdminAction from '../common/non-admin-action/NonAdminAction';
|
||||
import PopOver from '../common/popover/PopOver';
|
||||
import RichTextEditorPreviewer from '../common/rich-text-editor/RichTextEditorPreviewer';
|
||||
|
||||
type Props = {
|
||||
tableProfiles: Table['tableProfile'];
|
||||
columns: Array<{ constraint: string; colName: string }>;
|
||||
columns: Array<{
|
||||
constraint: string;
|
||||
colName: string;
|
||||
colType: string;
|
||||
colTests?: ColumnTest[];
|
||||
}>;
|
||||
qualityTestFormHandler: (
|
||||
tabValue: number,
|
||||
testMode: DatasetTestModeType
|
||||
) => void;
|
||||
};
|
||||
|
||||
type ProfilerGraphData = Array<{
|
||||
date: Date;
|
||||
value: number;
|
||||
}>;
|
||||
const PercentageGraph = ({
|
||||
percentage,
|
||||
title,
|
||||
}: {
|
||||
percentage: number;
|
||||
title: string;
|
||||
}) => {
|
||||
return (
|
||||
<PopOver
|
||||
position="top"
|
||||
title={`${percentage}% ${title}`}
|
||||
trigger="mouseenter">
|
||||
<div className="tw-border tw-border-primary tw-h-8 tw-w-20">
|
||||
<div
|
||||
className="tw-bg-primary tw-opacity-40 tw-h-full"
|
||||
style={{ width: `${percentage}%` }}
|
||||
/>
|
||||
</div>
|
||||
</PopOver>
|
||||
);
|
||||
};
|
||||
|
||||
const TableProfiler = ({ tableProfiles, columns }: Props) => {
|
||||
const [expandedColumn, setExpandedColumn] = useState<{
|
||||
name: string;
|
||||
isExpanded: boolean;
|
||||
}>({
|
||||
name: '',
|
||||
isExpanded: false,
|
||||
});
|
||||
const excludedMetrics = [
|
||||
'profilDate',
|
||||
'name',
|
||||
'nullCount',
|
||||
'nullProportion',
|
||||
'uniqueCount',
|
||||
'uniqueProportion',
|
||||
'rows',
|
||||
];
|
||||
|
||||
const TableProfiler: FC<Props> = ({
|
||||
tableProfiles,
|
||||
columns,
|
||||
qualityTestFormHandler,
|
||||
}) => {
|
||||
const { isAuthDisabled, isAdminUser } = useAuth();
|
||||
const modifiedData = tableProfiles?.map((tableProfile: TableProfile) => ({
|
||||
rows: tableProfile.rowCount,
|
||||
profileDate: tableProfile.profileDate,
|
||||
@ -49,19 +91,27 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => {
|
||||
(colProfile) => colProfile.name === column.colName
|
||||
);
|
||||
|
||||
return { profilDate: md.profileDate, ...currentColumn, rows: md.rows };
|
||||
return {
|
||||
profilDate: md.profileDate,
|
||||
...currentColumn,
|
||||
rows: md.rows,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
name: column,
|
||||
columnMetrics: Object.entries(data?.[0] ?? {}).map((d) => ({
|
||||
key: d[0],
|
||||
value: d[1],
|
||||
})),
|
||||
columnTests: column.colTests,
|
||||
data,
|
||||
min: data?.length ? data[0].min ?? 0 : 0,
|
||||
max: data?.length ? data[0].max ?? 0 : 0,
|
||||
type: column.colType,
|
||||
};
|
||||
});
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="tw-table-responsive tw-overflow-x-auto">
|
||||
{tableProfiles?.length ? (
|
||||
<table
|
||||
className="tw-w-full"
|
||||
@ -70,11 +120,13 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => {
|
||||
<thead>
|
||||
<tr className="tableHead-row">
|
||||
<th className="tableHead-cell">Column Name</th>
|
||||
<th className="tableHead-cell">Distinct Ratio (%)</th>
|
||||
<th className="tableHead-cell">Null Ratio (%)</th>
|
||||
<th className="tableHead-cell">Min</th>
|
||||
<th className="tableHead-cell">Max</th>
|
||||
<th className="tableHead-cell">Standard Deviation</th>
|
||||
<th className="tableHead-cell">Type</th>
|
||||
<th className="tableHead-cell">Null</th>
|
||||
<th className="tableHead-cell">Unique</th>
|
||||
<th className="tableHead-cell">Distinct</th>
|
||||
<th className="tableHead-cell">Metrics</th>
|
||||
<th className="tableHead-cell">Tests</th>
|
||||
<th className="tableHead-cell" />
|
||||
</tr>
|
||||
</thead>
|
||||
{columnSpecificData.map((col, colIndex) => {
|
||||
@ -88,27 +140,6 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => {
|
||||
className="tw-relative tableBody-cell"
|
||||
data-testid="tableBody-cell">
|
||||
<div className="tw-flex">
|
||||
<span
|
||||
className="tw-mr-2 tw-cursor-pointer"
|
||||
onClick={() =>
|
||||
setExpandedColumn((prevState) => ({
|
||||
name: col.name.colName,
|
||||
isExpanded:
|
||||
prevState.name === col.name.colName
|
||||
? !prevState.isExpanded
|
||||
: true,
|
||||
}))
|
||||
}>
|
||||
{expandedColumn.name === col.name.colName ? (
|
||||
expandedColumn.isExpanded ? (
|
||||
<i className="fas fa-caret-down" />
|
||||
) : (
|
||||
<i className="fas fa-caret-right" />
|
||||
)
|
||||
) : (
|
||||
<i className="fas fa-caret-right" />
|
||||
)}
|
||||
</span>
|
||||
{col.name.constraint && (
|
||||
<span className="tw-mr-3 tw--ml-2">
|
||||
{getConstraintIcon(
|
||||
@ -120,84 +151,140 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => {
|
||||
<span>{col.name.colName}</span>
|
||||
</div>
|
||||
</td>
|
||||
<td className="tw-relative tableBody-cell profiler-graph">
|
||||
<TableProfilerGraph
|
||||
data={
|
||||
col.data
|
||||
?.map((d) => ({
|
||||
date: d.profilDate,
|
||||
value: d.uniqueProportion ?? 0,
|
||||
}))
|
||||
.reverse() as ProfilerGraphData
|
||||
<td
|
||||
className="tw-relative tableBody-cell"
|
||||
data-testid="tableBody-cell">
|
||||
<div className="tw-flex">
|
||||
{col.name.colType.length > 25 ? (
|
||||
<span>
|
||||
<PopOver
|
||||
html={
|
||||
<div className="tw-break-words">
|
||||
<span>{col.name.colType.toLowerCase()}</span>
|
||||
</div>
|
||||
}
|
||||
position="bottom"
|
||||
theme="light"
|
||||
trigger="click">
|
||||
<div className="tw-cursor-pointer tw-underline tw-inline-block">
|
||||
<RichTextEditorPreviewer
|
||||
markdown={`${col.name.colType
|
||||
.slice(0, 20)
|
||||
.toLowerCase()}...`}
|
||||
/>
|
||||
</td>
|
||||
<td className="tw-relative tableBody-cell profiler-graph">
|
||||
<TableProfilerGraph
|
||||
data={
|
||||
col.data
|
||||
?.map((d) => ({
|
||||
date: d.profilDate,
|
||||
value: d.nullProportion ?? 0,
|
||||
}))
|
||||
.reverse() as ProfilerGraphData
|
||||
}
|
||||
/>
|
||||
</td>
|
||||
<td className="tw-relative tableBody-cell">{col.min}</td>
|
||||
<td className="tw-relative tableBody-cell">{col.max}</td>
|
||||
<td className="tw-relative tableBody-cell profiler-graph">
|
||||
<TableProfilerGraph
|
||||
data={
|
||||
col.data
|
||||
?.map((d) => ({
|
||||
date: d.profilDate,
|
||||
value: d.stddev ?? 0,
|
||||
}))
|
||||
.reverse() as ProfilerGraphData
|
||||
}
|
||||
/>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
{expandedColumn.name === col.name.colName &&
|
||||
expandedColumn.isExpanded && (
|
||||
<tbody>
|
||||
{col.data?.map((colData, index) => (
|
||||
<tr
|
||||
className={classNames(
|
||||
'tableBody-row tw-border-0 tw-border-l tw-border-r',
|
||||
{
|
||||
'tw-border-b':
|
||||
columnSpecificData.length - 1 === colIndex &&
|
||||
col.data?.length === index + 1,
|
||||
}
|
||||
)}
|
||||
key={index}>
|
||||
<td className="tw-relative tableBody-cell">
|
||||
<span className="tw-pl-6">
|
||||
{colData.profilDate}
|
||||
</div>
|
||||
</PopOver>
|
||||
</span>
|
||||
) : (
|
||||
col.name.colType.toLowerCase()
|
||||
)}
|
||||
</div>
|
||||
</td>
|
||||
<td className="tw-relative tableBody-cell">
|
||||
{colData.uniqueProportion ?? 0}
|
||||
<td className="tw-relative tableBody-cell profiler-graph">
|
||||
<PercentageGraph
|
||||
percentage={(col.data?.[0]?.nullProportion ?? 0) * 100}
|
||||
title="null value"
|
||||
/>
|
||||
</td>
|
||||
<td className="tw-relative tableBody-cell">
|
||||
{colData.nullProportion ?? 0}
|
||||
<td className="tw-relative tableBody-cell profiler-graph">
|
||||
<PercentageGraph
|
||||
percentage={
|
||||
(col.data?.[0]?.uniqueProportion ?? 0) * 100
|
||||
}
|
||||
title="unique value"
|
||||
/>
|
||||
</td>
|
||||
<td className="tw-relative tableBody-cell">
|
||||
{colData.min ?? 0}
|
||||
<td className="tw-relative tableBody-cell profiler-graph">
|
||||
<PercentageGraph
|
||||
percentage={
|
||||
((col.data?.[0]?.distinctCount ?? 0) /
|
||||
(col.data?.[0]?.rows ?? 0)) *
|
||||
100
|
||||
}
|
||||
title="distinct value"
|
||||
/>
|
||||
</td>
|
||||
<td className="tw-relative tableBody-cell">
|
||||
{colData.max ?? 0}
|
||||
<td
|
||||
className="tw-relative tableBody-cell"
|
||||
data-testid="tableBody-cell">
|
||||
<div className="tw-border tw-border-main tw-rounded tw-p-2 tw-min-h-32 tw-max-h-44 tw-overflow-y-auto">
|
||||
{col.columnMetrics
|
||||
.filter((m) => !excludedMetrics.includes(m.key))
|
||||
.map((m, i) => (
|
||||
<p className="tw-mb-1 tw-flex" key={i}>
|
||||
<span className="tw-mx-1">{m.key}</span>
|
||||
<span className="tw-mx-1">-</span>
|
||||
<span className="tw-mx-1">{m.value}</span>
|
||||
</p>
|
||||
))}
|
||||
</div>
|
||||
</td>
|
||||
<td className="tw-relative tableBody-cell">
|
||||
{colData.stddev ?? 0}
|
||||
<td
|
||||
className="tw-relative tableBody-cell"
|
||||
colSpan={2}
|
||||
data-testid="tableBody-cell">
|
||||
<div className="tw-flex tw-justify-between">
|
||||
{col.columnTests ? (
|
||||
<div className="tw-border tw-border-main tw-rounded tw-p-2 tw-min-h-32 tw-max-h-44 tw-overflow-y-auto tw-flex-1">
|
||||
{col.columnTests.map((m, i) => (
|
||||
<div className="tw-flex tw-mb-2" key={i}>
|
||||
<p className="tw-mr-2">
|
||||
{m.results?.every(
|
||||
(result) =>
|
||||
result.testCaseStatus === 'Success'
|
||||
) ? (
|
||||
<i className="fas fa-check-square tw-text-status-success" />
|
||||
) : (
|
||||
<i className="fas fa-times tw-text-status-failed" />
|
||||
)}
|
||||
</p>
|
||||
<div>
|
||||
<span className="tw-mx-1 tw-font-medium">
|
||||
{m.testCase.columnTestType}
|
||||
</span>
|
||||
<div className="tw-mx-1">
|
||||
{Object.entries(
|
||||
m.testCase.config ?? {}
|
||||
).map((config, i) => (
|
||||
<p className="tw-mx-1" key={i}>
|
||||
<span>{config[0]}:</span>
|
||||
<span>{config[1] ?? 'null'}</span>
|
||||
</p>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
`No tests available`
|
||||
)}
|
||||
<div className="tw-self-center tw-ml-5">
|
||||
<NonAdminAction
|
||||
position="bottom"
|
||||
title={TITLE_FOR_NON_ADMIN_ACTION}>
|
||||
<Button
|
||||
className={classNames(
|
||||
'tw-px-2 tw-py-0.5 tw-rounded tw-border-grey-muted',
|
||||
{
|
||||
'tw-opacity-40':
|
||||
!isAdminUser && !isAuthDisabled,
|
||||
}
|
||||
)}
|
||||
size="custom"
|
||||
type="button"
|
||||
variant="outlined"
|
||||
onClick={() =>
|
||||
qualityTestFormHandler(6, 'column')
|
||||
}>
|
||||
Add Test
|
||||
</Button>
|
||||
</NonAdminAction>
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
)}
|
||||
</Fragment>
|
||||
);
|
||||
})}
|
||||
@ -218,7 +305,7 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => {
|
||||
</Link>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
|
@ -15,86 +15,18 @@
|
||||
|
||||
export const mockFeedData = [
|
||||
{
|
||||
index: 'table_search_index',
|
||||
id: 'd336d794-6fce-4094-a064-7ceb3b208dab',
|
||||
name: 'dim_address',
|
||||
description:
|
||||
'This dimension table contains the billing and shipping addresses of customers. You can join this table with the sales table to generate lists of the billing and shipping addresses. Customers can enter their addresses more than once, so the same address can appear in more than one row in this table. This table contains one row per customer address.',
|
||||
fullyQualifiedName: 'bigquery_gcp.shopify.dim_address',
|
||||
tableType: 'Regular',
|
||||
tags: [
|
||||
'PersonalData.Personal',
|
||||
'PII.Sensitive',
|
||||
'PII.NonSensitive',
|
||||
'PII.Sensitive',
|
||||
'PersonalData.Personal',
|
||||
'User.FacePhoto',
|
||||
],
|
||||
service: 'bigquery_gcp',
|
||||
serviceType: 'BigQuery',
|
||||
database: 'shopify',
|
||||
entityType: 'table',
|
||||
changeDescriptions: [
|
||||
{
|
||||
id: '2b133a6d-8562-4220-a997-eaaa0376cad3',
|
||||
href: 'http://localhost:8585/api/v1/feed/2b133a6d-8562-4220-a997-eaaa0376cad3',
|
||||
threadTs: 1646577496489,
|
||||
about: '<#E/table/bigquery_gcp.shopify.dim_staff/description>',
|
||||
entityId: '37e50e09-d5b6-4609-821c-1ed84d92dd57',
|
||||
createdBy: 'aaron_johnson0',
|
||||
updatedAt: 1646577496489,
|
||||
updatedBy: 'anonymous',
|
||||
updatedAt: 1640239422867,
|
||||
fieldsAdded: [],
|
||||
fieldsUpdated: [],
|
||||
fieldsDeleted: [],
|
||||
},
|
||||
{
|
||||
updatedBy: 'anonymous',
|
||||
fieldsAdded: [
|
||||
{
|
||||
newValue:
|
||||
'[{"tagFQN":"PersonalData.Personal","labelType":"Manual","state":"Confirmed"}]',
|
||||
name: 'columns.address_id.tags',
|
||||
},
|
||||
],
|
||||
fieldsUpdated: [],
|
||||
fieldsDeleted: [],
|
||||
updatedAt: 1640247571788,
|
||||
},
|
||||
{
|
||||
updatedBy: 'anonymous',
|
||||
fieldsAdded: [
|
||||
{
|
||||
newValue:
|
||||
'[{"tagFQN":"PII.Sensitive","labelType":"Manual","state":"Confirmed"}]',
|
||||
name: 'columns.shop_id.tags',
|
||||
},
|
||||
],
|
||||
fieldsUpdated: [],
|
||||
fieldsDeleted: [],
|
||||
updatedAt: 1640248828684,
|
||||
},
|
||||
{
|
||||
updatedBy: 'anonymous',
|
||||
fieldsAdded: [
|
||||
{
|
||||
newValue:
|
||||
'[{"tagFQN":"PII.NonSensitive","labelType":"Manual","state":"Confirmed"}]',
|
||||
name: 'columns.first_name.tags',
|
||||
},
|
||||
],
|
||||
fieldsUpdated: [],
|
||||
fieldsDeleted: [],
|
||||
updatedAt: 1640248877069,
|
||||
},
|
||||
{
|
||||
updatedBy: 'anonymous',
|
||||
fieldsAdded: [
|
||||
{
|
||||
newValue:
|
||||
'[{"tagFQN":"PII.Sensitive","description":"PII which if lost, compromised, or disclosed without authorization, could result in substantial harm, embarrassment, inconvenience, or unfairness to an individual.","labelType":"Derived","state":"Confirmed"},{"tagFQN":"PersonalData.Personal","description":"Data that can be used to directly or indirectly identify a person.","labelType":"Derived","state":"Confirmed"},{"tagFQN":"User.FacePhoto","labelType":"Manual","state":"Confirmed"}]',
|
||||
name: 'columns.last_name.tags',
|
||||
},
|
||||
],
|
||||
fieldsUpdated: [],
|
||||
fieldsDeleted: [],
|
||||
updatedAt: 1640248886960,
|
||||
},
|
||||
],
|
||||
resolved: false,
|
||||
message: 'Can you add a description?',
|
||||
postsCount: 0,
|
||||
posts: [],
|
||||
},
|
||||
];
|
||||
|
||||
|
@ -628,6 +628,10 @@ export interface ColumnProfile {
|
||||
* Variance of a column.
|
||||
*/
|
||||
variance?: number;
|
||||
|
||||
minLength?: number;
|
||||
|
||||
maxLength?: number;
|
||||
}
|
||||
|
||||
export interface HistogramObject {
|
||||
|
@ -30,12 +30,19 @@ export interface TestCaseConfigType {
|
||||
missingValueMatch?: string;
|
||||
}
|
||||
|
||||
export interface CreateColumnTest {
|
||||
export interface Result {
|
||||
executionTime: number;
|
||||
testCaseStatus: string;
|
||||
result: string;
|
||||
}
|
||||
|
||||
export interface ColumnTest {
|
||||
id?: string;
|
||||
columnName: string;
|
||||
description?: string;
|
||||
executionFrequency?: TestCaseExecutionFrequency;
|
||||
owner?: EntityReference;
|
||||
results?: Result[];
|
||||
testCase: {
|
||||
columnTestType: ColumnTestType;
|
||||
config?: TestCaseConfigType;
|
||||
@ -45,7 +52,7 @@ export interface CreateColumnTest {
|
||||
export type DatasetTestModeType = 'table' | 'column';
|
||||
|
||||
export interface ModifiedTableColumn extends Column {
|
||||
columnTests?: CreateColumnTest[];
|
||||
columnTests?: ColumnTest[];
|
||||
}
|
||||
|
||||
export interface TableTestDataType {
|
||||
|
@ -77,7 +77,7 @@ import { EntityLineage } from '../../generated/type/entityLineage';
|
||||
import { TagLabel } from '../../generated/type/tagLabel';
|
||||
import useToastContext from '../../hooks/useToastContext';
|
||||
import {
|
||||
CreateColumnTest,
|
||||
ColumnTest,
|
||||
DatasetTestModeType,
|
||||
ModifiedTableColumn,
|
||||
} from '../../interface/dataQuality.interface';
|
||||
@ -118,7 +118,7 @@ const DatasetDetailsPage: FunctionComponent = () => {
|
||||
TitleBreadcrumbProps['titleLinks']
|
||||
>([]);
|
||||
const [description, setDescription] = useState('');
|
||||
const [columns, setColumns] = useState<Table['columns']>([]);
|
||||
const [columns, setColumns] = useState<ModifiedTableColumn[]>([]);
|
||||
const [sampleData, setSampleData] = useState<TableData>({
|
||||
columns: [],
|
||||
rows: [],
|
||||
@ -191,6 +191,15 @@ const DatasetDetailsPage: FunctionComponent = () => {
|
||||
}
|
||||
};
|
||||
|
||||
const qualityTestFormHandler = (
|
||||
tabValue: number,
|
||||
testMode: DatasetTestModeType
|
||||
) => {
|
||||
activeTabHandler(tabValue);
|
||||
setTestMode(testMode);
|
||||
setShowTestForm(true);
|
||||
};
|
||||
|
||||
const getLineageData = () => {
|
||||
setIsLineageLoading(true);
|
||||
getLineageByFQN(tableFQN, EntityType.TABLE)
|
||||
@ -647,7 +656,7 @@ const DatasetDetailsPage: FunctionComponent = () => {
|
||||
});
|
||||
};
|
||||
|
||||
const handleAddColumnTestCase = (data: CreateColumnTest) => {
|
||||
const handleAddColumnTestCase = (data: ColumnTest) => {
|
||||
addColumnTestCase(tableDetails.id, data)
|
||||
.then((res: AxiosResponse) => {
|
||||
const columnTestRes = res.data.columns.find(
|
||||
@ -786,6 +795,7 @@ const DatasetDetailsPage: FunctionComponent = () => {
|
||||
loadNodeHandler={loadNodeHandler}
|
||||
owner={owner as Table['owner'] & { displayName: string }}
|
||||
postFeedHandler={postFeedHandler}
|
||||
qualityTestFormHandler={qualityTestFormHandler}
|
||||
removeLineageHandler={removeLineageHandler}
|
||||
sampleData={sampleData}
|
||||
setActiveTabHandler={activeTabHandler}
|
||||
|
@ -127,7 +127,7 @@ const TourPage = () => {
|
||||
pipelineCount: 8,
|
||||
}}
|
||||
error=""
|
||||
feedData={mockFeedData as unknown as MyDataProps['feedData']}
|
||||
feedData={mockFeedData as MyDataProps['feedData']}
|
||||
feedFilter={FeedFilter.ALL}
|
||||
feedFilterHandler={() => {
|
||||
setMyDataSearchResult(mockData);
|
||||
@ -185,7 +185,7 @@ const TourPage = () => {
|
||||
entityLineage={mockDatasetData.entityLineage}
|
||||
entityLineageHandler={handleCountChange}
|
||||
entityName={mockDatasetData.entityName}
|
||||
entityThread={[]}
|
||||
entityThread={mockFeedData}
|
||||
feedCount={0}
|
||||
followTableHandler={handleCountChange}
|
||||
followers={mockDatasetData.followers}
|
||||
@ -205,6 +205,7 @@ const TourPage = () => {
|
||||
loadNodeHandler={handleCountChange}
|
||||
owner={undefined as unknown as DatasetOwner}
|
||||
postFeedHandler={handleCountChange}
|
||||
qualityTestFormHandler={handleCountChange}
|
||||
removeLineageHandler={handleCountChange}
|
||||
sampleData={mockDatasetData.sampleData}
|
||||
setActiveTabHandler={(tab) => setdatasetActiveTab(tab)}
|
||||
|
@ -14,7 +14,7 @@
|
||||
import classNames from 'classnames';
|
||||
import { upperCase } from 'lodash';
|
||||
import { EntityTags, TableDetail } from 'Models';
|
||||
import React from 'react';
|
||||
import React, { Fragment } from 'react';
|
||||
import AppState from '../AppState';
|
||||
import PopOver from '../components/common/popover/PopOver';
|
||||
import {
|
||||
@ -27,8 +27,10 @@ import {
|
||||
import { EntityType } from '../enums/entity.enum';
|
||||
import { SearchIndex } from '../enums/search.enum';
|
||||
import { ConstraintTypes } from '../enums/table.enum';
|
||||
import { Column, DataType, Table } from '../generated/entity/data/table';
|
||||
import { Column, DataType } from '../generated/entity/data/table';
|
||||
import { TableTest, TestCaseStatus } from '../generated/tests/tableTest';
|
||||
import { TagLabel } from '../generated/type/tagLabel';
|
||||
import { ModifiedTableColumn } from '../interface/dataQuality.interface';
|
||||
import { ordinalize } from './StringsUtils';
|
||||
import SVGIcons from './SvgUtils';
|
||||
|
||||
@ -253,7 +255,7 @@ export const makeRow = (column: Column) => {
|
||||
};
|
||||
|
||||
export const makeData = (
|
||||
columns: Table['columns'] = []
|
||||
columns: ModifiedTableColumn[] = []
|
||||
): Array<Column & { subRows: Column[] | undefined }> => {
|
||||
const data = columns.map((column) => ({
|
||||
...makeRow(column),
|
||||
@ -290,3 +292,38 @@ export const getDataTypeString = (dataType: string): string => {
|
||||
return dataType;
|
||||
}
|
||||
};
|
||||
|
||||
export const getTableTestsValue = (tableTestCase: TableTest[]) => {
|
||||
const tableTestLength = tableTestCase.length;
|
||||
|
||||
const failingTests = tableTestCase.filter((test) =>
|
||||
test.results?.some((t) => t.testCaseStatus === TestCaseStatus.Failed)
|
||||
);
|
||||
const passingTests = tableTestCase.filter((test) =>
|
||||
test.results?.some((t) => t.testCaseStatus === TestCaseStatus.Success)
|
||||
);
|
||||
|
||||
return (
|
||||
<Fragment>
|
||||
{tableTestLength ? (
|
||||
<Fragment>
|
||||
{failingTests.length ? (
|
||||
<div className="tw-flex">
|
||||
<p className="tw-mr-2">
|
||||
<i className="fas fa-times tw-text-status-failed" />
|
||||
</p>
|
||||
<p>{`${failingTests.length}/${tableTestLength} tests failing`}</p>
|
||||
</div>
|
||||
) : (
|
||||
<div className="tw-flex">
|
||||
<div className="tw-mr-2">
|
||||
<i className="fas fa-check-square tw-text-status-success" />
|
||||
</div>
|
||||
<p>{`${passingTests.length} tests`}</p>
|
||||
</div>
|
||||
)}
|
||||
</Fragment>
|
||||
) : null}
|
||||
</Fragment>
|
||||
);
|
||||
};
|
||||
|
Loading…
x
Reference in New Issue
Block a user