) => {
@@ -204,7 +215,7 @@ const EntityTable = ({
col.tags = getUpdatedTags(col);
} else {
updateColumnTags(
- col?.children as Table['columns'],
+ col?.children as ModifiedTableColumn[],
changedColName,
newColumnTags
);
@@ -309,7 +320,8 @@ const EntityTable = ({
{headerGroup.headers.map((column: any, index: number) => (
|
@@ -332,6 +344,22 @@ const EntityTable = ({
{...row.getRowProps()}>
{/* eslint-disable-next-line */}
{row.cells.map((cell: any, index: number) => {
+ const columnTests =
+ cell.column.id === 'columnTests'
+ ? ((cell.value ?? []) as ColumnTest[])
+ : ([] as ColumnTest[]);
+ const columnTestLength = columnTests.length;
+ const failingTests = columnTests.filter((test) =>
+ test.results?.some(
+ (t) => t.testCaseStatus === TestCaseStatus.Failed
+ )
+ );
+ const passingTests = columnTests.filter((test) =>
+ test.results?.some(
+ (t) => t.testCaseStatus === TestCaseStatus.Success
+ )
+ );
+
return (
|
) : null}
+ {cell.column.id === 'columnTests' && (
+
+ {columnTestLength ? (
+
+ {failingTests.length ? (
+
+
+
+
+
+ {`${failingTests.length}/${columnTestLength} tests failing`}
+
+
+
+
+
+ {`${passingTests.length} tests`}
+
{tableProfiles?.length ? (
{
| Column Name |
- Distinct Ratio (%) |
- Null Ratio (%) |
- Min |
- Max |
- Standard Deviation |
+ Type |
+ Null |
+ Unique |
+ Distinct |
+ Metrics |
+ Tests |
+ |
{columnSpecificData.map((col, colIndex) => {
@@ -88,27 +140,6 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => {
className="tw-relative tableBody-cell"
data-testid="tableBody-cell">
-
- setExpandedColumn((prevState) => ({
- name: col.name.colName,
- isExpanded:
- prevState.name === col.name.colName
- ? !prevState.isExpanded
- : true,
- }))
- }>
- {expandedColumn.name === col.name.colName ? (
- expandedColumn.isExpanded ? (
-
- ) : (
-
- )
- ) : (
-
- )}
-
{col.name.constraint && (
{getConstraintIcon(
@@ -120,84 +151,140 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => {
{col.name.colName}
+
+
+ {col.name.colType.length > 25 ? (
+
+
+ {col.name.colType.toLowerCase()}
+
+
+
+
+
+ ) : (
+ col.name.colType.toLowerCase()
+ )}
+
+ |
- ({
- date: d.profilDate,
- value: d.uniqueProportion ?? 0,
- }))
- .reverse() as ProfilerGraphData
- }
+
|
- ({
- date: d.profilDate,
- value: d.nullProportion ?? 0,
- }))
- .reverse() as ProfilerGraphData
+
|
- {col.min} |
- {col.max} |
- ({
- date: d.profilDate,
- value: d.stddev ?? 0,
- }))
- .reverse() as ProfilerGraphData
+
|
+
+
+ {col.columnMetrics
+ .filter((m) => !excludedMetrics.includes(m.key))
+ .map((m, i) => (
+
+ {m.key}
+ -
+ {m.value}
+
+ ))}
+ |
+
+
+ {col.columnTests ? (
+
+ {col.columnTests.map((m, i) => (
+
+
+ {m.results?.every(
+ (result) =>
+ result.testCaseStatus === 'Success'
+ ) ? (
+
+ ) : (
+
+ )}
+
+
+
+ {m.testCase.columnTestType}
+
+
+ {Object.entries(
+ m.testCase.config ?? {}
+ ).map((config, i) => (
+
+ {config[0]}:
+ {config[1] ?? 'null'}
+
+ ))}
+
+
+
+ ))}
+
+ ) : (
+ `No tests available`
+ )}
+
+
+
+
+
+ |
- {expandedColumn.name === col.name.colName &&
- expandedColumn.isExpanded && (
-
- {col.data?.map((colData, index) => (
-
- |
-
- {colData.profilDate}
-
- |
-
- {colData.uniqueProportion ?? 0}
- |
-
- {colData.nullProportion ?? 0}
- |
-
- {colData.min ?? 0}
- |
-
- {colData.max ?? 0}
- |
-
- {colData.stddev ?? 0}
- |
-
- ))}
-
- )}
);
})}
@@ -218,7 +305,7 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => {
)}
-
+
);
};
diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/mockTourData.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/mockTourData.constants.ts
index a0f3f2f9406..7e40105fdd2 100644
--- a/openmetadata-ui/src/main/resources/ui/src/constants/mockTourData.constants.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/constants/mockTourData.constants.ts
@@ -15,86 +15,18 @@
export const mockFeedData = [
{
- index: 'table_search_index',
- id: 'd336d794-6fce-4094-a064-7ceb3b208dab',
- name: 'dim_address',
- description:
- 'This dimension table contains the billing and shipping addresses of customers. You can join this table with the sales table to generate lists of the billing and shipping addresses. Customers can enter their addresses more than once, so the same address can appear in more than one row in this table. This table contains one row per customer address.',
- fullyQualifiedName: 'bigquery_gcp.shopify.dim_address',
- tableType: 'Regular',
- tags: [
- 'PersonalData.Personal',
- 'PII.Sensitive',
- 'PII.NonSensitive',
- 'PII.Sensitive',
- 'PersonalData.Personal',
- 'User.FacePhoto',
- ],
- service: 'bigquery_gcp',
- serviceType: 'BigQuery',
- database: 'shopify',
- entityType: 'table',
- changeDescriptions: [
- {
- updatedBy: 'anonymous',
- updatedAt: 1640239422867,
- fieldsAdded: [],
- fieldsUpdated: [],
- fieldsDeleted: [],
- },
- {
- updatedBy: 'anonymous',
- fieldsAdded: [
- {
- newValue:
- '[{"tagFQN":"PersonalData.Personal","labelType":"Manual","state":"Confirmed"}]',
- name: 'columns.address_id.tags',
- },
- ],
- fieldsUpdated: [],
- fieldsDeleted: [],
- updatedAt: 1640247571788,
- },
- {
- updatedBy: 'anonymous',
- fieldsAdded: [
- {
- newValue:
- '[{"tagFQN":"PII.Sensitive","labelType":"Manual","state":"Confirmed"}]',
- name: 'columns.shop_id.tags',
- },
- ],
- fieldsUpdated: [],
- fieldsDeleted: [],
- updatedAt: 1640248828684,
- },
- {
- updatedBy: 'anonymous',
- fieldsAdded: [
- {
- newValue:
- '[{"tagFQN":"PII.NonSensitive","labelType":"Manual","state":"Confirmed"}]',
- name: 'columns.first_name.tags',
- },
- ],
- fieldsUpdated: [],
- fieldsDeleted: [],
- updatedAt: 1640248877069,
- },
- {
- updatedBy: 'anonymous',
- fieldsAdded: [
- {
- newValue:
- '[{"tagFQN":"PII.Sensitive","description":"PII which if lost, compromised, or disclosed without authorization, could result in substantial harm, embarrassment, inconvenience, or unfairness to an individual.","labelType":"Derived","state":"Confirmed"},{"tagFQN":"PersonalData.Personal","description":"Data that can be used to directly or indirectly identify a person.","labelType":"Derived","state":"Confirmed"},{"tagFQN":"User.FacePhoto","labelType":"Manual","state":"Confirmed"}]',
- name: 'columns.last_name.tags',
- },
- ],
- fieldsUpdated: [],
- fieldsDeleted: [],
- updatedAt: 1640248886960,
- },
- ],
+ id: '2b133a6d-8562-4220-a997-eaaa0376cad3',
+ href: 'http://localhost:8585/api/v1/feed/2b133a6d-8562-4220-a997-eaaa0376cad3',
+ threadTs: 1646577496489,
+ about: '<#E/table/bigquery_gcp.shopify.dim_staff/description>',
+ entityId: '37e50e09-d5b6-4609-821c-1ed84d92dd57',
+ createdBy: 'aaron_johnson0',
+ updatedAt: 1646577496489,
+ updatedBy: 'anonymous',
+ resolved: false,
+ message: 'Can you add a description?',
+ postsCount: 0,
+ posts: [],
},
];
diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts
index 7829ea9b6f7..a68a70b94ed 100644
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts
@@ -628,6 +628,10 @@ export interface ColumnProfile {
* Variance of a column.
*/
variance?: number;
+
+ minLength?: number;
+
+ maxLength?: number;
}
export interface HistogramObject {
diff --git a/openmetadata-ui/src/main/resources/ui/src/interface/dataQuality.interface.ts b/openmetadata-ui/src/main/resources/ui/src/interface/dataQuality.interface.ts
index 3f28d11b86b..192c7201816 100644
--- a/openmetadata-ui/src/main/resources/ui/src/interface/dataQuality.interface.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/interface/dataQuality.interface.ts
@@ -30,12 +30,19 @@ export interface TestCaseConfigType {
missingValueMatch?: string;
}
-export interface CreateColumnTest {
+export interface Result {
+ executionTime: number;
+ testCaseStatus: string;
+ result: string;
+}
+
+export interface ColumnTest {
id?: string;
columnName: string;
description?: string;
executionFrequency?: TestCaseExecutionFrequency;
owner?: EntityReference;
+ results?: Result[];
testCase: {
columnTestType: ColumnTestType;
config?: TestCaseConfigType;
@@ -45,7 +52,7 @@ export interface CreateColumnTest {
export type DatasetTestModeType = 'table' | 'column';
export interface ModifiedTableColumn extends Column {
- columnTests?: CreateColumnTest[];
+ columnTests?: ColumnTest[];
}
export interface TableTestDataType {
diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/DatasetDetailsPage/DatasetDetailsPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/DatasetDetailsPage/DatasetDetailsPage.component.tsx
index 088a8cbd7aa..f4cb0614ff6 100644
--- a/openmetadata-ui/src/main/resources/ui/src/pages/DatasetDetailsPage/DatasetDetailsPage.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/pages/DatasetDetailsPage/DatasetDetailsPage.component.tsx
@@ -77,7 +77,7 @@ import { EntityLineage } from '../../generated/type/entityLineage';
import { TagLabel } from '../../generated/type/tagLabel';
import useToastContext from '../../hooks/useToastContext';
import {
- CreateColumnTest,
+ ColumnTest,
DatasetTestModeType,
ModifiedTableColumn,
} from '../../interface/dataQuality.interface';
@@ -118,7 +118,7 @@ const DatasetDetailsPage: FunctionComponent = () => {
TitleBreadcrumbProps['titleLinks']
>([]);
const [description, setDescription] = useState('');
- const [columns, setColumns] = useState([]);
+ const [columns, setColumns] = useState([]);
const [sampleData, setSampleData] = useState({
columns: [],
rows: [],
@@ -191,6 +191,15 @@ const DatasetDetailsPage: FunctionComponent = () => {
}
};
+ const qualityTestFormHandler = (
+ tabValue: number,
+ testMode: DatasetTestModeType
+ ) => {
+ activeTabHandler(tabValue);
+ setTestMode(testMode);
+ setShowTestForm(true);
+ };
+
const getLineageData = () => {
setIsLineageLoading(true);
getLineageByFQN(tableFQN, EntityType.TABLE)
@@ -647,7 +656,7 @@ const DatasetDetailsPage: FunctionComponent = () => {
});
};
- const handleAddColumnTestCase = (data: CreateColumnTest) => {
+ const handleAddColumnTestCase = (data: ColumnTest) => {
addColumnTestCase(tableDetails.id, data)
.then((res: AxiosResponse) => {
const columnTestRes = res.data.columns.find(
@@ -786,6 +795,7 @@ const DatasetDetailsPage: FunctionComponent = () => {
loadNodeHandler={loadNodeHandler}
owner={owner as Table['owner'] & { displayName: string }}
postFeedHandler={postFeedHandler}
+ qualityTestFormHandler={qualityTestFormHandler}
removeLineageHandler={removeLineageHandler}
sampleData={sampleData}
setActiveTabHandler={activeTabHandler}
diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/tour-page/TourPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/tour-page/TourPage.component.tsx
index 1126934daf8..ccb4a780a86 100644
--- a/openmetadata-ui/src/main/resources/ui/src/pages/tour-page/TourPage.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/pages/tour-page/TourPage.component.tsx
@@ -127,7 +127,7 @@ const TourPage = () => {
pipelineCount: 8,
}}
error=""
- feedData={mockFeedData as unknown as MyDataProps['feedData']}
+ feedData={mockFeedData as MyDataProps['feedData']}
feedFilter={FeedFilter.ALL}
feedFilterHandler={() => {
setMyDataSearchResult(mockData);
@@ -185,7 +185,7 @@ const TourPage = () => {
entityLineage={mockDatasetData.entityLineage}
entityLineageHandler={handleCountChange}
entityName={mockDatasetData.entityName}
- entityThread={[]}
+ entityThread={mockFeedData}
feedCount={0}
followTableHandler={handleCountChange}
followers={mockDatasetData.followers}
@@ -205,6 +205,7 @@ const TourPage = () => {
loadNodeHandler={handleCountChange}
owner={undefined as unknown as DatasetOwner}
postFeedHandler={handleCountChange}
+ qualityTestFormHandler={handleCountChange}
removeLineageHandler={handleCountChange}
sampleData={mockDatasetData.sampleData}
setActiveTabHandler={(tab) => setdatasetActiveTab(tab)}
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx
index 84e8ac0a282..dd886d537ec 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx
@@ -14,7 +14,7 @@
import classNames from 'classnames';
import { upperCase } from 'lodash';
import { EntityTags, TableDetail } from 'Models';
-import React from 'react';
+import React, { Fragment } from 'react';
import AppState from '../AppState';
import PopOver from '../components/common/popover/PopOver';
import {
@@ -27,8 +27,10 @@ import {
import { EntityType } from '../enums/entity.enum';
import { SearchIndex } from '../enums/search.enum';
import { ConstraintTypes } from '../enums/table.enum';
-import { Column, DataType, Table } from '../generated/entity/data/table';
+import { Column, DataType } from '../generated/entity/data/table';
+import { TableTest, TestCaseStatus } from '../generated/tests/tableTest';
import { TagLabel } from '../generated/type/tagLabel';
+import { ModifiedTableColumn } from '../interface/dataQuality.interface';
import { ordinalize } from './StringsUtils';
import SVGIcons from './SvgUtils';
@@ -253,7 +255,7 @@ export const makeRow = (column: Column) => {
};
export const makeData = (
- columns: Table['columns'] = []
+ columns: ModifiedTableColumn[] = []
): Array => {
const data = columns.map((column) => ({
...makeRow(column),
@@ -290,3 +292,38 @@ export const getDataTypeString = (dataType: string): string => {
return dataType;
}
};
+
+export const getTableTestsValue = (tableTestCase: TableTest[]) => {
+ const tableTestLength = tableTestCase.length;
+
+ const failingTests = tableTestCase.filter((test) =>
+ test.results?.some((t) => t.testCaseStatus === TestCaseStatus.Failed)
+ );
+ const passingTests = tableTestCase.filter((test) =>
+ test.results?.some((t) => t.testCaseStatus === TestCaseStatus.Success)
+ );
+
+ return (
+
+ {tableTestLength ? (
+
+ {failingTests.length ? (
+
+
+
+
+ {`${failingTests.length}/${tableTestLength} tests failing`}
+
+
+
+
+ {`${passingTests.length} tests`}
+ |