From 40b64d2d3e87218771ad3d945620b18a39f29027 Mon Sep 17 00:00:00 2001 From: Sachin Chaurasiya Date: Sun, 6 Mar 2022 22:16:21 +0530 Subject: [PATCH] UI : New Data profiler Layout. (#3184) * UI : New Data profiler Layout. * Add tests column in data profiler * Add percentage graph * Add data quality column * Adding table level tests * Minor change * Change mock data for feed * Addressing review comments --- .../resources/ui/src/axiosAPIs/tableAPI.ts | 4 +- .../AddDataQualityTest/AddDataQualityTest.tsx | 6 +- .../Forms/ColumnTestForm.tsx | 10 +- .../DataQualityTab/DataQualityTab.tsx | 6 +- .../DatasetDetails.component.tsx | 11 +- .../DatasetDetails.interface.ts | 11 +- .../DatasetDetails/DatasetDetails.test.tsx | 1 + .../EntityTable/EntityTable.component.tsx | 72 +++- .../TableProfiler/TableProfiler.component.tsx | 309 +++++++++++------- .../src/constants/mockTourData.constants.ts | 92 +----- .../ui/src/generated/entity/data/table.ts | 4 + .../ui/src/interface/dataQuality.interface.ts | 11 +- .../DatasetDetailsPage.component.tsx | 16 +- .../pages/tour-page/TourPage.component.tsx | 5 +- .../resources/ui/src/utils/TableUtils.tsx | 43 ++- 15 files changed, 375 insertions(+), 226 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/tableAPI.ts b/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/tableAPI.ts index 0ab04df751f..cdac8829396 100644 --- a/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/tableAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/tableAPI.ts @@ -16,7 +16,7 @@ import { Table } from 'Models'; import { ColumnTestType } from '../enums/columnTest.enum'; import { CreateTableTest } from '../generated/api/tests/createTableTest'; import { TableTestType } from '../generated/tests/tableTest'; -import { CreateColumnTest } from '../interface/dataQuality.interface'; +import { ColumnTest } from '../interface/dataQuality.interface'; import { getURLWithQueryFields } from '../utils/APIUtils'; import APIClient from './index'; @@ -138,7 +138,7 @@ export const deleteTableTestCase = ( ); }; -export const addColumnTestCase = (tableId: string, data: CreateColumnTest) => { +export const addColumnTestCase = (tableId: string, data: ColumnTest) => { const configOptions = { headers: { 'Content-type': 'application/json' }, }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTest.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTest.tsx index d327022a9f8..8c4281887d8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTest.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/AddDataQualityTest.tsx @@ -16,7 +16,7 @@ import { CreateTableTest } from '../../generated/api/tests/createTableTest'; import { Table } from '../../generated/entity/data/table'; import { TableTest } from '../../generated/tests/tableTest'; import { - CreateColumnTest, + ColumnTest, TableTestDataType, } from '../../interface/dataQuality.interface'; import ColumnTestForm from './Forms/ColumnTestForm'; @@ -28,7 +28,7 @@ type Props = { columnOptions: Table['columns']; tableTestCase: TableTest[]; handleAddTableTestCase: (data: CreateTableTest) => void; - handleAddColumnTestCase: (data: CreateColumnTest) => void; + handleAddColumnTestCase: (data: ColumnTest) => void; onFormCancel: () => void; }; @@ -53,7 +53,7 @@ const AddDataQualityTest = ({ ) : ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/ColumnTestForm.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/ColumnTestForm.tsx index b444928d9ac..87da488e976 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/ColumnTestForm.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddDataQualityTest/Forms/ColumnTestForm.tsx @@ -19,7 +19,7 @@ import { ColumnTestType } from '../../../enums/columnTest.enum'; import { TestCaseExecutionFrequency } from '../../../generated/api/tests/createTableTest'; import { Table } from '../../../generated/entity/data/table'; import { - CreateColumnTest, + ColumnTest, ModifiedTableColumn, } from '../../../interface/dataQuality.interface'; import { @@ -33,9 +33,9 @@ import { Button } from '../../buttons/Button/Button'; import MarkdownWithPreview from '../../common/editor/MarkdownWithPreview'; type Props = { - data: CreateColumnTest; + data: ColumnTest; column: ModifiedTableColumn[]; - handleAddColumnTestCase: (data: CreateColumnTest) => void; + handleAddColumnTestCase: (data: ColumnTest) => void; onFormCancel: () => void; }; @@ -118,7 +118,7 @@ const ColumnTestForm = ({ const selectedColumn = column.find((d) => d.name === name); const existingTests = selectedColumn?.columnTests?.map( - (d: CreateColumnTest) => d.testCase.columnTestType + (d: ColumnTest) => d.testCase.columnTestType ) || []; if (existingTests.length) { const newTest = Object.values(ColumnTestType).filter( @@ -222,7 +222,7 @@ const ColumnTestForm = ({ const handleSave = () => { if (validateForm()) { - const columnTestObj: CreateColumnTest = { + const columnTestObj: ColumnTest = { columnName: columnName, description: markdownRef.current?.getEditorContent() || undefined, executionFrequency: frequency, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQualityTab/DataQualityTab.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQualityTab/DataQualityTab.tsx index 1ac4cc22985..0f1af78b69b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQualityTab/DataQualityTab.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQualityTab/DataQualityTab.tsx @@ -17,7 +17,7 @@ import { CreateTableTest } from '../../generated/api/tests/createTableTest'; import { Table } from '../../generated/entity/data/table'; import { TableTest, TableTestType } from '../../generated/tests/tableTest'; import { - CreateColumnTest, + ColumnTest, DatasetTestModeType, ModifiedTableColumn, TableTestDataType, @@ -27,7 +27,7 @@ import DataQualityTest from '../DataQualityTest/DataQualityTest'; type Props = { handleAddTableTestCase: (data: CreateTableTest) => void; - handleAddColumnTestCase: (data: CreateColumnTest) => void; + handleAddColumnTestCase: (data: ColumnTest) => void; columnOptions: Table['columns']; testMode: DatasetTestModeType; handleTestModeChange: (mode: DatasetTestModeType) => void; @@ -93,7 +93,7 @@ const DataQualityTab = ({ setActiveData(undefined); }; - const onColumnTestSave = (data: CreateColumnTest) => { + const onColumnTestSave = (data: ColumnTest) => { handleAddColumnTestCase(data); setActiveData(undefined); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.component.tsx index 4e2f0bba00f..7e3d31a712e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.component.tsx @@ -36,7 +36,11 @@ import { import { getEntityFeedLink } from '../../utils/EntityUtils'; import { getDefaultValue } from '../../utils/FeedElementUtils'; import { getEntityFieldThreadCounts } from '../../utils/FeedUtils'; -import { getTagsWithoutTier, getUsagePercentile } from '../../utils/TableUtils'; +import { + getTableTestsValue, + getTagsWithoutTier, + getUsagePercentile, +} from '../../utils/TableUtils'; import ActivityFeedList from '../ActivityFeed/ActivityFeedList/ActivityFeedList'; import ActivityThreadPanel from '../ActivityFeed/ActivityThreadPanel/ActivityThreadPanel'; import Description from '../common/description/Description'; @@ -111,6 +115,7 @@ const DatasetDetails: React.FC = ({ handleShowTestForm, handleRemoveTableTest, handleRemoveColumnTest, + qualityTestFormHandler, }: DatasetDetailsProps) => { const { isAuthDisabled } = useAuth(); const [isEdit, setIsEdit] = useState(false); @@ -350,6 +355,7 @@ const DatasetDetails: React.FC = ({ '' ), }, + { key: 'Tests', value: getTableTestsValue(tableTestCase) }, ]; const onDescriptionEdit = (): void => { @@ -616,7 +622,10 @@ const DatasetDetails: React.FC = ({ columns={columns.map((col) => ({ constraint: col.constraint as string, colName: col.name, + colType: col.dataTypeDisplay as string, + colTests: col.columnTests, }))} + qualityTestFormHandler={qualityTestFormHandler} tableProfiles={tableProfile} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.interface.ts index e11c2a27aaf..361c3e80c01 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.interface.ts @@ -34,8 +34,9 @@ import { TableTest, TableTestType } from '../../generated/tests/tableTest'; import { EntityLineage } from '../../generated/type/entityLineage'; import { TagLabel } from '../../generated/type/tagLabel'; import { - CreateColumnTest, + ColumnTest, DatasetTestModeType, + ModifiedTableColumn, } from '../../interface/dataQuality.interface'; import { TitleBreadcrumbProps } from '../common/title-breadcrumb/title-breadcrumb.interface'; import { Edge, EdgeData } from '../EntityLineage/EntityLineage.interface'; @@ -60,7 +61,7 @@ export interface DatasetDetailsProps { description: string; tableProfile: Table['tableProfile']; tableQueries: Table['tableQueries']; - columns: Table['columns']; + columns: ModifiedTableColumn[]; tier: TagLabel; sampleData: TableData; entityLineage: EntityLineage; @@ -78,6 +79,10 @@ export interface DatasetDetailsProps { testMode: DatasetTestModeType; tableTestCase: TableTest[]; showTestForm: boolean; + qualityTestFormHandler: ( + tabValue: number, + testMode: DatasetTestModeType + ) => void; handleShowTestForm: (value: boolean) => void; handleTestModeChange: (mode: DatasetTestModeType) => void; createThread: (data: CreateThread) => void; @@ -94,7 +99,7 @@ export interface DatasetDetailsProps { entityLineageHandler: (lineage: EntityLineage) => void; postFeedHandler: (value: string, id: string) => void; handleAddTableTestCase: (data: CreateTableTest) => void; - handleAddColumnTestCase: (data: CreateColumnTest) => void; + handleAddColumnTestCase: (data: ColumnTest) => void; handleRemoveTableTest: (testType: TableTestType) => void; handleRemoveColumnTest: ( columnName: string, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.test.tsx index b482ad2976f..38ee62ea779 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DatasetDetails/DatasetDetails.test.tsx @@ -94,6 +94,7 @@ const DatasetDetailsProps = { handleRemoveTableTest: jest.fn(), handleRemoveColumnTest: jest.fn(), handleTestModeChange: jest.fn(), + qualityTestFormHandler: jest.fn(), }; jest.mock('../ManageTab/ManageTab.component', () => { return jest.fn().mockReturnValue(

ManageTab

); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/EntityTable/EntityTable.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/EntityTable/EntityTable.component.tsx index f87e0db3570..666987683a9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/EntityTable/EntityTable.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/EntityTable/EntityTable.component.tsx @@ -26,7 +26,12 @@ import { Table, } from '../../generated/entity/data/table'; import { Operation } from '../../generated/entity/policies/accessControl/rule'; +import { TestCaseStatus } from '../../generated/tests/tableTest'; import { LabelType, State, TagLabel } from '../../generated/type/tagLabel'; +import { + ColumnTest, + ModifiedTableColumn, +} from '../../interface/dataQuality.interface'; import { getHtmlForNonAdminAction, getPartialNameFromFQN, @@ -50,7 +55,7 @@ import Tags from '../tags/tags'; type Props = { owner: Table['owner']; - tableColumns: Table['columns']; + tableColumns: ModifiedTableColumn[]; joins: Array; searchText?: string; columnName: string; @@ -58,7 +63,7 @@ type Props = { isReadOnly?: boolean; entityFqn?: string; entityFieldThreads?: EntityFieldThreads[]; - onUpdate?: (columns: Table['columns']) => void; + onUpdate?: (columns: ModifiedTableColumn[]) => void; onThreadLinkSelect?: (value: string) => void; onEntityFieldSelect?: (value: string) => void; }; @@ -86,6 +91,10 @@ const EntityTable = ({ Header: 'Type', accessor: 'dataTypeDisplay', }, + { + Header: 'Data Quality', + accessor: 'columnTests', + }, { Header: 'Description', accessor: 'description', @@ -98,7 +107,9 @@ const EntityTable = ({ [] ); - const [searchedColumns, setSearchedColumns] = useState([]); + const [searchedColumns, setSearchedColumns] = useState( + [] + ); const data = React.useMemo( () => makeData(searchedColumns), @@ -158,7 +169,7 @@ const EntityTable = ({ }; const updateColumnDescription = ( - tableCols: Table['columns'], + tableCols: ModifiedTableColumn[], changedColName: string, description: string ) => { @@ -167,7 +178,7 @@ const EntityTable = ({ col.description = description; } else { updateColumnDescription( - col?.children as Table['columns'], + col?.children as ModifiedTableColumn[], changedColName, description ); @@ -176,7 +187,7 @@ const EntityTable = ({ }; const updateColumnTags = ( - tableCols: Table['columns'], + tableCols: ModifiedTableColumn[], changedColName: string, newColumnTags: Array ) => { @@ -204,7 +215,7 @@ const EntityTable = ({ col.tags = getUpdatedTags(col); } else { updateColumnTags( - col?.children as Table['columns'], + col?.children as ModifiedTableColumn[], changedColName, newColumnTags ); @@ -309,7 +320,8 @@ const EntityTable = ({ {headerGroup.headers.map((column: any, index: number) => ( @@ -332,6 +344,22 @@ const EntityTable = ({ {...row.getRowProps()}> {/* eslint-disable-next-line */} {row.cells.map((cell: any, index: number) => { + const columnTests = + cell.column.id === 'columnTests' + ? ((cell.value ?? []) as ColumnTest[]) + : ([] as ColumnTest[]); + const columnTestLength = columnTests.length; + const failingTests = columnTests.filter((test) => + test.results?.some( + (t) => t.testCaseStatus === TestCaseStatus.Failed + ) + ); + const passingTests = columnTests.filter((test) => + test.results?.some( + (t) => t.testCaseStatus === TestCaseStatus.Success + ) + ); + return ( ) : null} + {cell.column.id === 'columnTests' && ( + + {columnTestLength ? ( + + {failingTests.length ? ( +
+

+ +

+

+ {`${failingTests.length}/${columnTestLength} tests failing`} +

+
+ ) : ( +
+
+ +
+

{`${passingTests.length} tests`}

+
+ )} + + ) : ( + '--' + )} + + )} + {cell.column.id === 'dataTypeDisplay' && ( <> {isReadOnly ? ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.component.tsx index 6699f5061b6..aa066fb451a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/TableProfiler.component.tsx @@ -12,31 +12,73 @@ */ import classNames from 'classnames'; -import React, { Fragment, useState } from 'react'; +import React, { FC, Fragment } from 'react'; import { Link } from 'react-router-dom'; +import { TITLE_FOR_NON_ADMIN_ACTION } from '../../constants/constants'; import { Table, TableProfile } from '../../generated/entity/data/table'; +import { useAuth } from '../../hooks/authHooks'; +import { + ColumnTest, + DatasetTestModeType, +} from '../../interface/dataQuality.interface'; import { getConstraintIcon } from '../../utils/TableUtils'; -import TableProfilerGraph from './TableProfilerGraph.component'; +import { Button } from '../buttons/Button/Button'; +import NonAdminAction from '../common/non-admin-action/NonAdminAction'; +import PopOver from '../common/popover/PopOver'; +import RichTextEditorPreviewer from '../common/rich-text-editor/RichTextEditorPreviewer'; type Props = { tableProfiles: Table['tableProfile']; - columns: Array<{ constraint: string; colName: string }>; + columns: Array<{ + constraint: string; + colName: string; + colType: string; + colTests?: ColumnTest[]; + }>; + qualityTestFormHandler: ( + tabValue: number, + testMode: DatasetTestModeType + ) => void; }; -type ProfilerGraphData = Array<{ - date: Date; - value: number; -}>; +const PercentageGraph = ({ + percentage, + title, +}: { + percentage: number; + title: string; +}) => { + return ( + +
+
+
+ + ); +}; -const TableProfiler = ({ tableProfiles, columns }: Props) => { - const [expandedColumn, setExpandedColumn] = useState<{ - name: string; - isExpanded: boolean; - }>({ - name: '', - isExpanded: false, - }); +const excludedMetrics = [ + 'profilDate', + 'name', + 'nullCount', + 'nullProportion', + 'uniqueCount', + 'uniqueProportion', + 'rows', +]; +const TableProfiler: FC = ({ + tableProfiles, + columns, + qualityTestFormHandler, +}) => { + const { isAuthDisabled, isAdminUser } = useAuth(); const modifiedData = tableProfiles?.map((tableProfile: TableProfile) => ({ rows: tableProfile.rowCount, profileDate: tableProfile.profileDate, @@ -49,19 +91,27 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => { (colProfile) => colProfile.name === column.colName ); - return { profilDate: md.profileDate, ...currentColumn, rows: md.rows }; + return { + profilDate: md.profileDate, + ...currentColumn, + rows: md.rows, + }; }); return { name: column, + columnMetrics: Object.entries(data?.[0] ?? {}).map((d) => ({ + key: d[0], + value: d[1], + })), + columnTests: column.colTests, data, - min: data?.length ? data[0].min ?? 0 : 0, - max: data?.length ? data[0].max ?? 0 : 0, + type: column.colType, }; }); return ( - <> +
{tableProfiles?.length ? ( { - - - - - + + + + + + + {columnSpecificData.map((col, colIndex) => { @@ -88,27 +140,6 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => { className="tw-relative tableBody-cell" data-testid="tableBody-cell">
- - setExpandedColumn((prevState) => ({ - name: col.name.colName, - isExpanded: - prevState.name === col.name.colName - ? !prevState.isExpanded - : true, - })) - }> - {expandedColumn.name === col.name.colName ? ( - expandedColumn.isExpanded ? ( - - ) : ( - - ) - ) : ( - - )} - {col.name.constraint && ( {getConstraintIcon( @@ -120,84 +151,140 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => { {col.name.colName}
+ - - + + - {expandedColumn.name === col.name.colName && - expandedColumn.isExpanded && ( - - {col.data?.map((colData, index) => ( - - - - - - - - - ))} - - )} ); })} @@ -218,7 +305,7 @@ const TableProfiler = ({ tableProfiles, columns }: Props) => { )} - + ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/mockTourData.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/mockTourData.constants.ts index a0f3f2f9406..7e40105fdd2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/mockTourData.constants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/mockTourData.constants.ts @@ -15,86 +15,18 @@ export const mockFeedData = [ { - index: 'table_search_index', - id: 'd336d794-6fce-4094-a064-7ceb3b208dab', - name: 'dim_address', - description: - 'This dimension table contains the billing and shipping addresses of customers. You can join this table with the sales table to generate lists of the billing and shipping addresses. Customers can enter their addresses more than once, so the same address can appear in more than one row in this table. This table contains one row per customer address.', - fullyQualifiedName: 'bigquery_gcp.shopify.dim_address', - tableType: 'Regular', - tags: [ - 'PersonalData.Personal', - 'PII.Sensitive', - 'PII.NonSensitive', - 'PII.Sensitive', - 'PersonalData.Personal', - 'User.FacePhoto', - ], - service: 'bigquery_gcp', - serviceType: 'BigQuery', - database: 'shopify', - entityType: 'table', - changeDescriptions: [ - { - updatedBy: 'anonymous', - updatedAt: 1640239422867, - fieldsAdded: [], - fieldsUpdated: [], - fieldsDeleted: [], - }, - { - updatedBy: 'anonymous', - fieldsAdded: [ - { - newValue: - '[{"tagFQN":"PersonalData.Personal","labelType":"Manual","state":"Confirmed"}]', - name: 'columns.address_id.tags', - }, - ], - fieldsUpdated: [], - fieldsDeleted: [], - updatedAt: 1640247571788, - }, - { - updatedBy: 'anonymous', - fieldsAdded: [ - { - newValue: - '[{"tagFQN":"PII.Sensitive","labelType":"Manual","state":"Confirmed"}]', - name: 'columns.shop_id.tags', - }, - ], - fieldsUpdated: [], - fieldsDeleted: [], - updatedAt: 1640248828684, - }, - { - updatedBy: 'anonymous', - fieldsAdded: [ - { - newValue: - '[{"tagFQN":"PII.NonSensitive","labelType":"Manual","state":"Confirmed"}]', - name: 'columns.first_name.tags', - }, - ], - fieldsUpdated: [], - fieldsDeleted: [], - updatedAt: 1640248877069, - }, - { - updatedBy: 'anonymous', - fieldsAdded: [ - { - newValue: - '[{"tagFQN":"PII.Sensitive","description":"PII which if lost, compromised, or disclosed without authorization, could result in substantial harm, embarrassment, inconvenience, or unfairness to an individual.","labelType":"Derived","state":"Confirmed"},{"tagFQN":"PersonalData.Personal","description":"Data that can be used to directly or indirectly identify a person.","labelType":"Derived","state":"Confirmed"},{"tagFQN":"User.FacePhoto","labelType":"Manual","state":"Confirmed"}]', - name: 'columns.last_name.tags', - }, - ], - fieldsUpdated: [], - fieldsDeleted: [], - updatedAt: 1640248886960, - }, - ], + id: '2b133a6d-8562-4220-a997-eaaa0376cad3', + href: 'http://localhost:8585/api/v1/feed/2b133a6d-8562-4220-a997-eaaa0376cad3', + threadTs: 1646577496489, + about: '<#E/table/bigquery_gcp.shopify.dim_staff/description>', + entityId: '37e50e09-d5b6-4609-821c-1ed84d92dd57', + createdBy: 'aaron_johnson0', + updatedAt: 1646577496489, + updatedBy: 'anonymous', + resolved: false, + message: 'Can you add a description?', + postsCount: 0, + posts: [], }, ]; diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts index 7829ea9b6f7..a68a70b94ed 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts @@ -628,6 +628,10 @@ export interface ColumnProfile { * Variance of a column. */ variance?: number; + + minLength?: number; + + maxLength?: number; } export interface HistogramObject { diff --git a/openmetadata-ui/src/main/resources/ui/src/interface/dataQuality.interface.ts b/openmetadata-ui/src/main/resources/ui/src/interface/dataQuality.interface.ts index 3f28d11b86b..192c7201816 100644 --- a/openmetadata-ui/src/main/resources/ui/src/interface/dataQuality.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/interface/dataQuality.interface.ts @@ -30,12 +30,19 @@ export interface TestCaseConfigType { missingValueMatch?: string; } -export interface CreateColumnTest { +export interface Result { + executionTime: number; + testCaseStatus: string; + result: string; +} + +export interface ColumnTest { id?: string; columnName: string; description?: string; executionFrequency?: TestCaseExecutionFrequency; owner?: EntityReference; + results?: Result[]; testCase: { columnTestType: ColumnTestType; config?: TestCaseConfigType; @@ -45,7 +52,7 @@ export interface CreateColumnTest { export type DatasetTestModeType = 'table' | 'column'; export interface ModifiedTableColumn extends Column { - columnTests?: CreateColumnTest[]; + columnTests?: ColumnTest[]; } export interface TableTestDataType { diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/DatasetDetailsPage/DatasetDetailsPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/DatasetDetailsPage/DatasetDetailsPage.component.tsx index 088a8cbd7aa..f4cb0614ff6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/DatasetDetailsPage/DatasetDetailsPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/DatasetDetailsPage/DatasetDetailsPage.component.tsx @@ -77,7 +77,7 @@ import { EntityLineage } from '../../generated/type/entityLineage'; import { TagLabel } from '../../generated/type/tagLabel'; import useToastContext from '../../hooks/useToastContext'; import { - CreateColumnTest, + ColumnTest, DatasetTestModeType, ModifiedTableColumn, } from '../../interface/dataQuality.interface'; @@ -118,7 +118,7 @@ const DatasetDetailsPage: FunctionComponent = () => { TitleBreadcrumbProps['titleLinks'] >([]); const [description, setDescription] = useState(''); - const [columns, setColumns] = useState([]); + const [columns, setColumns] = useState([]); const [sampleData, setSampleData] = useState({ columns: [], rows: [], @@ -191,6 +191,15 @@ const DatasetDetailsPage: FunctionComponent = () => { } }; + const qualityTestFormHandler = ( + tabValue: number, + testMode: DatasetTestModeType + ) => { + activeTabHandler(tabValue); + setTestMode(testMode); + setShowTestForm(true); + }; + const getLineageData = () => { setIsLineageLoading(true); getLineageByFQN(tableFQN, EntityType.TABLE) @@ -647,7 +656,7 @@ const DatasetDetailsPage: FunctionComponent = () => { }); }; - const handleAddColumnTestCase = (data: CreateColumnTest) => { + const handleAddColumnTestCase = (data: ColumnTest) => { addColumnTestCase(tableDetails.id, data) .then((res: AxiosResponse) => { const columnTestRes = res.data.columns.find( @@ -786,6 +795,7 @@ const DatasetDetailsPage: FunctionComponent = () => { loadNodeHandler={loadNodeHandler} owner={owner as Table['owner'] & { displayName: string }} postFeedHandler={postFeedHandler} + qualityTestFormHandler={qualityTestFormHandler} removeLineageHandler={removeLineageHandler} sampleData={sampleData} setActiveTabHandler={activeTabHandler} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/tour-page/TourPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/tour-page/TourPage.component.tsx index 1126934daf8..ccb4a780a86 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/tour-page/TourPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/tour-page/TourPage.component.tsx @@ -127,7 +127,7 @@ const TourPage = () => { pipelineCount: 8, }} error="" - feedData={mockFeedData as unknown as MyDataProps['feedData']} + feedData={mockFeedData as MyDataProps['feedData']} feedFilter={FeedFilter.ALL} feedFilterHandler={() => { setMyDataSearchResult(mockData); @@ -185,7 +185,7 @@ const TourPage = () => { entityLineage={mockDatasetData.entityLineage} entityLineageHandler={handleCountChange} entityName={mockDatasetData.entityName} - entityThread={[]} + entityThread={mockFeedData} feedCount={0} followTableHandler={handleCountChange} followers={mockDatasetData.followers} @@ -205,6 +205,7 @@ const TourPage = () => { loadNodeHandler={handleCountChange} owner={undefined as unknown as DatasetOwner} postFeedHandler={handleCountChange} + qualityTestFormHandler={handleCountChange} removeLineageHandler={handleCountChange} sampleData={mockDatasetData.sampleData} setActiveTabHandler={(tab) => setdatasetActiveTab(tab)} diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx index 84e8ac0a282..dd886d537ec 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx @@ -14,7 +14,7 @@ import classNames from 'classnames'; import { upperCase } from 'lodash'; import { EntityTags, TableDetail } from 'Models'; -import React from 'react'; +import React, { Fragment } from 'react'; import AppState from '../AppState'; import PopOver from '../components/common/popover/PopOver'; import { @@ -27,8 +27,10 @@ import { import { EntityType } from '../enums/entity.enum'; import { SearchIndex } from '../enums/search.enum'; import { ConstraintTypes } from '../enums/table.enum'; -import { Column, DataType, Table } from '../generated/entity/data/table'; +import { Column, DataType } from '../generated/entity/data/table'; +import { TableTest, TestCaseStatus } from '../generated/tests/tableTest'; import { TagLabel } from '../generated/type/tagLabel'; +import { ModifiedTableColumn } from '../interface/dataQuality.interface'; import { ordinalize } from './StringsUtils'; import SVGIcons from './SvgUtils'; @@ -253,7 +255,7 @@ export const makeRow = (column: Column) => { }; export const makeData = ( - columns: Table['columns'] = [] + columns: ModifiedTableColumn[] = [] ): Array => { const data = columns.map((column) => ({ ...makeRow(column), @@ -290,3 +292,38 @@ export const getDataTypeString = (dataType: string): string => { return dataType; } }; + +export const getTableTestsValue = (tableTestCase: TableTest[]) => { + const tableTestLength = tableTestCase.length; + + const failingTests = tableTestCase.filter((test) => + test.results?.some((t) => t.testCaseStatus === TestCaseStatus.Failed) + ); + const passingTests = tableTestCase.filter((test) => + test.results?.some((t) => t.testCaseStatus === TestCaseStatus.Success) + ); + + return ( + + {tableTestLength ? ( + + {failingTests.length ? ( +
+

+ +

+

{`${failingTests.length}/${tableTestLength} tests failing`}

+
+ ) : ( +
+
+ +
+

{`${passingTests.length} tests`}

+
+ )} + + ) : null} + + ); +};
Column NameDistinct Ratio (%)Null Ratio (%)MinMaxStandard DeviationTypeNullUniqueDistinctMetricsTests
+
+ {col.name.colType.length > 25 ? ( + + + {col.name.colType.toLowerCase()} +
+ } + position="bottom" + theme="light" + trigger="click"> +
+ +
+ + + ) : ( + col.name.colType.toLowerCase() + )} + + 		- ({ - date: d.profilDate, - value: d.uniqueProportion ?? 0, - })) - .reverse() as ProfilerGraphData - } + - ({ - date: d.profilDate, - value: d.nullProportion ?? 0, - })) - .reverse() as ProfilerGraphData + {col.min}{col.max} - ({ - date: d.profilDate, - value: d.stddev ?? 0, - })) - .reverse() as ProfilerGraphData + +
+ {col.columnMetrics + .filter((m) => !excludedMetrics.includes(m.key)) + .map((m, i) => ( +

+ {m.key} + - + {m.value} +

+ ))} +
+ 		+
+ {col.columnTests ? ( +
+ {col.columnTests.map((m, i) => ( +
+

+ {m.results?.every( + (result) => + result.testCaseStatus === 'Success' + ) ? ( + + ) : ( + + )} +

+
+ + {m.testCase.columnTestType} + +
+ {Object.entries( + m.testCase.config ?? {} + ).map((config, i) => ( +

+ {config[0]}: + {config[1] ?? 'null'} +

+ ))} +
+
+
+ ))} +
+ ) : ( + `No tests available` + )} +
+ + + +
+
+
- - {colData.profilDate} - - - {colData.uniqueProportion ?? 0} - - {colData.nullProportion ?? 0} - - {colData.min ?? 0} - - {colData.max ?? 0} - - {colData.stddev ?? 0} -