From 101cd0ebac6deb433e9f41ed162fbc492491a796 Mon Sep 17 00:00:00 2001 From: Teddy Date: Fri, 25 Aug 2023 08:47:16 +0200 Subject: [PATCH] Issue 8930 - Update profiler timestamp from seconds to milliseconds (#12948) --- .../native/1.1.2/mysql/schemaChanges.sql | 2 +- .../native/1.1.3/mysql/schemaChanges.sql | 11 ++++++++ .../native/1.1.3/postgres/schemaChanges.sql | 11 ++++++++ .../ingestion/ometa/mixins/table_mixin.py | 5 ---- .../ingestion/source/database/sample_data.py | 26 ++++++++++++------- .../interface/pandas/profiler_interface.py | 4 ++- .../sqlalchemy/profiler_interface.py | 4 ++- .../src/metadata/profiler/processor/core.py | 2 +- .../orm_profiler/test_orm_profiler_e2e.py | 6 +---- .../DatePickerMenu.component.tsx | 12 ++++----- .../Component/TableProfilerChart.test.tsx | 10 +++---- .../Component/TableProfilerChart.tsx | 4 +-- .../ui/src/constants/profiler.constant.ts | 8 +++--- .../DataInsightPage.component.tsx | 5 ++-- 14 files changed, 67 insertions(+), 43 deletions(-) diff --git a/bootstrap/sql/migrations/native/1.1.2/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.1.2/mysql/schemaChanges.sql index 6b09a3a84af..3c69c9fd152 100644 --- a/bootstrap/sql/migrations/native/1.1.2/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.1.2/mysql/schemaChanges.sql @@ -20,4 +20,4 @@ SET json = JSON_INSERT( '$.connection.config.authType.password', JSON_EXTRACT(json, '$.connection.config.password')) where serviceType = 'Trino' - AND JSON_EXTRACT(json, '$.connection.config.password') IS NOT NULL; \ No newline at end of file + AND JSON_EXTRACT(json, '$.connection.config.password') IS NOT NULL; diff --git a/bootstrap/sql/migrations/native/1.1.3/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.1.3/mysql/schemaChanges.sql index d52b8f8dd0e..1348267f2ec 100644 --- a/bootstrap/sql/migrations/native/1.1.3/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.1.3/mysql/schemaChanges.sql @@ -1,3 +1,14 @@ +-- Update table and column profile timestamps to be in milliseconds +UPDATE entity_extension_time_series + SET json = JSON_INSERT( + JSON_REMOVE(json, '$.timestamp'), + '$.timestamp', + JSON_EXTRACT(json, '$.timestamp') * 1000 + ) +WHERE + extension in ('table.tableProfile', 'table.columnProfile'); +; + ALTER TABLE automations_workflow MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin,MODIFY COLUMN workflowType VARCHAR(256) COLLATE ascii_bin, MODIFY COLUMN status VARCHAR(256) COLLATE ascii_bin; ALTER TABLE entity_extension MODIFY COLUMN extension VARCHAR(256) COLLATE ascii_bin; ALTER TABLE entity_extension_time_series MODIFY COLUMN entityFQNHash VARCHAR(768) COLLATE ascii_bin, MODIFY COLUMN jsonSchema VARCHAR(50) COLLATE ascii_bin, MODIFY COLUMN extension VARCHAR(100) COLLATE ascii_bin, diff --git a/bootstrap/sql/migrations/native/1.1.3/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.1.3/postgres/schemaChanges.sql index e6de4bf12ec..d2f5197c94a 100644 --- a/bootstrap/sql/migrations/native/1.1.3/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.1.3/postgres/schemaChanges.sql @@ -1,3 +1,14 @@ +-- Update table and column profile timestamps to be in milliseconds +UPDATE entity_extension_time_series +SET json = jsonb_set( + json, + '{timestamp}', + to_jsonb(cast(json#>'{timestamp}' as int8) *1000) +) +WHERE + extension in ('table.tableProfile', 'table.columnProfile'); +; + ALTER TABLE entity_extension_time_series ALTER COLUMN entityFQNHash TYPE VARCHAR(768), ALTER COLUMN jsonSchema TYPE VARCHAR(50) , ALTER COLUMN extension TYPE VARCHAR(100) , ADD CONSTRAINT entity_extension_time_series_constraint UNIQUE (entityFQNHash, extension, timestamp); ALTER TABLE field_relationship ALTER COLUMN fromFQNHash TYPE VARCHAR(768), ALTER COLUMN toFQNHash TYPE VARCHAR(768); diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/table_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/table_mixin.py index f098bcf01e8..511ceb30deb 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/table_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/table_mixin.py @@ -254,11 +254,6 @@ class OMetaTableMixin: url_after = f"&after={after}" if after else "" profile_type_url = profile_type.__name__[0].lower() + profile_type.__name__[1:] - # system profile uses milliseconds - if profile_type is not SystemProfile: - start_ts = start_ts // 1000 - end_ts = end_ts // 1000 - resp = self.client.get( f"{self.get_suffix(Table)}/{fqn}/{profile_type_url}?limit={limit}{url_after}", data={"startTs": start_ts, "endTs": end_ts}, diff --git a/ingestion/src/metadata/ingestion/source/database/sample_data.py b/ingestion/src/metadata/ingestion/source/database/sample_data.py index e70ecfdfec4..184a52d7809 100644 --- a/ingestion/src/metadata/ingestion/source/database/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/database/sample_data.py @@ -1249,15 +1249,22 @@ class SampleDataSource( rowCount=profile["rowCount"], createDateTime=profile.get("createDateTime"), sizeInByte=profile.get("sizeInByte"), - timestamp=( - datetime.now(tz=timezone.utc) - timedelta(days=days) - ).timestamp(), + timestamp=int( + ( + datetime.now(tz=timezone.utc) - timedelta(days=days) + ).timestamp() + * 1000 + ), ), columnProfile=[ ColumnProfile( - timestamp=( - datetime.now(tz=timezone.utc) - timedelta(days=days) - ).timestamp(), + timestamp=int( + ( + datetime.now(tz=timezone.utc) + - timedelta(days=days) + ).timestamp() + * 1000 + ), **col_profile, ) for col_profile in profile["columnProfile"] @@ -1344,9 +1351,10 @@ class SampleDataSource( for days, result in enumerate(test_case_results["results"]): yield OMetaTestCaseResultsSample( test_case_results=TestCaseResult( - timestamp=( - datetime.now() - timedelta(days=days) - ).timestamp(), + timestamp=int( + (datetime.now() - timedelta(days=days)).timestamp() + * 1000 + ), testCaseStatus=result["testCaseStatus"], result=result["result"], testResultValue=[ diff --git a/ingestion/src/metadata/profiler/interface/pandas/profiler_interface.py b/ingestion/src/metadata/profiler/interface/pandas/profiler_interface.py index dbbecb9a292..5c1e8939701 100644 --- a/ingestion/src/metadata/profiler/interface/pandas/profiler_interface.py +++ b/ingestion/src/metadata/profiler/interface/pandas/profiler_interface.py @@ -354,7 +354,9 @@ class PandasProfilerInterface(ProfilerInterface, PandasInterfaceMixin): profile_results["columns"][column].update( { "name": column, - "timestamp": datetime.now(tz=timezone.utc).timestamp(), + "timestamp": int( + datetime.now(tz=timezone.utc).timestamp() * 1000 + ), **profile, } ) diff --git a/ingestion/src/metadata/profiler/interface/sqlalchemy/profiler_interface.py b/ingestion/src/metadata/profiler/interface/sqlalchemy/profiler_interface.py index 99f26c96ea5..18730ef5f0a 100644 --- a/ingestion/src/metadata/profiler/interface/sqlalchemy/profiler_interface.py +++ b/ingestion/src/metadata/profiler/interface/sqlalchemy/profiler_interface.py @@ -471,7 +471,9 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin): profile_results["columns"][column].update( { "name": column, - "timestamp": datetime.now(tz=timezone.utc).timestamp(), + "timestamp": int( + datetime.now(tz=timezone.utc).timestamp() * 1000 + ), **profile, } ) diff --git a/ingestion/src/metadata/profiler/processor/core.py b/ingestion/src/metadata/profiler/processor/core.py index 342295ae14d..8a8354c0242 100644 --- a/ingestion/src/metadata/profiler/processor/core.py +++ b/ingestion/src/metadata/profiler/processor/core.py @@ -92,7 +92,7 @@ class Profiler(Generic[TMetric]): self.include_columns = include_columns self.exclude_columns = exclude_columns self._metrics = metrics - self._profile_date = datetime.now(tz=timezone.utc).timestamp() + self._profile_date = int(datetime.now(tz=timezone.utc).timestamp() * 1000) self.profile_sample_config = self.profiler_interface.profile_sample_config self.validate_composed_metric() diff --git a/ingestion/tests/integration/orm_profiler/test_orm_profiler_e2e.py b/ingestion/tests/integration/orm_profiler/test_orm_profiler_e2e.py index 5fedf2a96a6..fea99939141 100644 --- a/ingestion/tests/integration/orm_profiler/test_orm_profiler_e2e.py +++ b/ingestion/tests/integration/orm_profiler/test_orm_profiler_e2e.py @@ -212,9 +212,6 @@ class ProfilerWorkflowTest(TestCase): ) assert table_entity.fullyQualifiedName.__root__ == "test_sqlite.main.main.users" - @pytest.mark.skip( - "need to reactivate once https://github.com/open-metadata/OpenMetadata/issues/8930 is handled. Skipping to prevent Cypress failure" - ) def test_profiler_workflow(self): """ Prepare and execute the profiler workflow @@ -539,8 +536,7 @@ class ProfilerWorkflowTest(TestCase): ).profile assert profile.rowCount == 4.0 - # uncomment when reactivate once https://github.com/open-metadata/OpenMetadata/issues/8930 is fixed - # assert profile.profileSample is None + assert profile.profileSample is None workflow_config["processor"] = { "type": "orm-profiler", diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DatePickerMenu/DatePickerMenu.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DatePickerMenu/DatePickerMenu.component.tsx index 8b5e5baaf56..ca93c1358da 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DatePickerMenu/DatePickerMenu.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DatePickerMenu/DatePickerMenu.component.tsx @@ -25,8 +25,8 @@ import React, { useState } from 'react'; import { useTranslation } from 'react-i18next'; import { getDaysCount, getTimestampLabel } from 'utils/DatePickerMenuUtils'; import { - getCurrentDateTimeStamp, - getPastDatesTimeStampFromCurrentDate, + getCurrentDateTimeMillis, + getPastDaysDateTimeMillis, } from 'utils/TimeUtils'; import { ReactComponent as DropdownIcon } from '../../assets/svg/DropDown.svg'; import './DatePickerMenu.style.less'; @@ -57,9 +57,9 @@ function DatePickerMenu({ dateStrings ) => { if (values) { - const startTs = values[0]?.set({ h: 0, m: 0 }).utc().unix() ?? 0; + const startTs = (values[0]?.set({ h: 0, m: 0 }).utc().unix() ?? 0) * 1000; - const endTs = values[1]?.set({ h: 23, m: 59 }).utc().unix() ?? 0; + const endTs = (values[1]?.set({ h: 23, m: 59 }).utc().unix() ?? 0) * 1000; const daysCount = getDaysCount(dateStrings[0], dateStrings[1]); @@ -87,9 +87,9 @@ function DatePickerMenu({ const selectedNumberOfDays = filterRange.days; const keyString = key as keyof typeof PROFILER_FILTER_RANGE; - const startTs = getPastDatesTimeStampFromCurrentDate(selectedNumberOfDays); + const startTs = getPastDaysDateTimeMillis(selectedNumberOfDays); - const endTs = getCurrentDateTimeStamp(); + const endTs = getCurrentDateTimeMillis(); setSelectedTimeRange(PROFILER_FILTER_RANGE[keyString].title); setSelectedTimeRangeKey(keyString); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/TableProfilerChart.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/TableProfilerChart.test.tsx index 3f2ea806c6c..ec285a4d3f1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/TableProfilerChart.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/TableProfilerChart.test.tsx @@ -23,7 +23,7 @@ const mockTimeValue = { endMilli: 1670667984000, startMilli: 1670408784000, }; -const mockDateRangeObject = { startTs: 1670408784, endTs: 1670667984 }; +const mockDateRangeObject = { startTs: 1670408784000, endTs: 1670667984000 }; jest.mock('react-router-dom', () => ({ useParams: jest.fn().mockImplementation(() => ({ datasetFQN: mockFQN })), @@ -87,8 +87,8 @@ describe('TableProfilerChart component test', () => { endTs: mockTimeValue.endMilli, }); expect(mockGetTableProfilesList.mock.calls[0][1]).toEqual({ - startTs: mockTimeValue.startSec, - endTs: mockTimeValue.endSec, + startTs: mockTimeValue.startMilli, + endTs: mockTimeValue.endMilli, }); }); @@ -110,8 +110,8 @@ describe('TableProfilerChart component test', () => { endTs: mockTimeValue.endMilli, }); expect(mockGetTableProfilesList.mock.calls[0][1]).toEqual({ - startTs: startTime.inSec, - endTs: mockTimeValue.endSec, + startTs: startTime.inMilli, + endTs: mockTimeValue.endMilli, }); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/TableProfilerChart.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/TableProfilerChart.tsx index 7bcfa97365b..fb795907cb6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/TableProfilerChart.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/TableProfilerChart.tsx @@ -87,8 +87,8 @@ const TableProfilerChart = ({ setIsLoading(true); await fetchTableProfiler(fqn, dateRangeObj); await fetchSystemProfiler(fqn, { - startTs: dateRangeObj.startTs * 1000, - endTs: dateRangeObj.endTs * 1000, + startTs: dateRangeObj.startTs, + endTs: dateRangeObj.endTs, }); setIsLoading(false); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/profiler.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/profiler.constant.ts index 0b2974fd6e9..62050b908f9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/profiler.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/profiler.constant.ts @@ -15,8 +15,8 @@ import { t } from 'i18next'; import { StepperStepType } from 'Models'; import i18n from 'utils/i18next/LocalUtil'; import { - getCurrentDateTimeStamp, - getPastDatesTimeStampFromCurrentDate, + getCurrentDateTimeMillis, + getPastDaysDateTimeMillis, } from 'utils/TimeUtils'; import { CSMode } from '../enums/codemirror.enum'; import { DMLOperationType } from '../generated/api/data/createTableProfile'; @@ -117,9 +117,9 @@ export const DEFAULT_SELECTED_RANGE = { }; export const DEFAULT_RANGE_DATA = { - startTs: getPastDatesTimeStampFromCurrentDate(DEFAULT_SELECTED_RANGE.days), + startTs: getPastDaysDateTimeMillis(DEFAULT_SELECTED_RANGE.days), - endTs: getCurrentDateTimeStamp(), + endTs: getCurrentDateTimeMillis(), }; export const COLORS = ['#7147E8', '#B02AAC', '#B02AAC', '#1890FF', '#008376']; diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/DataInsightPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/DataInsightPage.component.tsx index ffac0b27513..26af0416369 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/DataInsightPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/DataInsightPage.component.tsx @@ -161,9 +161,8 @@ const DataInsightPage = () => { setSelectedDaysFilter(daysValue ?? 0); setChartFilter((previous) => ({ ...previous, - // Converting coming data to milliseconds - startTs: value.startTs * 1000, - endTs: value.endTs * 1000, + startTs: value.startTs, + endTs: value.endTs, })); } };