yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-25 06:28:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Issue 8930 - Update profiler timestamp from seconds to milliseconds (#12948)

Browse Source
This commit is contained in:
Teddy 2023-08-25 08:47:16 +02:00 committed by GitHub
parent 19b5c946a7
commit 101cd0ebac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 67 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bootstrap/sql/migrations/native/1.1.2/mysql/schemaChanges.sql
View File

@ -20,4 +20,4 @@ SET json = JSON_INSERT(
'$.connection.config.authType.password',
JSON_EXTRACT(json, '$.connection.config.password'))
where serviceType = 'Trino'
AND JSON_EXTRACT(json, '$.connection.config.password') IS NOT NULL;
AND JSON_EXTRACT(json, '$.connection.config.password') IS NOT NULL;

11
bootstrap/sql/migrations/native/1.1.3/mysql/schemaChanges.sql
View File

@ -1,3 +1,14 @@
-- Update table and column profile timestamps to be in milliseconds
UPDATE entity_extension_time_series
SET json = JSON_INSERT(
JSON_REMOVE(json, '$.timestamp'),
'$.timestamp',
JSON_EXTRACT(json, '$.timestamp') * 1000
)
WHERE
extension in ('table.tableProfile', 'table.columnProfile');
;
ALTER TABLE automations_workflow MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin,MODIFY COLUMN workflowType VARCHAR(256) COLLATE ascii_bin, MODIFY COLUMN status VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE entity_extension MODIFY COLUMN extension VARCHAR(256) COLLATE ascii_bin;
ALTER TABLE entity_extension_time_series MODIFY COLUMN entityFQNHash VARCHAR(768) COLLATE ascii_bin, MODIFY COLUMN jsonSchema VARCHAR(50) COLLATE ascii_bin, MODIFY COLUMN extension VARCHAR(100) COLLATE ascii_bin,

11
bootstrap/sql/migrations/native/1.1.3/postgres/schemaChanges.sql
View File

@ -1,3 +1,14 @@
-- Update table and column profile timestamps to be in milliseconds
UPDATE entity_extension_time_series
SET json = jsonb_set(
json,
'{timestamp}',
to_jsonb(cast(json#>'{timestamp}' as int8) *1000)
)
WHERE
extension in ('table.tableProfile', 'table.columnProfile');
;
ALTER TABLE entity_extension_time_series ALTER COLUMN entityFQNHash TYPE VARCHAR(768), ALTER COLUMN jsonSchema TYPE VARCHAR(50) , ALTER COLUMN extension TYPE VARCHAR(100) ,
ADD CONSTRAINT entity_extension_time_series_constraint UNIQUE (entityFQNHash, extension, timestamp);
ALTER TABLE field_relationship ALTER COLUMN fromFQNHash TYPE VARCHAR(768), ALTER COLUMN toFQNHash TYPE VARCHAR(768);

5
ingestion/src/metadata/ingestion/ometa/mixins/table_mixin.py
View File

@ -254,11 +254,6 @@ class OMetaTableMixin:
url_after = f"&after={after}" if after else ""
profile_type_url = profile_type.__name__[0].lower() + profile_type.__name__[1:]
# system profile uses milliseconds
if profile_type is not SystemProfile:
start_ts = start_ts // 1000
end_ts = end_ts // 1000
resp = self.client.get(
f"{self.get_suffix(Table)}/{fqn}/{profile_type_url}?limit={limit}{url_after}",
data={"startTs": start_ts, "endTs": end_ts},

26
ingestion/src/metadata/ingestion/source/database/sample_data.py
View File

@ -1249,15 +1249,22 @@ class SampleDataSource(
rowCount=profile["rowCount"],
createDateTime=profile.get("createDateTime"),
sizeInByte=profile.get("sizeInByte"),
timestamp=(
datetime.now(tz=timezone.utc) - timedelta(days=days)
).timestamp(),
timestamp=int(
(
datetime.now(tz=timezone.utc) - timedelta(days=days)
).timestamp()
* 1000
),
),
columnProfile=[
ColumnProfile(
timestamp=(
datetime.now(tz=timezone.utc) - timedelta(days=days)
).timestamp(),
timestamp=int(
(
datetime.now(tz=timezone.utc)
- timedelta(days=days)
).timestamp()
* 1000
),
**col_profile,
)
for col_profile in profile["columnProfile"]
@ -1344,9 +1351,10 @@ class SampleDataSource(
for days, result in enumerate(test_case_results["results"]):
yield OMetaTestCaseResultsSample(
test_case_results=TestCaseResult(
timestamp=(
datetime.now() - timedelta(days=days)
).timestamp(),
timestamp=int(
(datetime.now() - timedelta(days=days)).timestamp()
* 1000
),
testCaseStatus=result["testCaseStatus"],
result=result["result"],
testResultValue=[

4
ingestion/src/metadata/profiler/interface/pandas/profiler_interface.py
View File

@ -354,7 +354,9 @@ class PandasProfilerInterface(ProfilerInterface, PandasInterfaceMixin):
profile_results["columns"][column].update(
{
"name": column,
"timestamp": datetime.now(tz=timezone.utc).timestamp(),
"timestamp": int(
datetime.now(tz=timezone.utc).timestamp() * 1000
),
**profile,
}
)

4
ingestion/src/metadata/profiler/interface/sqlalchemy/profiler_interface.py
View File

@ -471,7 +471,9 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
profile_results["columns"][column].update(
{
"name": column,
"timestamp": datetime.now(tz=timezone.utc).timestamp(),
"timestamp": int(
datetime.now(tz=timezone.utc).timestamp() * 1000
),
**profile,
}
)

2
ingestion/src/metadata/profiler/processor/core.py
View File

@ -92,7 +92,7 @@ class Profiler(Generic[TMetric]):
self.include_columns = include_columns
self.exclude_columns = exclude_columns
self._metrics = metrics
self._profile_date = datetime.now(tz=timezone.utc).timestamp()
self._profile_date = int(datetime.now(tz=timezone.utc).timestamp() * 1000)
self.profile_sample_config = self.profiler_interface.profile_sample_config
self.validate_composed_metric()

6
ingestion/tests/integration/orm_profiler/test_orm_profiler_e2e.py
View File

@ -212,9 +212,6 @@ class ProfilerWorkflowTest(TestCase):
)
assert table_entity.fullyQualifiedName.__root__ == "test_sqlite.main.main.users"
@pytest.mark.skip(
"need to reactivate once https://github.com/open-metadata/OpenMetadata/issues/8930 is handled. Skipping to prevent Cypress failure"
)
def test_profiler_workflow(self):
"""
Prepare and execute the profiler workflow
@ -539,8 +536,7 @@ class ProfilerWorkflowTest(TestCase):
).profile
assert profile.rowCount == 4.0
# uncomment when reactivate once https://github.com/open-metadata/OpenMetadata/issues/8930 is fixed
# assert profile.profileSample is None
assert profile.profileSample is None
workflow_config["processor"] = {
"type": "orm-profiler",

12
openmetadata-ui/src/main/resources/ui/src/components/DatePickerMenu/DatePickerMenu.component.tsx
View File

@ -25,8 +25,8 @@ import React, { useState } from 'react';
import { useTranslation } from 'react-i18next';
import { getDaysCount, getTimestampLabel } from 'utils/DatePickerMenuUtils';
import {
getCurrentDateTimeStamp,
getPastDatesTimeStampFromCurrentDate,
getCurrentDateTimeMillis,
getPastDaysDateTimeMillis,
} from 'utils/TimeUtils';
import { ReactComponent as DropdownIcon } from '../../assets/svg/DropDown.svg';
import './DatePickerMenu.style.less';
@ -57,9 +57,9 @@ function DatePickerMenu({
dateStrings
) => {
if (values) {
const startTs = values[0]?.set({ h: 0, m: 0 }).utc().unix() ?? 0;
const startTs = (values[0]?.set({ h: 0, m: 0 }).utc().unix() ?? 0) * 1000;
const endTs = values[1]?.set({ h: 23, m: 59 }).utc().unix() ?? 0;
const endTs = (values[1]?.set({ h: 23, m: 59 }).utc().unix() ?? 0) * 1000;
const daysCount = getDaysCount(dateStrings[0], dateStrings[1]);
@ -87,9 +87,9 @@ function DatePickerMenu({
const selectedNumberOfDays = filterRange.days;
const keyString = key as keyof typeof PROFILER_FILTER_RANGE;
const startTs = getPastDatesTimeStampFromCurrentDate(selectedNumberOfDays);
const startTs = getPastDaysDateTimeMillis(selectedNumberOfDays);
const endTs = getCurrentDateTimeStamp();
const endTs = getCurrentDateTimeMillis();
setSelectedTimeRange(PROFILER_FILTER_RANGE[keyString].title);
setSelectedTimeRangeKey(keyString);

10
openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/TableProfilerChart.test.tsx
View File

@ -23,7 +23,7 @@ const mockTimeValue = {
endMilli: 1670667984000,
startMilli: 1670408784000,
};
const mockDateRangeObject = { startTs: 1670408784, endTs: 1670667984 };
const mockDateRangeObject = { startTs: 1670408784000, endTs: 1670667984000 };
jest.mock('react-router-dom', () => ({
useParams: jest.fn().mockImplementation(() => ({ datasetFQN: mockFQN })),
@ -87,8 +87,8 @@ describe('TableProfilerChart component test', () => {
endTs: mockTimeValue.endMilli,
});
expect(mockGetTableProfilesList.mock.calls[0][1]).toEqual({
startTs: mockTimeValue.startSec,
endTs: mockTimeValue.endSec,
startTs: mockTimeValue.startMilli,
endTs: mockTimeValue.endMilli,
});
});
@ -110,8 +110,8 @@ describe('TableProfilerChart component test', () => {
endTs: mockTimeValue.endMilli,
});
expect(mockGetTableProfilesList.mock.calls[0][1]).toEqual({
startTs: startTime.inSec,
endTs: mockTimeValue.endSec,
startTs: startTime.inMilli,
endTs: mockTimeValue.endMilli,
});
});
});

4
openmetadata-ui/src/main/resources/ui/src/components/TableProfiler/Component/TableProfilerChart.tsx
View File

@ -87,8 +87,8 @@ const TableProfilerChart = ({
setIsLoading(true);
await fetchTableProfiler(fqn, dateRangeObj);
await fetchSystemProfiler(fqn, {
startTs: dateRangeObj.startTs * 1000,
endTs: dateRangeObj.endTs * 1000,
startTs: dateRangeObj.startTs,
endTs: dateRangeObj.endTs,
});
setIsLoading(false);
};

8
openmetadata-ui/src/main/resources/ui/src/constants/profiler.constant.ts
View File

@ -15,8 +15,8 @@ import { t } from 'i18next';
import { StepperStepType } from 'Models';
import i18n from 'utils/i18next/LocalUtil';
import {
getCurrentDateTimeStamp,
getPastDatesTimeStampFromCurrentDate,
getCurrentDateTimeMillis,
getPastDaysDateTimeMillis,
} from 'utils/TimeUtils';
import { CSMode } from '../enums/codemirror.enum';
import { DMLOperationType } from '../generated/api/data/createTableProfile';
@ -117,9 +117,9 @@ export const DEFAULT_SELECTED_RANGE = {
};
export const DEFAULT_RANGE_DATA = {
startTs: getPastDatesTimeStampFromCurrentDate(DEFAULT_SELECTED_RANGE.days),
startTs: getPastDaysDateTimeMillis(DEFAULT_SELECTED_RANGE.days),
endTs: getCurrentDateTimeStamp(),
endTs: getCurrentDateTimeMillis(),
};
export const COLORS = ['#7147E8', '#B02AAC', '#B02AAC', '#1890FF', '#008376'];

5
openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/DataInsightPage.component.tsx
View File

@ -161,9 +161,8 @@ const DataInsightPage = () => {
setSelectedDaysFilter(daysValue ?? 0);
setChartFilter((previous) => ({
...previous,
// Converting coming data to milliseconds
startTs: value.startTs * 1000,
endTs: value.endTs * 1000,
startTs: value.startTs,
endTs: value.endTs,
}));
}
};