yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-29 17:49:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Convert dbt test result timestamps to millis (#13903)

Browse Source 
* Fixed dbt test res time to millis

* added postgres migration
This commit is contained in:
Onkar Ravgan 2023-11-10 16:15:16 +05:30 committed by GitHub
parent 71a67d6bbe
commit b7e7625071
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 28 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
bootstrap/sql/migrations/native/1.2.1/mysql/schemaChanges.sql
View File

@ -0,0 +1,11 @@
--update the timestamps to millis for dbt test results
UPDATE data_quality_data_time_series dqdts
SET dqdts.json = JSON_INSERT(
JSON_REMOVE(dqdts.json, '$.timestamp'),
'$.timestamp',
JSON_EXTRACT(dqdts.json, '$.timestamp') * 1000
)
WHERE dqdts.extension = 'testCase.testCaseResult'
AND JSON_EXTRACT(dqdts.json, '$.timestamp') REGEXP '^[0-9]{10}$'
;

13
bootstrap/sql/migrations/native/1.2.1/postgres/schemaChanges.sql
View File

@ -8,4 +8,15 @@ SET json = jsonb_set(
true
)
WHERE json #>> '{pipelineType}' = 'metadata'
AND json #>> '{sourceConfig,config,type}' = 'DatabaseMetadata';
AND json #>> '{sourceConfig,config,type}' = 'DatabaseMetadata';
--update the timestamps to millis for dbt test results
UPDATE data_quality_data_time_series dqdts
SET json = jsonb_set(
dqdts.json::jsonb,
'{timestamp}',
to_jsonb(((dqdts.json ->> 'timestamp')::bigint)*1000)
)
WHERE dqdts.extension = 'testCase.testCaseResult'
AND (json->>'timestamp') ~ '^[0-9]{10}$';

7
ingestion/src/metadata/ingestion/source/database/dbt/metadata.py
View File

@ -86,6 +86,7 @@ from metadata.utils import fqn
from metadata.utils.elasticsearch import get_entity_from_es_result
from metadata.utils.logger import ingestion_logger
from metadata.utils.tag_utils import get_ometa_tag_and_classification, get_tag_labels
from metadata.utils.time_utils import convert_timestamp_to_milliseconds
logger = ingestion_logger()
@ -845,7 +846,9 @@ class DbtSource(DbtServiceSource):
testCaseFailureReason=None,
testCaseFailureComment=None,
updatedAt=Timestamp(
__root__=int(datetime.utcnow().timestamp() * 1000)
__root__=convert_timestamp_to_milliseconds(
datetime.utcnow().timestamp()
)
),
updatedBy=None,
)
@ -863,7 +866,7 @@ class DbtSource(DbtServiceSource):
dbt_timestamp = self.context.run_results_generate_time.timestamp()
# Create the test case result object
test_case_result = TestCaseResult(
timestamp=dbt_timestamp,
timestamp=convert_timestamp_to_milliseconds(dbt_timestamp),
testCaseStatus=test_case_status,
testResultValue=[
TestResultValue(