mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-10-30 18:17:53 +00:00
Convert dbt test result timestamps to millis (#13903)
* Fixed dbt test res time to millis * added postgres migration
This commit is contained in:
parent
71a67d6bbe
commit
b7e7625071
@ -0,0 +1,11 @@
|
|||||||
|
|
||||||
|
--update the timestamps to millis for dbt test results
|
||||||
|
UPDATE data_quality_data_time_series dqdts
|
||||||
|
SET dqdts.json = JSON_INSERT(
|
||||||
|
JSON_REMOVE(dqdts.json, '$.timestamp'),
|
||||||
|
'$.timestamp',
|
||||||
|
JSON_EXTRACT(dqdts.json, '$.timestamp') * 1000
|
||||||
|
)
|
||||||
|
WHERE dqdts.extension = 'testCase.testCaseResult'
|
||||||
|
AND JSON_EXTRACT(dqdts.json, '$.timestamp') REGEXP '^[0-9]{10}$'
|
||||||
|
;
|
||||||
@ -9,3 +9,14 @@ SET json = jsonb_set(
|
|||||||
)
|
)
|
||||||
WHERE json #>> '{pipelineType}' = 'metadata'
|
WHERE json #>> '{pipelineType}' = 'metadata'
|
||||||
AND json #>> '{sourceConfig,config,type}' = 'DatabaseMetadata';
|
AND json #>> '{sourceConfig,config,type}' = 'DatabaseMetadata';
|
||||||
|
|
||||||
|
|
||||||
|
--update the timestamps to millis for dbt test results
|
||||||
|
UPDATE data_quality_data_time_series dqdts
|
||||||
|
SET json = jsonb_set(
|
||||||
|
dqdts.json::jsonb,
|
||||||
|
'{timestamp}',
|
||||||
|
to_jsonb(((dqdts.json ->> 'timestamp')::bigint)*1000)
|
||||||
|
)
|
||||||
|
WHERE dqdts.extension = 'testCase.testCaseResult'
|
||||||
|
AND (json->>'timestamp') ~ '^[0-9]{10}$';
|
||||||
|
|||||||
@ -86,6 +86,7 @@ from metadata.utils import fqn
|
|||||||
from metadata.utils.elasticsearch import get_entity_from_es_result
|
from metadata.utils.elasticsearch import get_entity_from_es_result
|
||||||
from metadata.utils.logger import ingestion_logger
|
from metadata.utils.logger import ingestion_logger
|
||||||
from metadata.utils.tag_utils import get_ometa_tag_and_classification, get_tag_labels
|
from metadata.utils.tag_utils import get_ometa_tag_and_classification, get_tag_labels
|
||||||
|
from metadata.utils.time_utils import convert_timestamp_to_milliseconds
|
||||||
|
|
||||||
logger = ingestion_logger()
|
logger = ingestion_logger()
|
||||||
|
|
||||||
@ -845,7 +846,9 @@ class DbtSource(DbtServiceSource):
|
|||||||
testCaseFailureReason=None,
|
testCaseFailureReason=None,
|
||||||
testCaseFailureComment=None,
|
testCaseFailureComment=None,
|
||||||
updatedAt=Timestamp(
|
updatedAt=Timestamp(
|
||||||
__root__=int(datetime.utcnow().timestamp() * 1000)
|
__root__=convert_timestamp_to_milliseconds(
|
||||||
|
datetime.utcnow().timestamp()
|
||||||
|
)
|
||||||
),
|
),
|
||||||
updatedBy=None,
|
updatedBy=None,
|
||||||
)
|
)
|
||||||
@ -863,7 +866,7 @@ class DbtSource(DbtServiceSource):
|
|||||||
dbt_timestamp = self.context.run_results_generate_time.timestamp()
|
dbt_timestamp = self.context.run_results_generate_time.timestamp()
|
||||||
# Create the test case result object
|
# Create the test case result object
|
||||||
test_case_result = TestCaseResult(
|
test_case_result = TestCaseResult(
|
||||||
timestamp=dbt_timestamp,
|
timestamp=convert_timestamp_to_milliseconds(dbt_timestamp),
|
||||||
testCaseStatus=test_case_status,
|
testCaseStatus=test_case_status,
|
||||||
testResultValue=[
|
testResultValue=[
|
||||||
TestResultValue(
|
TestResultValue(
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user