diff --git a/bootstrap/sql/migrations/native/1.2.1/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.2.1/mysql/schemaChanges.sql index e69de29bb2d..581299a72e4 100644 --- a/bootstrap/sql/migrations/native/1.2.1/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.2.1/mysql/schemaChanges.sql @@ -0,0 +1,11 @@ + +--update the timestamps to millis for dbt test results +UPDATE data_quality_data_time_series dqdts +SET dqdts.json = JSON_INSERT( + JSON_REMOVE(dqdts.json, '$.timestamp'), + '$.timestamp', + JSON_EXTRACT(dqdts.json, '$.timestamp') * 1000 + ) +WHERE dqdts.extension = 'testCase.testCaseResult' + AND JSON_EXTRACT(dqdts.json, '$.timestamp') REGEXP '^[0-9]{10}$' +; \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.2.1/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.2.1/postgres/schemaChanges.sql index 16144ff5bdd..cac3a9ea8b4 100644 --- a/bootstrap/sql/migrations/native/1.2.1/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.2.1/postgres/schemaChanges.sql @@ -8,4 +8,15 @@ SET json = jsonb_set( true ) WHERE json #>> '{pipelineType}' = 'metadata' -AND json #>> '{sourceConfig,config,type}' = 'DatabaseMetadata'; \ No newline at end of file +AND json #>> '{sourceConfig,config,type}' = 'DatabaseMetadata'; + + +--update the timestamps to millis for dbt test results +UPDATE data_quality_data_time_series dqdts +SET json = jsonb_set( + dqdts.json::jsonb, + '{timestamp}', + to_jsonb(((dqdts.json ->> 'timestamp')::bigint)*1000) +) +WHERE dqdts.extension = 'testCase.testCaseResult' + AND (json->>'timestamp') ~ '^[0-9]{10}$'; diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py b/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py index a1d3862cab3..479f85c0843 100644 --- a/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py @@ -86,6 +86,7 @@ from metadata.utils import fqn from metadata.utils.elasticsearch import get_entity_from_es_result from metadata.utils.logger import ingestion_logger from metadata.utils.tag_utils import get_ometa_tag_and_classification, get_tag_labels +from metadata.utils.time_utils import convert_timestamp_to_milliseconds logger = ingestion_logger() @@ -845,7 +846,9 @@ class DbtSource(DbtServiceSource): testCaseFailureReason=None, testCaseFailureComment=None, updatedAt=Timestamp( - __root__=int(datetime.utcnow().timestamp() * 1000) + __root__=convert_timestamp_to_milliseconds( + datetime.utcnow().timestamp() + ) ), updatedBy=None, ) @@ -863,7 +866,7 @@ class DbtSource(DbtServiceSource): dbt_timestamp = self.context.run_results_generate_time.timestamp() # Create the test case result object test_case_result = TestCaseResult( - timestamp=dbt_timestamp, + timestamp=convert_timestamp_to_milliseconds(dbt_timestamp), testCaseStatus=test_case_status, testResultValue=[ TestResultValue(