mirror of
				https://github.com/open-metadata/OpenMetadata.git
				synced 2025-10-31 10:39:30 +00:00 
			
		
		
		
	Convert dbt test result timestamps to millis (#13903)
* Fixed dbt test res time to millis * added postgres migration
This commit is contained in:
		
							parent
							
								
									71a67d6bbe
								
							
						
					
					
						commit
						b7e7625071
					
				| @ -0,0 +1,11 @@ | |||||||
|  | 
 | ||||||
|  | --update the timestamps to millis for dbt test results | ||||||
|  | UPDATE data_quality_data_time_series dqdts | ||||||
|  | SET dqdts.json = JSON_INSERT( | ||||||
|  |     JSON_REMOVE(dqdts.json, '$.timestamp'), | ||||||
|  |     '$.timestamp', | ||||||
|  |     JSON_EXTRACT(dqdts.json, '$.timestamp') * 1000 | ||||||
|  |  ) | ||||||
|  | WHERE dqdts.extension = 'testCase.testCaseResult' | ||||||
|  |   AND JSON_EXTRACT(dqdts.json, '$.timestamp') REGEXP '^[0-9]{10}$' | ||||||
|  | ; | ||||||
| @ -9,3 +9,14 @@ SET json = jsonb_set( | |||||||
| ) | ) | ||||||
| WHERE json #>> '{pipelineType}' = 'metadata' | WHERE json #>> '{pipelineType}' = 'metadata' | ||||||
| AND json #>> '{sourceConfig,config,type}' = 'DatabaseMetadata'; | AND json #>> '{sourceConfig,config,type}' = 'DatabaseMetadata'; | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | --update the timestamps to millis for dbt test results | ||||||
|  | UPDATE data_quality_data_time_series dqdts | ||||||
|  | SET json = jsonb_set( | ||||||
|  | 	dqdts.json::jsonb, | ||||||
|  | 	'{timestamp}', | ||||||
|  | 	to_jsonb(((dqdts.json ->> 'timestamp')::bigint)*1000) | ||||||
|  | ) | ||||||
|  | WHERE dqdts.extension = 'testCase.testCaseResult' | ||||||
|  |   AND (json->>'timestamp') ~ '^[0-9]{10}$'; | ||||||
|  | |||||||
| @ -86,6 +86,7 @@ from metadata.utils import fqn | |||||||
| from metadata.utils.elasticsearch import get_entity_from_es_result | from metadata.utils.elasticsearch import get_entity_from_es_result | ||||||
| from metadata.utils.logger import ingestion_logger | from metadata.utils.logger import ingestion_logger | ||||||
| from metadata.utils.tag_utils import get_ometa_tag_and_classification, get_tag_labels | from metadata.utils.tag_utils import get_ometa_tag_and_classification, get_tag_labels | ||||||
|  | from metadata.utils.time_utils import convert_timestamp_to_milliseconds | ||||||
| 
 | 
 | ||||||
| logger = ingestion_logger() | logger = ingestion_logger() | ||||||
| 
 | 
 | ||||||
| @ -845,7 +846,9 @@ class DbtSource(DbtServiceSource): | |||||||
|                         testCaseFailureReason=None, |                         testCaseFailureReason=None, | ||||||
|                         testCaseFailureComment=None, |                         testCaseFailureComment=None, | ||||||
|                         updatedAt=Timestamp( |                         updatedAt=Timestamp( | ||||||
|                             __root__=int(datetime.utcnow().timestamp() * 1000) |                             __root__=convert_timestamp_to_milliseconds( | ||||||
|  |                                 datetime.utcnow().timestamp() | ||||||
|  |                             ) | ||||||
|                         ), |                         ), | ||||||
|                         updatedBy=None, |                         updatedBy=None, | ||||||
|                     ) |                     ) | ||||||
| @ -863,7 +866,7 @@ class DbtSource(DbtServiceSource): | |||||||
|                     dbt_timestamp = self.context.run_results_generate_time.timestamp() |                     dbt_timestamp = self.context.run_results_generate_time.timestamp() | ||||||
|                 # Create the test case result object |                 # Create the test case result object | ||||||
|                 test_case_result = TestCaseResult( |                 test_case_result = TestCaseResult( | ||||||
|                     timestamp=dbt_timestamp, |                     timestamp=convert_timestamp_to_milliseconds(dbt_timestamp), | ||||||
|                     testCaseStatus=test_case_status, |                     testCaseStatus=test_case_status, | ||||||
|                     testResultValue=[ |                     testResultValue=[ | ||||||
|                         TestResultValue( |                         TestResultValue( | ||||||
|  | |||||||
		Loading…
	
	
			
			x
			
			
		
	
		Reference in New Issue
	
	Block a user
	 Onkar Ravgan
						Onkar Ravgan