diff --git a/ingestion/src/metadata/data_insight/processor/kpi/kpi_runner.py b/ingestion/src/metadata/data_insight/processor/kpi/kpi_runner.py index a5a5145815a..b332ed67f40 100644 --- a/ingestion/src/metadata/data_insight/processor/kpi/kpi_runner.py +++ b/ingestion/src/metadata/data_insight/processor/kpi/kpi_runner.py @@ -15,7 +15,7 @@ Runner class used to check KPI status from __future__ import annotations import time as tme -from datetime import datetime +from datetime import datetime, timezone from typing import Iterator, Optional from metadata.data_insight.processor.kpi.run_result_registry import run_result_registry @@ -51,7 +51,7 @@ class KpiRunner: def __init__(self, metadata: OpenMetadata) -> None: self.metadata = metadata - self.datetime = int(datetime.utcnow().timestamp() * 1000) + self.datetime = int(datetime.now(timezone.utc).timestamp() * 1000) self.processor_status = Status() self.kpis = self.get_active_kpis() diff --git a/ingestion/src/metadata/data_insight/source/metadata.py b/ingestion/src/metadata/data_insight/source/metadata.py index 47ab829600b..15f2b86e480 100644 --- a/ingestion/src/metadata/data_insight/source/metadata.py +++ b/ingestion/src/metadata/data_insight/source/metadata.py @@ -13,7 +13,7 @@ OpenMetadata source for the data insight workflow """ import traceback -from datetime import datetime +from datetime import datetime, timezone from types import MappingProxyType from typing import Dict, Iterable, Optional, Union, cast @@ -73,7 +73,7 @@ class DataInsightSource(Source): """Instantiate source object""" super().__init__() self.metadata = metadata - self.date = datetime.utcnow().strftime("%Y-%m-%d") + self.date = datetime.now(timezone.utc).strftime("%Y-%m-%d") self.entities_cache = {} _processors = self._instantiate_processors() diff --git a/ingestion/src/metadata/data_quality/validations/table/pandas/tableRowInsertedCountToBeBetween.py b/ingestion/src/metadata/data_quality/validations/table/pandas/tableRowInsertedCountToBeBetween.py index 61568ae98d8..ccea8742301 100644 --- a/ingestion/src/metadata/data_quality/validations/table/pandas/tableRowInsertedCountToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/table/pandas/tableRowInsertedCountToBeBetween.py @@ -13,7 +13,7 @@ Validator for table row inserted count to be between test case """ -from datetime import datetime +from datetime import datetime, timezone from dateutil.relativedelta import relativedelta @@ -47,7 +47,7 @@ class TableRowInsertedCountToBeBetweenValidator( "MONTH": relativedelta(months=range_interval), "YEAR": relativedelta(years=range_interval), } - utc_now = datetime.utcnow() + utc_now = datetime.now(timezone.utc) threshold_date = utc_now - interval_type_matching_table[range_type] if range_type == "HOUR": threshold_date = threshold_date.replace(minute=0, second=0, microsecond=0) diff --git a/ingestion/src/metadata/ingestion/ometa/client.py b/ingestion/src/metadata/ingestion/ometa/client.py index 50627c79253..55422dc8aff 100644 --- a/ingestion/src/metadata/ingestion/ometa/client.py +++ b/ingestion/src/metadata/ingestion/ometa/client.py @@ -11,9 +11,9 @@ """ Python API REST wrapper and helpers """ -import datetime import time import traceback +from datetime import datetime, timezone from typing import Callable, Dict, List, Optional, Union import requests @@ -145,16 +145,16 @@ class REST: url: URL = URL(base_url + "/" + version + path) if ( self.config.expires_in - and datetime.datetime.utcnow().timestamp() >= self.config.expires_in + and datetime.now(timezone.utc).timestamp() >= self.config.expires_in or not self.config.access_token ): self.config.access_token, expiry = self._auth_token() if not self.config.access_token == "no_token": - if isinstance(expiry, datetime.datetime): + if isinstance(expiry, datetime): self.config.expires_in = expiry.timestamp() - 120 else: self.config.expires_in = ( - datetime.datetime.utcnow().timestamp() + expiry - 120 + datetime.now(timezone.utc).timestamp() + expiry - 120 ) headers[self.config.auth_header] = ( diff --git a/ingestion/src/metadata/ingestion/source/database/query/usage.py b/ingestion/src/metadata/ingestion/source/database/query/usage.py index ee93b15f236..f65c6cc212e 100644 --- a/ingestion/src/metadata/ingestion/source/database/query/usage.py +++ b/ingestion/src/metadata/ingestion/source/database/query/usage.py @@ -11,7 +11,7 @@ """ Common Query Log Connector """ -from datetime import datetime +from datetime import datetime, timezone from typing import Optional from metadata.generated.schema.metadataIngestion.workflow import ( @@ -24,7 +24,9 @@ from metadata.ingestion.source.database.usage_source import UsageSource class QueryLogUsageSource(UsageSource): def __init__(self, config: WorkflowSource, metadata: OpenMetadata): super().__init__(config, metadata) - self.analysis_date = datetime.utcnow().date().strftime("%Y-%m-%d %H:%M:%S") + self.analysis_date = ( + datetime.now(timezone.utc).date().strftime("%Y-%m-%d %H:%M:%S") + ) @classmethod def create( diff --git a/ingestion/src/metadata/ingestion/source/database/sample_data.py b/ingestion/src/metadata/ingestion/source/database/sample_data.py index 3951410832f..334eb37afc9 100644 --- a/ingestion/src/metadata/ingestion/source/database/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/database/sample_data.py @@ -18,7 +18,7 @@ import string import time import traceback from collections import namedtuple -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Any, Dict, Iterable, List, Optional, Union from pydantic import ValidationError @@ -1552,9 +1552,10 @@ class SampleDataSource( for report_datum in report_data: if report_type == ReportDataType.rawCostAnalysisReportData.value: start_ts = int( - (datetime.utcnow() - timedelta(days=60)).timestamp() * 1000 + (datetime.now(timezone.utc) - timedelta(days=60)).timestamp() + * 1000 ) - end_ts = int(datetime.utcnow().timestamp() * 1000) + end_ts = int(datetime.now(timezone.utc).timestamp() * 1000) tmstp = random.randint(start_ts, end_ts) report_datum["data"]["lifeCycle"]["accessed"]["timestamp"] = tmstp record = OMetaDataInsightSample( diff --git a/ingestion/src/metadata/ingestion/source/database/usage_source.py b/ingestion/src/metadata/ingestion/source/database/usage_source.py index 769f1c125ed..c799b966bdf 100644 --- a/ingestion/src/metadata/ingestion/source/database/usage_source.py +++ b/ingestion/src/metadata/ingestion/source/database/usage_source.py @@ -14,7 +14,7 @@ Usage Source Module import csv import traceback from abc import ABC -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Iterable from metadata.generated.schema.type.basic import DateTime @@ -46,7 +46,7 @@ class UsageSource(QueryParserSource, ABC): query_dict = dict(record) analysis_date = ( - datetime.utcnow() + datetime.now(timezone.utc) if not query_dict.get("start_time") else datetime.strptime( query_dict.get("start_time"), "%Y-%m-%d %H:%M:%S.%f" diff --git a/ingestion/src/metadata/utils/helpers.py b/ingestion/src/metadata/utils/helpers.py index af6738b14ce..537c20607fa 100644 --- a/ingestion/src/metadata/utils/helpers.py +++ b/ingestion/src/metadata/utils/helpers.py @@ -19,7 +19,7 @@ import itertools import re import shutil import sys -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from math import floor, log from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Tuple, Union @@ -123,7 +123,7 @@ def get_start_and_end(duration: int = 0): Method to return start and end time based on duration """ - today = datetime.utcnow() + today = datetime.now(timezone.utc) start = (today + timedelta(0 - duration)).replace( hour=0, minute=0, second=0, microsecond=0 ) diff --git a/ingestion/src/metadata/utils/time_utils.py b/ingestion/src/metadata/utils/time_utils.py index 55493e3d698..1d816ef6442 100644 --- a/ingestion/src/metadata/utils/time_utils.py +++ b/ingestion/src/metadata/utils/time_utils.py @@ -55,7 +55,7 @@ def get_beginning_of_day_timestamp_mill( Returns: int: timestamp milliseconds """ - now_utc = datetime.utcnow() + now_utc = datetime.now(timezone.utc) delta = timedelta( weeks=weeks, days=days, @@ -87,7 +87,7 @@ def get_end_of_day_timestamp_mill( Returns: int: timestamp milliseconds """ - now_utc = datetime.utcnow() + now_utc = datetime.now(timezone.utc) delta = timedelta( weeks=weeks, days=days, diff --git a/ingestion/tests/integration/data_insight/test_data_insight_workflow.py b/ingestion/tests/integration/data_insight/test_data_insight_workflow.py index 9940d1967c2..95176617a1f 100644 --- a/ingestion/tests/integration/data_insight/test_data_insight_workflow.py +++ b/ingestion/tests/integration/data_insight/test_data_insight_workflow.py @@ -19,7 +19,7 @@ import random import unittest import uuid from copy import deepcopy -from datetime import datetime, time, timedelta +from datetime import datetime, time, timedelta, timezone from random import randint from time import sleep @@ -136,10 +136,10 @@ class DataInsightWorkflowTests(unittest.TestCase): def setUp(self) -> None: """Set up om client for the test class""" self.start_ts = int( - datetime.combine(datetime.utcnow(), time.min).timestamp() * 1000 + datetime.combine(datetime.now(timezone.utc), time.min).timestamp() * 1000 ) - random.randint(1, 999) self.end_ts = int( - datetime.combine(datetime.utcnow(), time.max).timestamp() * 1000 + datetime.combine(datetime.now(timezone.utc), time.max).timestamp() * 1000 ) - random.randint(1, 999) completed_description_chart = self.metadata.get_by_name( @@ -177,7 +177,7 @@ class DataInsightWorkflowTests(unittest.TestCase): event.timestamp = Timestamp( int( ( - datetime.utcnow() + datetime.now(timezone.utc) - timedelta(days=1, milliseconds=randint(0, 999)) ).timestamp() * 1000 @@ -192,7 +192,7 @@ class DataInsightWorkflowTests(unittest.TestCase): timestamp=Timestamp( int( ( - datetime.utcnow() + datetime.now(timezone.utc) - timedelta(days=1, milliseconds=randint(0, 999)) ).timestamp() * 1000 @@ -341,7 +341,7 @@ class DataInsightWorkflowTests(unittest.TestCase): self.metadata.add_kpi_result( fqn, KpiResult( - timestamp=int(datetime.utcnow().timestamp() * 1000), + timestamp=int(datetime.now(timezone.utc).timestamp() * 1000), kpiFqn="CompletedDescription__test_write_kpi_result", targetResult=[ KpiTarget( diff --git a/ingestion/tests/integration/data_insight/test_web_analytic_events.py b/ingestion/tests/integration/data_insight/test_web_analytic_events.py index bef6d176966..c0a080f4be0 100644 --- a/ingestion/tests/integration/data_insight/test_web_analytic_events.py +++ b/ingestion/tests/integration/data_insight/test_web_analytic_events.py @@ -17,7 +17,7 @@ from __future__ import annotations import unittest import uuid -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from random import randint from metadata.generated.schema.analytics.basic import WebAnalyticEventType @@ -75,7 +75,9 @@ class WebAnalyticsEndpointsTests(unittest.TestCase): cls.start_ts = get_beginning_of_day_timestamp_mill(days=1) cls.end_ts = get_end_of_day_timestamp_mill(days=1) - cls.yesterday = int((datetime.utcnow() - timedelta(days=1)).timestamp() * 1000) + cls.yesterday = int( + (datetime.now(timezone.utc) - timedelta(days=1)).timestamp() * 1000 + ) def test_web_analytic_events(self): """Test web analytic get function""" @@ -122,7 +124,7 @@ class WebAnalyticsEndpointsTests(unittest.TestCase): for delta in range(7): delta = timedelta(days=delta, milliseconds=randint(100, 999)) - tmsp = datetime_to_ts(datetime.utcnow() - delta) + tmsp = datetime_to_ts(datetime.now(timezone.utc) - delta) user_id = uuid.uuid4() session_id = uuid.uuid4() @@ -149,21 +151,21 @@ class WebAnalyticsEndpointsTests(unittest.TestCase): self.metadata.delete_web_analytic_event_before_ts_exclusive( WebAnalyticEventType.PageView, - int((datetime.utcnow() - timedelta(days=3)).timestamp() * 1000), + int((datetime.now(timezone.utc) - timedelta(days=3)).timestamp() * 1000), ) event = self.metadata.get_web_analytic_events( WebAnalyticEventType.PageView, - int((datetime.utcnow() - timedelta(days=2)).timestamp() * 1000), - int((datetime.utcnow() - timedelta(days=1)).timestamp() * 1000), + int((datetime.now(timezone.utc) - timedelta(days=2)).timestamp() * 1000), + int((datetime.now(timezone.utc) - timedelta(days=1)).timestamp() * 1000), ) assert event empty_event = self.metadata.get_web_analytic_events( WebAnalyticEventType.PageView, - int((datetime.utcnow() - timedelta(days=7)).timestamp() * 1000), - int((datetime.utcnow() - timedelta(days=6)).timestamp() * 1000), + int((datetime.now(timezone.utc) - timedelta(days=7)).timestamp() * 1000), + int((datetime.now(timezone.utc) - timedelta(days=6)).timestamp() * 1000), ) assert not empty_event diff --git a/ingestion/tests/integration/ometa/test_ometa_test_suite.py b/ingestion/tests/integration/ometa/test_ometa_test_suite.py index e1f646fcc07..c9391066a7b 100644 --- a/ingestion/tests/integration/ometa/test_ometa_test_suite.py +++ b/ingestion/tests/integration/ometa/test_ometa_test_suite.py @@ -12,7 +12,7 @@ """ OpenMetadata API test suite mixin test """ -from datetime import datetime +from datetime import datetime, timezone from unittest import TestCase from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest @@ -120,7 +120,7 @@ class OMetaTestSuiteTest(TestCase): cls.metadata.add_test_case_results( test_results=TestCaseResult( - timestamp=datetime_to_ts(datetime.utcnow()), + timestamp=datetime_to_ts(datetime.now(timezone.utc)), testCaseStatus=TestCaseStatus.Success, result="Test Case Success", sampleData=None, diff --git a/ingestion/tests/unit/test_time_utils.py b/ingestion/tests/unit/test_time_utils.py index 4becde87743..c30d72d2b8e 100644 --- a/ingestion/tests/unit/test_time_utils.py +++ b/ingestion/tests/unit/test_time_utils.py @@ -25,11 +25,11 @@ NOW_UTC = datetime(2022, 11, 15, 10, 30, 45, 776132) @patch("metadata.utils.time_utils.datetime", wraps=datetime) def test_get_beginning_of_day_timestamp_mill(mock_dt): - mock_dt.utcnow = mock.Mock(return_value=NOW_UTC) + mock_dt.now = mock.Mock(return_value=NOW_UTC) assert get_beginning_of_day_timestamp_mill() == 1668470400000 @patch("metadata.utils.time_utils.datetime", wraps=datetime) def test_get_end_of_day_timestamp_mill(mock_dt): - mock_dt.utcnow.return_value = NOW_UTC + mock_dt.now.return_value = NOW_UTC assert get_end_of_day_timestamp_mill() == 1668556799999 diff --git a/scripts/generate_docs_schemas.py b/scripts/generate_docs_schemas.py index 6ccfc583208..4376b13f791 100644 --- a/scripts/generate_docs_schemas.py +++ b/scripts/generate_docs_schemas.py @@ -27,7 +27,7 @@ which is incorrect and should be removed when pasting this in. import json import os -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import List @@ -42,7 +42,7 @@ PARSER = jsonschema2md.Parser( show_examples="all", ) -NOW = datetime.utcnow() +NOW = datetime.now(timezone.utc) def build_new_file(file: Path) -> Path: