yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-02 11:49:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(ingest): limit GMS retries in test (#5509)

Browse Source
This commit is contained in:
Harshal Sheth 2022-07-29 09:41:31 +00:00 committed by GitHub
parent c0eb8a1b8b
commit 55cb34e5d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 19 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
metadata-ingestion/src/datahub/emitter/rest_emitter.py
View File

@ -2,6 +2,7 @@ import datetime
import functools
import json
import logging
import os
from json.decoder import JSONDecodeError
from typing import Any, Dict, List, Optional, Tuple, Union
@ -35,7 +36,9 @@ class DataHubRestEmitter:
504,
]
DEFAULT_RETRY_METHODS = ["HEAD", "GET", "POST", "PUT", "DELETE", "OPTIONS", "TRACE"]
DEFAULT_RETRY_MAX_TIMES = 3
DEFAULT_RETRY_MAX_TIMES = int(
os.getenv("DATAHUB_REST_EMITTER_DEFAULT_RETRY_MAX_TIMES", "3")
)
_gms_server: str
_token: Optional[str]

24
metadata-ingestion/tests/conftest.py
View File

@ -4,8 +4,21 @@ import time
import pytest
from tests.test_helpers.docker_helpers import docker_compose_runner # noqa: F401
from tests.test_helpers.state_helpers import mock_datahub_graph # noqa: F401
# Enable debug logging.
logging.getLogger().setLevel(logging.DEBUG)
os.environ["DATAHUB_DEBUG"] = "1"
# Disable telemetry
os.environ["DATAHUB_TELEMETRY_ENABLED"] = "false"
# Reduce retries on GMS, because this causes tests to hang while sleeping
# between retries.
os.environ["DATAHUB_REST_EMITTER_DEFAULT_RETRY_MAX_TIMES"] = "1"
# We need our imports to go below the os.environ updates, since mere act
# of importing some datahub modules will load env variables.
from tests.test_helpers.docker_helpers import docker_compose_runner # noqa: F401,E402
from tests.test_helpers.state_helpers import mock_datahub_graph # noqa: F401,E402
try:
# See https://github.com/spulec/freezegun/issues/98#issuecomment-590553475.
@ -13,13 +26,6 @@ try:
except ImportError:
pass
# Enable debug logging.
logging.getLogger().setLevel(logging.DEBUG)
os.putenv("DATAHUB_DEBUG", "1")
# Disable telemetry
os.putenv("DATAHUB_TELEMETRY_ENABLED", "false")
@pytest.fixture
def mock_time(monkeypatch):