yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-17 03:43:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(ingest): limit GMS retries in test (#5509)

Browse Source
This commit is contained in:
Harshal Sheth 2022-07-29 09:41:31 +00:00 committed by GitHub
parent c0eb8a1b8b
commit 55cb34e5d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 19 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
metadata-ingestion/src/datahub/emitter/rest_emitter.py
View File

@ -2,6 +2,7 @@ import datetime
import functools import functools
import json import json
import logging import logging
import os
from json.decoder import JSONDecodeError from json.decoder import JSONDecodeError
from typing import Any, Dict, List, Optional, Tuple, Union from typing import Any, Dict, List, Optional, Tuple, Union
@ -35,7 +36,9 @@ class DataHubRestEmitter:
504, 504,
] ]
DEFAULT_RETRY_METHODS = ["HEAD", "GET", "POST", "PUT", "DELETE", "OPTIONS", "TRACE"] DEFAULT_RETRY_METHODS = ["HEAD", "GET", "POST", "PUT", "DELETE", "OPTIONS", "TRACE"]
DEFAULT_RETRY_MAX_TIMES = 3 DEFAULT_RETRY_MAX_TIMES = int(
os.getenv("DATAHUB_REST_EMITTER_DEFAULT_RETRY_MAX_TIMES", "3")
)
_gms_server: str _gms_server: str
_token: Optional[str] _token: Optional[str]

24
metadata-ingestion/tests/conftest.py
View File

@ -4,8 +4,21 @@ import time
import pytest import pytest
from tests.test_helpers.docker_helpers import docker_compose_runner # noqa: F401 # Enable debug logging.
from tests.test_helpers.state_helpers import mock_datahub_graph # noqa: F401 logging.getLogger().setLevel(logging.DEBUG)
os.environ["DATAHUB_DEBUG"] = "1"
# Disable telemetry
os.environ["DATAHUB_TELEMETRY_ENABLED"] = "false"
# Reduce retries on GMS, because this causes tests to hang while sleeping
# between retries.
os.environ["DATAHUB_REST_EMITTER_DEFAULT_RETRY_MAX_TIMES"] = "1"
# We need our imports to go below the os.environ updates, since mere act
# of importing some datahub modules will load env variables.
from tests.test_helpers.docker_helpers import docker_compose_runner # noqa: F401,E402
from tests.test_helpers.state_helpers import mock_datahub_graph # noqa: F401,E402
try: try:
# See https://github.com/spulec/freezegun/issues/98#issuecomment-590553475. # See https://github.com/spulec/freezegun/issues/98#issuecomment-590553475.
@ -13,13 +26,6 @@ try:
except ImportError: except ImportError:
pass pass
# Enable debug logging.
logging.getLogger().setLevel(logging.DEBUG)
os.putenv("DATAHUB_DEBUG", "1")
# Disable telemetry
os.putenv("DATAHUB_TELEMETRY_ENABLED", "false")
@pytest.fixture @pytest.fixture
def mock_time(monkeypatch): def mock_time(monkeypatch):