mirror of
https://github.com/datahub-project/datahub.git
synced 2025-07-06 16:49:03 +00:00
82 lines
2.8 KiB
Python
82 lines
2.8 KiB
Python
import subprocess
|
|
from typing import List
|
|
|
|
import pytest
|
|
from freezegun import freeze_time
|
|
|
|
from tests.test_helpers import mce_helpers
|
|
from tests.test_helpers.click_helpers import run_datahub_cmd
|
|
from tests.test_helpers.docker_helpers import cleanup_image, wait_for_port
|
|
|
|
FROZEN_TIME = "2020-04-14 07:00:00"
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def test_resources_dir(pytestconfig):
|
|
return pytestconfig.rootpath / "tests/integration/vertica"
|
|
|
|
|
|
def is_vertica_responsive(container_name: str) -> bool:
|
|
cmd = f"docker logs {container_name} 2>&1 | grep 'Vertica is now running' "
|
|
ret = subprocess.run(
|
|
cmd,
|
|
shell=True,
|
|
)
|
|
return ret.returncode == 0
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def vertica_runner(docker_compose_runner, test_resources_dir):
|
|
with docker_compose_runner(
|
|
test_resources_dir / "docker-compose.yml", "vertica"
|
|
) as docker_services:
|
|
wait_for_port(
|
|
docker_services,
|
|
"vertica-ce",
|
|
5433,
|
|
timeout=120,
|
|
checker=lambda: is_vertica_responsive("vertica-ce"),
|
|
)
|
|
|
|
commands = """
|
|
docker cp tests/integration/vertica/ddl.sql vertica-ce:/home/dbadmin/ &&
|
|
docker exec vertica-ce sh -c "/opt/vertica/bin/vsql -w abc123 -f /home/dbadmin/ddl.sql"
|
|
"""
|
|
|
|
ret = subprocess.run(commands, shell=True, stdout=subprocess.DEVNULL)
|
|
|
|
assert ret.returncode == 0
|
|
|
|
yield docker_services
|
|
|
|
# The image is pretty large, so we remove it after the test.
|
|
cleanup_image("vertica/vertica-ce")
|
|
|
|
|
|
@freeze_time(FROZEN_TIME)
|
|
@pytest.mark.integration
|
|
def test_vertica_ingest_with_db(vertica_runner, pytestconfig, tmp_path):
|
|
test_resources_dir = pytestconfig.rootpath / "tests/integration/vertica"
|
|
|
|
# Run the metadata ingestion pipeline.
|
|
config_file = (test_resources_dir / "vertica_to_file.yml").resolve()
|
|
run_datahub_cmd(
|
|
["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path
|
|
)
|
|
|
|
ignore_paths: List[str] = [
|
|
r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['create_time'\]",
|
|
r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['table_size'\]",
|
|
r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['Projection_size'\]",
|
|
r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['ROS_Count'\]",
|
|
r"root\[\d+\]\['aspect'\].+\['customProperties'\]\['cluster_size'\]",
|
|
r"root\[\d+\]\['aspect'\].+\['customProperties'\]\['udx_language'\]",
|
|
]
|
|
# Verify the output.
|
|
mce_helpers.check_golden_file(
|
|
pytestconfig,
|
|
ignore_paths=ignore_paths,
|
|
output_path=tmp_path / "vertica.json",
|
|
golden_path=test_resources_dir / "vertica_mces_with_db_golden.json",
|
|
)
|