datahub/metadata-ingestion/tests/integration/vertica/test_vertica.py

import subprocess
from typing import List

import pytest
from freezegun import freeze_time

from tests.test_helpers import mce_helpers
from tests.test_helpers.click_helpers import run_datahub_cmd
from tests.test_helpers.docker_helpers import cleanup_image, wait_for_port

FROZEN_TIME = "2020-04-14 07:00:00"


@pytest.fixture(scope="module")
def test_resources_dir(pytestconfig):
    return pytestconfig.rootpath / "tests/integration/vertica"


def is_vertica_responsive(container_name: str) -> bool:
    cmd = f"docker logs {container_name} 2>&1 | grep 'Vertica is now running' "
    ret = subprocess.run(
        cmd,
        shell=True,
    )
    return ret.returncode == 0


@pytest.fixture(scope="module")
def vertica_runner(docker_compose_runner, test_resources_dir):
    with docker_compose_runner(
        test_resources_dir / "docker-compose.yml", "vertica"
    ) as docker_services:
        wait_for_port(
            docker_services,
            "vertica-ce",
            5433,
            timeout=120,
            checker=lambda: is_vertica_responsive("vertica-ce"),
        )

        commands = """
                    docker cp tests/integration/vertica/ddl.sql vertica-ce:/home/dbadmin/ &&
                    docker exec vertica-ce sh -c "/opt/vertica/bin/vsql -w abc123 -f /home/dbadmin/ddl.sql"
                """

        ret = subprocess.run(commands, shell=True, stdout=subprocess.DEVNULL)

        assert ret.returncode == 0

        yield docker_services

    # The image is pretty large, so we remove it after the test.
    cleanup_image("vertica/vertica-ce")


@freeze_time(FROZEN_TIME)
@pytest.mark.integration
def test_vertica_ingest_with_db(vertica_runner, pytestconfig, tmp_path):
    test_resources_dir = pytestconfig.rootpath / "tests/integration/vertica"

    # Run the metadata ingestion pipeline.
    config_file = (test_resources_dir / "vertica_to_file.yml").resolve()
    run_datahub_cmd(
        ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path
    )

    ignore_paths: List[str] = [
        r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['create_time'\]",
        r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['table_size'\]",
        r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['Projection_size'\]",
        r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['ROS_Count'\]",
        r"root\[\d+\]\['aspect'\].+\['customProperties'\]\['cluster_size'\]",
        r"root\[\d+\]\['aspect'\].+\['customProperties'\]\['udx_language'\]",
    ]
    # Verify the output.
    mce_helpers.check_golden_file(
        pytestconfig,
        ignore_paths=ignore_paths,
        output_path=tmp_path / "vertica.json",
        golden_path=test_resources_dir / "vertica_mces_with_db_golden.json",
    )