diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml index ba0f91862d1..3e5607e80d6 100644 --- a/.github/workflows/metadata-ingestion.yml +++ b/.github/workflows/metadata-ingestion.yml @@ -55,7 +55,7 @@ jobs: strategy: matrix: python-version: ["3.6", "3.9"] - command: ["installAirflow1", "testIntegration", "testSlowIntegration"] + command: ["installAirflow1", "testIntegration", "testIntegrationBatch1", "testSlowIntegration"] fail-fast: false steps: - uses: actions/checkout@v2 diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle index 419722fdeea..a4c7ed10372 100644 --- a/metadata-ingestion/build.gradle +++ b/metadata-ingestion/build.gradle @@ -84,7 +84,7 @@ task testQuick(type: Exec, dependsOn: installDev) { inputs.files(project.fileTree(dir: "tests/")) outputs.dir("${venv_name}") commandLine 'bash', '-x', '-c', - "source ${venv_name}/bin/activate && pytest --durations=20 -m 'not slow_unit and not integration and not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" + "source ${venv_name}/bin/activate && pytest --durations=20 -m 'not integration and not integration_batch_1 and not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" } task installDevTest(type: Exec, dependsOn: [install]) { @@ -119,7 +119,12 @@ task installAirflow1(type: Exec, dependsOn: [install]) { task testIntegration(type: Exec, dependsOn: [installDevTest]) { commandLine 'bash', '-x', '-c', - "source ${venv_name}/bin/activate && pytest --durations=50 -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.integration.xml" + "source ${venv_name}/bin/activate && pytest --durations=50 -m 'integration' -vv --continue-on-collection-errors --junit-xml=junit.integration.xml" +} + +task testIntegrationBatch1(type: Exec, dependsOn: [installDevTest]) { + commandLine 'bash', '-x', '-c', + "source ${venv_name}/bin/activate && pytest --durations=50 -m 'integration_batch_1' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch1.xml" } task testFull(type: Exec, dependsOn: [installDevTest]) { diff --git a/metadata-ingestion/setup.cfg b/metadata-ingestion/setup.cfg index 9aaa70fdc1c..4bd5f1771dd 100644 --- a/metadata-ingestion/setup.cfg +++ b/metadata-ingestion/setup.cfg @@ -56,6 +56,7 @@ addopts = --cov=src --cov-report term-missing --cov-config setup.cfg --strict-ma markers = slow_unit: marks tests to only run slow unit tests (deselect with '-m not slow_unit') integration: marks tests to only run in integration (deselect with '-m "not integration"') + integration_batch_1: mark tests to only run in batch 1 of integration tests. This is done mainly for parallelisation (deselect with '-m not integration_batch_1') slow_integration: marks tests that are too slow to even run in integration (deselect with '-m "not slow_integration"') testpaths = tests/unit diff --git a/metadata-ingestion/tests/integration/data_lake/test_data_lake.py b/metadata-ingestion/tests/integration/data_lake/test_data_lake.py index d49b950fb8a..4d51d5d16ff 100644 --- a/metadata-ingestion/tests/integration/data_lake/test_data_lake.py +++ b/metadata-ingestion/tests/integration/data_lake/test_data_lake.py @@ -6,7 +6,7 @@ from tests.test_helpers import mce_helpers FROZEN_TIME = "2020-04-14 07:00:00" -@pytest.mark.integration +@pytest.mark.integration_batch_1 def test_data_lake_ingest(pytestconfig, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/data_lake/" diff --git a/metadata-ingestion/tests/integration/feast-legacy/test_feast.py b/metadata-ingestion/tests/integration/feast-legacy/test_feast.py index 2f8a9abe04c..ce55ca336f1 100644 --- a/metadata-ingestion/tests/integration/feast-legacy/test_feast.py +++ b/metadata-ingestion/tests/integration/feast-legacy/test_feast.py @@ -13,7 +13,7 @@ FROZEN_TIME = "2020-04-14 07:00:00" # make sure that mock_time is excluded here because it messes with feast @freeze_time(FROZEN_TIME) -@pytest.mark.integration +@pytest.mark.integration_batch_1 def test_feast_ingest(docker_compose_runner, pytestconfig, tmp_path): test_resources_dir = pytestconfig.rootpath / "tests/integration/feast-legacy" diff --git a/metadata-ingestion/tests/integration/hive/test_hive.py b/metadata-ingestion/tests/integration/hive/test_hive.py index d24f6c7de0d..89853cf7bfe 100644 --- a/metadata-ingestion/tests/integration/hive/test_hive.py +++ b/metadata-ingestion/tests/integration/hive/test_hive.py @@ -55,7 +55,7 @@ def base_pipeline_config(events_file): @freeze_time(FROZEN_TIME) -@pytest.mark.integration +@pytest.mark.integration_batch_1 def test_hive_ingest( loaded_hive, pytestconfig, test_resources_dir, tmp_path, mock_time ): @@ -85,7 +85,7 @@ def test_hive_ingest( @freeze_time(FROZEN_TIME) -@pytest.mark.integration +@pytest.mark.integration_batch_1 def test_hive_instance_check(loaded_hive, test_resources_dir, tmp_path, pytestconfig): instance: str = "production_warehouse" diff --git a/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py b/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py index 2ae90a62630..4dd54305ec8 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py +++ b/metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py @@ -12,7 +12,7 @@ FROZEN_TIME = "2021-10-25 13:00:00" @freeze_time(FROZEN_TIME) -@pytest.mark.integration +@pytest.mark.integration_batch_1 def test_kafka_connect_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/kafka-connect" test_resources_dir_kafka = pytestconfig.rootpath / "tests/integration/kafka" diff --git a/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py b/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py index 0c639da5991..905bf4f7ca0 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py +++ b/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py @@ -52,7 +52,7 @@ def loaded_presto_on_hive(presto_on_hive_runner): @freeze_time(FROZEN_TIME) -@pytest.mark.integration +@pytest.mark.integration_batch_1 def test_presto_on_hive_ingest( loaded_presto_on_hive, test_resources_dir, pytestconfig, tmp_path, mock_time ): @@ -112,7 +112,7 @@ def test_presto_on_hive_ingest( @freeze_time(FROZEN_TIME) -@pytest.mark.integration +@pytest.mark.integration_batch_1 def test_presto_on_hive_instance_ingest( loaded_presto_on_hive, test_resources_dir, pytestconfig, tmp_path, mock_time ):