yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-09 07:53:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ci): split integration tests to reduce run time (#5205)

Browse Source
This commit is contained in:
Aseem Bansal 2022-06-20 21:08:37 +05:30 committed by GitHub
parent d518b5a085
commit c942350e3c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/metadata-ingestion.yml vendored
View File

@ -55,7 +55,7 @@ jobs:
strategy: strategy:
matrix: matrix:
python-version: ["3.6", "3.9"] python-version: ["3.6", "3.9"]
command: ["installAirflow1", "testIntegration", "testSlowIntegration"] command: ["installAirflow1", "testIntegration", "testIntegrationBatch1", "testSlowIntegration"]
fail-fast: false fail-fast: false
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2

9
metadata-ingestion/build.gradle
View File

@ -84,7 +84,7 @@ task testQuick(type: Exec, dependsOn: installDev) {
inputs.files(project.fileTree(dir: "tests/")) inputs.files(project.fileTree(dir: "tests/"))
outputs.dir("${venv_name}") outputs.dir("${venv_name}")
commandLine 'bash', '-x', '-c', commandLine 'bash', '-x', '-c',
"source ${venv_name}/bin/activate && pytest --durations=20 -m 'not slow_unit and not integration and not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" "source ${venv_name}/bin/activate && pytest --durations=20 -m 'not integration and not integration_batch_1 and not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
} }
task installDevTest(type: Exec, dependsOn: [install]) { task installDevTest(type: Exec, dependsOn: [install]) {
@ -119,7 +119,12 @@ task installAirflow1(type: Exec, dependsOn: [install]) {
task testIntegration(type: Exec, dependsOn: [installDevTest]) { task testIntegration(type: Exec, dependsOn: [installDevTest]) {
commandLine 'bash', '-x', '-c', commandLine 'bash', '-x', '-c',
"source ${venv_name}/bin/activate && pytest --durations=50 -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.integration.xml" "source ${venv_name}/bin/activate && pytest --durations=50 -m 'integration' -vv --continue-on-collection-errors --junit-xml=junit.integration.xml"
}
task testIntegrationBatch1(type: Exec, dependsOn: [installDevTest]) {
commandLine 'bash', '-x', '-c',
"source ${venv_name}/bin/activate && pytest --durations=50 -m 'integration_batch_1' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch1.xml"
} }
task testFull(type: Exec, dependsOn: [installDevTest]) { task testFull(type: Exec, dependsOn: [installDevTest]) {

1
metadata-ingestion/setup.cfg
View File

@ -56,6 +56,7 @@ addopts = --cov=src --cov-report term-missing --cov-config setup.cfg --strict-ma
markers = markers =
slow_unit: marks tests to only run slow unit tests (deselect with '-m not slow_unit') slow_unit: marks tests to only run slow unit tests (deselect with '-m not slow_unit')
integration: marks tests to only run in integration (deselect with '-m "not integration"') integration: marks tests to only run in integration (deselect with '-m "not integration"')
integration_batch_1: mark tests to only run in batch 1 of integration tests. This is done mainly for parallelisation (deselect with '-m not integration_batch_1')
slow_integration: marks tests that are too slow to even run in integration (deselect with '-m "not slow_integration"') slow_integration: marks tests that are too slow to even run in integration (deselect with '-m "not slow_integration"')
testpaths = testpaths =
tests/unit tests/unit

2
metadata-ingestion/tests/integration/data_lake/test_data_lake.py
View File

@ -6,7 +6,7 @@ from tests.test_helpers import mce_helpers
FROZEN_TIME = "2020-04-14 07:00:00" FROZEN_TIME = "2020-04-14 07:00:00"
@pytest.mark.integration @pytest.mark.integration_batch_1
def test_data_lake_ingest(pytestconfig, tmp_path, mock_time): def test_data_lake_ingest(pytestconfig, tmp_path, mock_time):
test_resources_dir = pytestconfig.rootpath / "tests/integration/data_lake/" test_resources_dir = pytestconfig.rootpath / "tests/integration/data_lake/"

2
metadata-ingestion/tests/integration/feast-legacy/test_feast.py
View File

@ -13,7 +13,7 @@ FROZEN_TIME = "2020-04-14 07:00:00"
# make sure that mock_time is excluded here because it messes with feast # make sure that mock_time is excluded here because it messes with feast
@freeze_time(FROZEN_TIME) @freeze_time(FROZEN_TIME)
@pytest.mark.integration @pytest.mark.integration_batch_1
def test_feast_ingest(docker_compose_runner, pytestconfig, tmp_path): def test_feast_ingest(docker_compose_runner, pytestconfig, tmp_path):
test_resources_dir = pytestconfig.rootpath / "tests/integration/feast-legacy" test_resources_dir = pytestconfig.rootpath / "tests/integration/feast-legacy"

4
metadata-ingestion/tests/integration/hive/test_hive.py
View File

@ -55,7 +55,7 @@ def base_pipeline_config(events_file):
@freeze_time(FROZEN_TIME) @freeze_time(FROZEN_TIME)
@pytest.mark.integration @pytest.mark.integration_batch_1
def test_hive_ingest( def test_hive_ingest(
loaded_hive, pytestconfig, test_resources_dir, tmp_path, mock_time loaded_hive, pytestconfig, test_resources_dir, tmp_path, mock_time
): ):
@ -85,7 +85,7 @@ def test_hive_ingest(
@freeze_time(FROZEN_TIME) @freeze_time(FROZEN_TIME)
@pytest.mark.integration @pytest.mark.integration_batch_1
def test_hive_instance_check(loaded_hive, test_resources_dir, tmp_path, pytestconfig): def test_hive_instance_check(loaded_hive, test_resources_dir, tmp_path, pytestconfig):
instance: str = "production_warehouse" instance: str = "production_warehouse"

2
metadata-ingestion/tests/integration/kafka-connect/test_kafka_connect.py
View File

@ -12,7 +12,7 @@ FROZEN_TIME = "2021-10-25 13:00:00"
@freeze_time(FROZEN_TIME) @freeze_time(FROZEN_TIME)
@pytest.mark.integration @pytest.mark.integration_batch_1
def test_kafka_connect_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time): def test_kafka_connect_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time):
test_resources_dir = pytestconfig.rootpath / "tests/integration/kafka-connect" test_resources_dir = pytestconfig.rootpath / "tests/integration/kafka-connect"
test_resources_dir_kafka = pytestconfig.rootpath / "tests/integration/kafka" test_resources_dir_kafka = pytestconfig.rootpath / "tests/integration/kafka"

4
metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py
View File

@ -52,7 +52,7 @@ def loaded_presto_on_hive(presto_on_hive_runner):
@freeze_time(FROZEN_TIME) @freeze_time(FROZEN_TIME)
@pytest.mark.integration @pytest.mark.integration_batch_1
def test_presto_on_hive_ingest( def test_presto_on_hive_ingest(
loaded_presto_on_hive, test_resources_dir, pytestconfig, tmp_path, mock_time loaded_presto_on_hive, test_resources_dir, pytestconfig, tmp_path, mock_time
): ):
@ -112,7 +112,7 @@ def test_presto_on_hive_ingest(
@freeze_time(FROZEN_TIME) @freeze_time(FROZEN_TIME)
@pytest.mark.integration @pytest.mark.integration_batch_1
def test_presto_on_hive_instance_ingest( def test_presto_on_hive_instance_ingest(
loaded_presto_on_hive, test_resources_dir, pytestconfig, tmp_path, mock_time loaded_presto_on_hive, test_resources_dir, pytestconfig, tmp_path, mock_time
): ):