yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-08 07:23:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(ingest): make hive/trino test more reliable (#6741)

Browse Source
This commit is contained in:
Harshal Sheth 2022-12-12 21:02:52 -05:00 committed by GitHub
parent 5658fd5a54
commit 85bb1f5030
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 9 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
metadata-ingestion/tests/integration/hive/docker-compose.yml
View File

@ -12,7 +12,7 @@ services:
env_file: env_file:
- ./hadoop-hive.env - ./hadoop-hive.env
ports: ports:
- "50070:50070" - "50070"
datanode: datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
volumes: volumes:

2
metadata-ingestion/tests/integration/presto-on-hive/docker-compose.yml
View File

@ -29,7 +29,7 @@ services:
env_file: env_file:
- ./setup/hadoop-hive.env - ./setup/hadoop-hive.env
ports: ports:
- "50070:50070" - "50070"
datanode: datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
volumes: volumes:

5
metadata-ingestion/tests/integration/trino/docker-compose.yml
View File

@ -33,7 +33,7 @@ services:
env_file: env_file:
- ./setup/hadoop-hive.env - ./setup/hadoop-hive.env
ports: ports:
- "50070:50070" - "50070"
datanode: datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
volumes: volumes:
@ -43,7 +43,7 @@ services:
environment: environment:
SERVICE_PRECONDITION: "namenode:50070" SERVICE_PRECONDITION: "namenode:50070"
ports: ports:
- "50075:50075" - "50075"
hive-server: hive-server:
image: bde2020/hive:2.3.2-postgresql-metastore image: bde2020/hive:2.3.2-postgresql-metastore
container_name: "testhiveserver2" container_name: "testhiveserver2"
@ -58,7 +58,6 @@ services:
- ./setup/hive_setup.sql:/hive_setup.sql - ./setup/hive_setup.sql:/hive_setup.sql
hive-metastore: hive-metastore:
image: bde2020/hive:2.3.2-postgresql-metastore image: bde2020/hive:2.3.2-postgresql-metastore
container_name: "hive-metastore"
env_file: env_file:
- ./setup/hadoop-hive.env - ./setup/hadoop-hive.env
command: /opt/hive/bin/hive --service metastore command: /opt/hive/bin/hive --service metastore

4
metadata-ingestion/tests/test_helpers/docker_helpers.py
View File

@ -48,14 +48,14 @@ def docker_compose_runner(
): ):
@contextlib.contextmanager @contextlib.contextmanager
def run( def run(
compose_file_path: Union[str, list], key: str compose_file_path: Union[str, list], key: str, cleanup: bool = True
) -> pytest_docker.plugin.Services: ) -> pytest_docker.plugin.Services:
with pytest_docker.plugin.get_docker_services( with pytest_docker.plugin.get_docker_services(
docker_compose_command=docker_compose_command, docker_compose_command=docker_compose_command,
docker_compose_file=compose_file_path, docker_compose_file=compose_file_path,
docker_compose_project_name=f"{docker_compose_project_name}-{key}", docker_compose_project_name=f"{docker_compose_project_name}-{key}",
docker_setup=docker_setup, docker_setup=docker_setup,
docker_cleanup=docker_cleanup, docker_cleanup=docker_cleanup if cleanup else False,
) as docker_services: ) as docker_services:
yield docker_services yield docker_services