yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-08 23:43:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(ingest): make hive/trino test more reliable (#6741)

Browse Source
This commit is contained in:
Harshal Sheth 2022-12-12 21:02:52 -05:00 committed by GitHub
parent 5658fd5a54
commit 85bb1f5030
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 9 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
metadata-ingestion/tests/integration/hive/docker-compose.yml
View File

@ -12,7 +12,7 @@ services:
env_file: env_file:
- ./hadoop-hive.env - ./hadoop-hive.env
ports: ports:
- "50070:50070" - "50070"
datanode: datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
volumes: volumes:

2
metadata-ingestion/tests/integration/presto-on-hive/docker-compose.yml
View File

@ -29,7 +29,7 @@ services:
env_file: env_file:
- ./setup/hadoop-hive.env - ./setup/hadoop-hive.env
ports: ports:
- "50070:50070" - "50070"
datanode: datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
volumes: volumes:

11
metadata-ingestion/tests/integration/trino/docker-compose.yml
View File

@ -3,7 +3,7 @@
version: "3" version: "3"
services: services:
testtrino: testtrino:
image: trinodb/trino:369 image: trinodb/trino:369
container_name: "testtrino" container_name: "testtrino"
@ -14,13 +14,13 @@ services:
depends_on: depends_on:
- "trinodb_postgres" - "trinodb_postgres"
- "hive-metastore" - "hive-metastore"
trinodb_postgres: trinodb_postgres:
image: postgres:alpine image: postgres:alpine
container_name: "trinodb_postgres" container_name: "trinodb_postgres"
environment: environment:
POSTGRES_PASSWORD: datahub POSTGRES_PASSWORD: datahub
volumes: volumes:
- ./setup/setup.sql:/docker-entrypoint-initdb.d/postgres_setup.sql - ./setup/setup.sql:/docker-entrypoint-initdb.d/postgres_setup.sql
ports: ports:
- "5432:5432" - "5432:5432"
@ -33,7 +33,7 @@ services:
env_file: env_file:
- ./setup/hadoop-hive.env - ./setup/hadoop-hive.env
ports: ports:
- "50070:50070" - "50070"
datanode: datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
volumes: volumes:
@ -43,7 +43,7 @@ services:
environment: environment:
SERVICE_PRECONDITION: "namenode:50070" SERVICE_PRECONDITION: "namenode:50070"
ports: ports:
- "50075:50075" - "50075"
hive-server: hive-server:
image: bde2020/hive:2.3.2-postgresql-metastore image: bde2020/hive:2.3.2-postgresql-metastore
container_name: "testhiveserver2" container_name: "testhiveserver2"
@ -58,7 +58,6 @@ services:
- ./setup/hive_setup.sql:/hive_setup.sql - ./setup/hive_setup.sql:/hive_setup.sql
hive-metastore: hive-metastore:
image: bde2020/hive:2.3.2-postgresql-metastore image: bde2020/hive:2.3.2-postgresql-metastore
container_name: "hive-metastore"
env_file: env_file:
- ./setup/hadoop-hive.env - ./setup/hadoop-hive.env
command: /opt/hive/bin/hive --service metastore command: /opt/hive/bin/hive --service metastore

4
metadata-ingestion/tests/test_helpers/docker_helpers.py
View File

@ -48,14 +48,14 @@ def docker_compose_runner(
): ):
@contextlib.contextmanager @contextlib.contextmanager
def run( def run(
compose_file_path: Union[str, list], key: str compose_file_path: Union[str, list], key: str, cleanup: bool = True
) -> pytest_docker.plugin.Services: ) -> pytest_docker.plugin.Services:
with pytest_docker.plugin.get_docker_services( with pytest_docker.plugin.get_docker_services(
docker_compose_command=docker_compose_command, docker_compose_command=docker_compose_command,
docker_compose_file=compose_file_path, docker_compose_file=compose_file_path,
docker_compose_project_name=f"{docker_compose_project_name}-{key}", docker_compose_project_name=f"{docker_compose_project_name}-{key}",
docker_setup=docker_setup, docker_setup=docker_setup,
docker_cleanup=docker_cleanup, docker_cleanup=docker_cleanup if cleanup else False,
) as docker_services: ) as docker_services:
yield docker_services yield docker_services