yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-22 16:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix: Update Ingestion docker images and fix python libraries dependencies (#9342)

Browse Source 
* Update Ingestion docker images and fix python libraries dependencies

* Install also apache-airlfow-providers-http
This commit is contained in:
Nahuel 2022-12-16 15:46:25 +01:00 committed by GitHub
parent cfc23a0f4e
commit a2b34dd0f4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docker/local-metadata/docker-compose-postgres.yml
View File

@ -138,7 +138,7 @@ services:
openmetadata-server:
condition: service_healthy
environment:
AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
AIRFLOW__CORE__EXECUTOR: LocalExecutor
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
DB_HOST: ${AIRFLOW_DB_HOST:-postgresql}

2
docker/local-metadata/docker-compose.yml
View File

@ -132,7 +132,7 @@ services:
INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all}
container_name: openmetadata_ingestion
environment:
AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
AIRFLOW__CORE__EXECUTOR: LocalExecutor
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
DB_HOST: ${AIRFLOW_DB_HOST:-mysql}

2
docker/metadata/docker-compose-postgres.yml
View File

@ -127,7 +127,7 @@ services:
openmetadata-server:
condition: service_healthy
environment:
AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
AIRFLOW__CORE__EXECUTOR: LocalExecutor
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
DB_HOST: ${AIRFLOW_DB_HOST:-postgresql}

2
docker/metadata/docker-compose.yml
View File

@ -123,7 +123,7 @@ services:
openmetadata-server:
condition: service_healthy
environment:
AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
AIRFLOW__CORE__EXECUTOR: LocalExecutor
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
DB_HOST: ${AIRFLOW_DB_HOST:-mysql}

2
ingestion/Dockerfile
View File

@ -60,6 +60,8 @@ ARG INGESTION_DEPENDENCY="all"
RUN pip install --upgrade pip
RUN pip install "openmetadata-managed-apis==0.12.2.4" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.3.3/constraints-3.9.txt"
RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]==0.12.2.4"
# remove all airflow providers except for docker
RUN pip freeze | grep "apache-airlfow-providers" | grep -v "docker\|http" | xargs pip uninstall -y
# Uninstalling psycopg2-binary and installing psycopg2 instead
# because the psycopg2-binary generates a architecture specific error
# while authrenticating connection with the airflow, psycopg2 solves this error

3
ingestion/Dockerfile.ci
View File

@ -69,6 +69,9 @@ WORKDIR /home/airflow/ingestion
ARG INGESTION_DEPENDENCY="all"
RUN pip install ".[${INGESTION_DEPENDENCY}]"
# remove all airflow providers except for docker
RUN pip freeze | grep "apache-airflow-providers" | grep -v "docker\|http" | xargs pip uninstall -y
# Uninstalling psycopg2-binary and installing psycopg2 instead
# because the psycopg2-binary generates a architecture specific error
# while authrenticating connection with the airflow, psycopg2 solves this error

5
ingestion/setup.py
View File

@ -45,8 +45,7 @@ base_requirements = {
"boto3~=1.19.12",
"botocore==1.22.12",
"avro-python3==1.10.2",
"grpcio-tools",
"protobuf",
"grpcio-tools==1.48.2",
# compatibility requirements for 3.7
"typing-compat~=0.1.0",
"importlib-metadata~=4.12.0", # From airflow constraints
@ -54,7 +53,7 @@ base_requirements = {
"requests-aws4auth==1.1.2",
"pymysql>=1.0.2",
"cached-property==1.5.2",
"chardet",
"chardet==4.0.0",
}