From a2b34dd0f49315b7fb75de81f07eb7558b12f04c Mon Sep 17 00:00:00 2001 From: Nahuel Date: Fri, 16 Dec 2022 15:46:25 +0100 Subject: [PATCH] Fix: Update Ingestion docker images and fix python libraries dependencies (#9342) * Update Ingestion docker images and fix python libraries dependencies * Install also apache-airlfow-providers-http --- docker/local-metadata/docker-compose-postgres.yml | 2 +- docker/local-metadata/docker-compose.yml | 2 +- docker/metadata/docker-compose-postgres.yml | 2 +- docker/metadata/docker-compose.yml | 2 +- ingestion/Dockerfile | 2 ++ ingestion/Dockerfile.ci | 3 +++ ingestion/setup.py | 5 ++--- 7 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docker/local-metadata/docker-compose-postgres.yml b/docker/local-metadata/docker-compose-postgres.yml index 766322d237c..305e143149c 100644 --- a/docker/local-metadata/docker-compose-postgres.yml +++ b/docker/local-metadata/docker-compose-postgres.yml @@ -138,7 +138,7 @@ services: openmetadata-server: condition: service_healthy environment: - AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth + AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session" AIRFLOW__CORE__EXECUTOR: LocalExecutor AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs" DB_HOST: ${AIRFLOW_DB_HOST:-postgresql} diff --git a/docker/local-metadata/docker-compose.yml b/docker/local-metadata/docker-compose.yml index 4427b43a0e3..ad87107e673 100644 --- a/docker/local-metadata/docker-compose.yml +++ b/docker/local-metadata/docker-compose.yml @@ -132,7 +132,7 @@ services: INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all} container_name: openmetadata_ingestion environment: - AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth + AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session" AIRFLOW__CORE__EXECUTOR: LocalExecutor AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs" DB_HOST: ${AIRFLOW_DB_HOST:-mysql} diff --git a/docker/metadata/docker-compose-postgres.yml b/docker/metadata/docker-compose-postgres.yml index a41fe79c2eb..a89663c0525 100644 --- a/docker/metadata/docker-compose-postgres.yml +++ b/docker/metadata/docker-compose-postgres.yml @@ -127,7 +127,7 @@ services: openmetadata-server: condition: service_healthy environment: - AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth + AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session" AIRFLOW__CORE__EXECUTOR: LocalExecutor AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs" DB_HOST: ${AIRFLOW_DB_HOST:-postgresql} diff --git a/docker/metadata/docker-compose.yml b/docker/metadata/docker-compose.yml index d7f01419bbe..29184984f67 100644 --- a/docker/metadata/docker-compose.yml +++ b/docker/metadata/docker-compose.yml @@ -123,7 +123,7 @@ services: openmetadata-server: condition: service_healthy environment: - AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth + AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session" AIRFLOW__CORE__EXECUTOR: LocalExecutor AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs" DB_HOST: ${AIRFLOW_DB_HOST:-mysql} diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile index 2abf3995d38..049f5b4c8da 100644 --- a/ingestion/Dockerfile +++ b/ingestion/Dockerfile @@ -60,6 +60,8 @@ ARG INGESTION_DEPENDENCY="all" RUN pip install --upgrade pip RUN pip install "openmetadata-managed-apis==0.12.2.4" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.3.3/constraints-3.9.txt" RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]==0.12.2.4" +# remove all airflow providers except for docker +RUN pip freeze | grep "apache-airlfow-providers" | grep -v "docker\|http" | xargs pip uninstall -y # Uninstalling psycopg2-binary and installing psycopg2 instead # because the psycopg2-binary generates a architecture specific error # while authrenticating connection with the airflow, psycopg2 solves this error diff --git a/ingestion/Dockerfile.ci b/ingestion/Dockerfile.ci index ea6c380cc9f..eda93d8507d 100644 --- a/ingestion/Dockerfile.ci +++ b/ingestion/Dockerfile.ci @@ -69,6 +69,9 @@ WORKDIR /home/airflow/ingestion ARG INGESTION_DEPENDENCY="all" RUN pip install ".[${INGESTION_DEPENDENCY}]" +# remove all airflow providers except for docker +RUN pip freeze | grep "apache-airflow-providers" | grep -v "docker\|http" | xargs pip uninstall -y + # Uninstalling psycopg2-binary and installing psycopg2 instead # because the psycopg2-binary generates a architecture specific error # while authrenticating connection with the airflow, psycopg2 solves this error diff --git a/ingestion/setup.py b/ingestion/setup.py index 64ceadbe369..c4f5c4d0e6d 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -45,8 +45,7 @@ base_requirements = { "boto3~=1.19.12", "botocore==1.22.12", "avro-python3==1.10.2", - "grpcio-tools", - "protobuf", + "grpcio-tools==1.48.2", # compatibility requirements for 3.7 "typing-compat~=0.1.0", "importlib-metadata~=4.12.0", # From airflow constraints @@ -54,7 +53,7 @@ base_requirements = { "requests-aws4auth==1.1.2", "pymysql>=1.0.2", "cached-property==1.5.2", - "chardet", + "chardet==4.0.0", }