diff --git a/docker/kafka-setup/Dockerfile b/docker/kafka-setup/Dockerfile index e379ae4385..8cf9d0869d 100644 --- a/docker/kafka-setup/Dockerfile +++ b/docker/kafka-setup/Dockerfile @@ -5,7 +5,7 @@ ARG MAVEN_REPO="https://repo1.maven.org/maven2" ARG SNAKEYAML_VERSION="2.0" RUN rm /usr/share/java/cp-base-new/snakeyaml-*.jar \ - && wget -P /usr/share/java/cp-base-new $MAVEN_REPO/org/yaml/snakeyaml/$SNAKEYAML_VERSION/snakeyaml-$SNAKEYAML_VERSION.jar + && wget -P /usr/share/java/cp-base-new $MAVEN_REPO/org/yaml/snakeyaml/$SNAKEYAML_VERSION/snakeyaml-$SNAKEYAML_VERSION.jar # Based on https://github.com/blacktop's alpine kafka build FROM python:3-alpine @@ -17,7 +17,7 @@ ENV SCALA_VERSION 2.13 ENV CUB_CLASSPATH='"/usr/share/java/cp-base-new/*"' # Confluent Docker Utils Version (Namely the tag or branch to grab from git to install) -ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="v0.0.58" +ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="v0.0.60" # This can be overriden for an offline/air-gapped builds ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" @@ -36,7 +36,9 @@ RUN mkdir -p /opt \ && adduser -DH -s /sbin/nologin kafka \ && chown -R kafka: /opt/kafka \ && echo "===> Installing python packages ..." \ - && pip install --no-cache-dir jinja2 requests \ + && pip install --no-cache-dir --upgrade pip wheel setuptools \ + && pip install jinja2 requests \ + && pip install "Cython<3.0" "PyYAML<6" --no-build-isolation \ && pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ && rm -rf /tmp/* \ && apk del --purge .build-deps diff --git a/docker/quickstart/generate_and_compare.sh b/docker/quickstart/generate_and_compare.sh index e34abeb982..d568eb3a4c 100755 --- a/docker/quickstart/generate_and_compare.sh +++ b/docker/quickstart/generate_and_compare.sh @@ -8,5 +8,6 @@ set -euxo pipefail python3 -m venv venv source venv/bin/activate +pip install --upgrade pip wheel setuptools pip install -r requirements.txt python generate_docker_quickstart.py check-all diff --git a/docker/quickstart/requirements.txt b/docker/quickstart/requirements.txt index a20e96afc8..5392413311 100644 --- a/docker/quickstart/requirements.txt +++ b/docker/quickstart/requirements.txt @@ -1,3 +1,3 @@ -PyYAML==5.4.1 +PyYAML==6.0 python-dotenv==0.17.0 click diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle index 9ab590b156..d895e29229 100644 --- a/metadata-ingestion-modules/airflow-plugin/build.gradle +++ b/metadata-ingestion-modules/airflow-plugin/build.gradle @@ -22,6 +22,10 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) { task installPackage(type: Exec, dependsOn: environmentSetup) { inputs.file file('setup.py') outputs.dir("${venv_name}") + // Workaround for https://github.com/yaml/pyyaml/issues/601. + // See https://github.com/yaml/pyyaml/issues/601#issuecomment-1638509577. + // and https://github.com/datahub-project/datahub/pull/8435. + commandLine 'bash', '-x', '-c', "${pip_install_command} install 'Cython<3.0' 'PyYAML<6' --no-build-isolation" commandLine 'bash', '-x', '-c', "${pip_install_command} -e ." }