fix: build vercel python from source (#7972)

docs-website/vercel-setup.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+./metadata-ingestion/scripts/install_deps.sh
+
+# Build python from source.
+# Amazon Linux 2 has Python 3.8, but it's version of OpenSSL is super old and hence it
+# doesn't work with the packages we use. As such, we have to build Python from source.
+# TODO: This process is extremely slow - ideally we should cache the built Python binary
+# for reuse.
+
+yum groupinstall "Development Tools" -y
+yum erase openssl-devel -y
+yum install openssl11 openssl11-devel  libffi-devel bzip2-devel wget -y
+
+wget https://www.python.org/ftp/python/3.10.11/Python-3.10.11.tgz
+tar -xf Python-3.10.11.tgz
+cd Python-3.10.11
+./configure #--enable-optimizations
+
+make -j $(nproc)
+
+make install
+
+# Set python3.10 as the default version.
+py3="$(which python3)"
+rm "$py3"
+ln "$(which python3.10)" "$py3"
+python3 --version
+

metadata-ingestion-modules/airflow-plugin/build.gradle

@@ -16,7 +16,7 @@ task checkPythonVersion(type: Exec) {
 task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
   inputs.file file('setup.py')
   outputs.dir("${venv_name}")
-  commandLine 'bash', '-c', "${python_executable} -m venv ${venv_name} && ${venv_name}/bin/python -m pip install --upgrade pip wheel setuptools==57.5.0"
+  commandLine 'bash', '-c', "${python_executable} -m venv ${venv_name} && ${venv_name}/bin/python -m pip install --upgrade pip wheel 'setuptools>=63.0.0'"
 }
 
 task installPackage(type: Exec, dependsOn: environmentSetup) {

metadata-ingestion/scripts/install_deps.sh

@@ -14,7 +14,9 @@ else
             librdkafka-devel \
             openldap-devel \
             cyrus-sasl-devel \
-            openldap-clients
+            openldap-clients \
+            sqlite-devel \
+            xz-devel
     else
         $sudo_cmd apt-get update && $sudo_cmd apt-get install -y \
             librdkafka-dev \

vercel.json

@@ -4,6 +4,6 @@
     "silent": true,
     "autoJobCancelation": true
   },
-  "installCommand": "amazon-linux-extras install python3.8 && py3=\"$(which python3)\" && rm \"$py3\" && ln \"$(which python3.8)\" \"$py3\" && ./metadata-ingestion/scripts/install_deps.sh && yum install -y gcc python38-devel",
+  "installCommand": "./docs-website/vercel-setup.sh",
   "outputDirectory": "docs-website/build"
 }