2022-10-11 07:50:49 +02:00
|
|
|
FROM python:3.9-buster
|
|
|
|
|
|
|
|
# Install Dependencies (listed in alphabetical order)
|
|
|
|
RUN apt-get update \
|
2023-01-19 08:08:56 +01:00
|
|
|
&& apt-get install -y alien \
|
|
|
|
build-essential \
|
2022-10-11 07:50:49 +02:00
|
|
|
ca-certificates \
|
|
|
|
default-libmysqlclient-dev \
|
|
|
|
freetds-bin \
|
|
|
|
freetds-dev \
|
|
|
|
gcc \
|
|
|
|
gnupg \
|
2023-01-19 08:08:56 +01:00
|
|
|
libaio1 \
|
2022-10-11 07:50:49 +02:00
|
|
|
libevent-dev \
|
|
|
|
libffi-dev \
|
|
|
|
libpq-dev \
|
|
|
|
libsasl2-dev \
|
|
|
|
libsasl2-modules \
|
|
|
|
libssl-dev \
|
|
|
|
libxml2 \
|
|
|
|
openjdk-11-jre \
|
|
|
|
openssl \
|
|
|
|
postgresql \
|
|
|
|
postgresql-contrib \
|
|
|
|
tdsodbc \
|
|
|
|
unixodbc \
|
|
|
|
unixodbc-dev \
|
2023-01-19 08:08:56 +01:00
|
|
|
unzip \
|
2022-10-11 07:50:49 +02:00
|
|
|
wget --no-install-recommends
|
|
|
|
|
|
|
|
# Prep to install msodbcsql18
|
|
|
|
RUN apt-get update && \
|
|
|
|
apt-get install -y apt-transport-https && \
|
|
|
|
curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \
|
|
|
|
curl https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list && \
|
|
|
|
apt-get update && \
|
|
|
|
ACCEPT_EULA=Y apt-get install msodbcsql18 unixodbc-dev -y
|
|
|
|
|
|
|
|
# Prep to install confluent-kafka https://github.com/confluentinc/confluent-kafka-python/issues/1326
|
|
|
|
RUN apt-get update && \
|
|
|
|
apt-get install -y --no-install-recommends git g++ make && \
|
|
|
|
cd /tmp && git clone https://github.com/edenhill/librdkafka.git && \
|
|
|
|
cd librdkafka && git checkout tags/v1.9.0 && \
|
|
|
|
./configure && make && make install && \
|
|
|
|
cd ../ && rm -rf librdkafka
|
|
|
|
|
2023-01-19 08:08:56 +01:00
|
|
|
RUN if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; \
|
|
|
|
then \
|
|
|
|
wget https://download.oracle.com/otn_software/linux/instantclient/191000/instantclient-basic-linux.arm64-19.10.0.0.0dbru.zip -O /oracle-instantclient.zip && \
|
|
|
|
unzip -d /instantclient -j /oracle-instantclient.zip && rm -f /oracle-instantclient.zip; \
|
|
|
|
else \
|
|
|
|
wget https://download.oracle.com/otn_software/linux/instantclient/1917000/instantclient-basic-linux.x64-19.17.0.0.0dbru.zip -O /oracle-instantclient.zip && \
|
|
|
|
unzip -d /instantclient -j /oracle-instantclient.zip && rm -f /oracle-instantclient.zip; \
|
|
|
|
fi
|
|
|
|
|
|
|
|
ENV LD_LIBRARY_PATH=/instantclient
|
|
|
|
|
2022-10-11 07:50:49 +02:00
|
|
|
WORKDIR ingestion/
|
|
|
|
|
|
|
|
# Required for Airflow DockerOperator, as we need to run the workflows from a `python main.py` command in the container.
|
|
|
|
COPY ingestion/operators/docker/main.py .
|
|
|
|
|
|
|
|
RUN pip install --upgrade pip
|
|
|
|
|
|
|
|
ARG INGESTION_DEPENDENCY="all"
|
2022-10-13 11:06:27 +02:00
|
|
|
RUN pip install --upgrade openmetadata-ingestion[airflow]
|
2022-10-11 07:50:49 +02:00
|
|
|
RUN pip install --upgrade openmetadata-ingestion[${INGESTION_DEPENDENCY}]
|
|
|
|
# Uninstalling psycopg2-binary and installing psycopg2 instead
|
|
|
|
# because the psycopg2-binary generates a architecture specific error
|
|
|
|
# while authrenticating connection with the airflow, psycopg2 solves this error
|
|
|
|
RUN pip uninstall psycopg2-binary -y
|
|
|
|
RUN pip install psycopg2 mysqlclient
|