2022-04-11 18:38:26 +02:00
|
|
|
FROM python:3.9-slim as base
|
2021-10-30 21:35:30 +05:30
|
|
|
ENV AIRFLOW_HOME=/airflow
|
2021-08-03 01:24:50 +05:30
|
|
|
RUN apt-get update && \
|
2022-05-11 14:11:13 +05:30
|
|
|
apt-get install -y gcc libsasl2-modules libsasl2-dev curl build-essential libssl-dev libffi-dev librdkafka-dev unixodbc-dev python3.9-dev openjdk-11-jre unixodbc freetds-dev freetds-bin tdsodbc libevent-dev wget --no-install-recommends && \
|
2021-08-03 01:24:50 +05:30
|
|
|
rm -rf /var/lib/apt/lists/*
|
2022-04-11 18:38:26 +02:00
|
|
|
# RUN wget https://github.com/open-metadata/openmetadata-airflow-apis/releases/download/0.1/openmetadata-airflow-apis-plugin.tar.gz
|
|
|
|
# RUN tar zxvf openmetadata-airflow-apis-plugin.tar.gz
|
|
|
|
|
|
|
|
FROM base as airflow
|
2022-03-09 03:21:07 +05:30
|
|
|
ENV AIRFLOW_VERSION=2.1.4
|
2021-11-11 10:52:32 +05:30
|
|
|
ENV CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.9.txt"
|
2021-12-18 16:41:38 +01:00
|
|
|
# Add docker provider for the DockerOperator
|
|
|
|
RUN pip install "apache-airflow[docker]==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}"
|
2022-04-11 18:38:26 +02:00
|
|
|
|
|
|
|
|
|
|
|
FROM airflow as apis
|
|
|
|
WORKDIR /openmetadata-airflow-apis
|
|
|
|
COPY openmetadata-airflow-apis /openmetadata-airflow-apis
|
|
|
|
|
|
|
|
RUN mkdir /om-airflow
|
|
|
|
RUN mv /openmetadata-airflow-apis/src/plugins /om-airflow
|
|
|
|
|
|
|
|
RUN pip install "."
|
|
|
|
|
|
|
|
FROM apis as ingestion
|
|
|
|
WORKDIR /ingestion
|
2021-11-11 10:52:32 +05:30
|
|
|
COPY ingestion /ingestion
|
2021-12-18 16:41:38 +01:00
|
|
|
|
2022-05-02 16:09:44 +05:30
|
|
|
ARG INGESTION_DEPENDENCY=all
|
|
|
|
RUN pip install ".[${INGESTION_DEPENDENCY}]"
|
2022-04-11 18:38:26 +02:00
|
|
|
|
2022-04-22 19:52:59 +05:30
|
|
|
|
2021-11-11 10:52:32 +05:30
|
|
|
RUN airflow db init
|
2022-05-11 07:36:33 +02:00
|
|
|
RUN cp -r /ingestion/airflow.cfg /airflow/airflow.cfg
|
2021-11-11 10:52:32 +05:30
|
|
|
RUN cp -r /om-airflow/plugins /airflow/plugins
|
|
|
|
RUN cp -r /om-airflow/plugins/dag_templates /airflow/
|
|
|
|
RUN mkdir -p /airflow/dag_generated_configs
|
2021-10-26 21:44:24 +05:30
|
|
|
RUN chmod 755 ingestion_dependency.sh
|
2022-05-03 14:19:31 +05:30
|
|
|
VOLUME ingestion-volume-dag-airflow:/airflow/dag_generated_configs
|
|
|
|
VOLUME ingestion-volume-dags:/airflow/dags
|
|
|
|
VOLUME ingestion-volume-tmp:/tmp
|
2021-11-03 20:48:52 +05:30
|
|
|
EXPOSE 8080
|
2021-11-11 10:52:32 +05:30
|
|
|
CMD [ "./ingestion_dependency.sh" ]
|