Docker local (#723)

* added docker file for local build and ingestion

* Docker Modified for local setup

* Ingestion script optimized

* Added script to directly run Local Docker

Co-authored-by: Parth Panchal <parthpanchal@Parths-MacBook-Air.local>

docker/local-metadata/Dockerfile

@@ -0,0 +1,12 @@
+FROM openjdk:11
+
+EXPOSE 8585 3306
+
+RUN apt-get update -y && \
+    apt-get install --no-install-recommends wget -y && \
+    rm -rf /var/lib/apt/lists/*
+COPY docker/local-metadata/openmetadata-start.sh /
+COPY docker/local-metadata/openmetadata.yaml /
+COPY dist/target/openmetadata-*.tar.gz /
+RUN chmod 777 openmetadata-start.sh && tar zxvf openmetadata-*.tar.gz
+CMD ["./openmetadata-start.sh"]

docker/local-metadata/docker-compose.yml

@@ -0,0 +1,88 @@
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License. You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+version: "3.9"
+services:
+  db:
+    platform: linux/x86_64
+    image: mysql:latest
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_USER: openmetadata_user
+      MYSQL_PASSWORD: openmetadata_password
+      MYSQL_DATABASE: openmetadata_db
+    expose:
+      - 3306
+    networks:
+      app_net:
+        ipv4_address: 172.16.239.10
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
+    environment:
+      - discovery.type=single-node
+    networks:
+      app_net:
+        ipv4_address: 172.16.239.11
+    expose:
+      - 9200
+      - 9300
+    ports:
+      - 9200:9200
+      - 9300:9300
+
+  openmetadata-server:
+    build:
+      context: ../../.
+      dockerfile: docker/local-metadata/Dockerfile
+    expose:
+      - 8585
+      - 9200
+      - 9300
+      - 3306
+    ports:
+      - 8585:8585
+    depends_on:
+      - db
+      - elasticsearch
+    networks:
+      app_net:
+        ipv4_address: 172.16.239.12
+    extra_hosts:
+      - "localhost:172.16.239.10"
+      - "elasticsearch:172.16.239.11"
+
+  ingestion:
+    build:
+      context: ../../ingestion/.
+      dockerfile: Dockerfile_local
+    expose:
+      - 7777
+    ports:
+      - 7777:7777
+    networks:
+      - app_net
+    extra_hosts:
+      - "localhost:172.16.239.10"
+      - "localhost:172.16.239.11"
+      - "localhost:172.16.239.12"
+
+networks:
+  app_net:
+    ipam:
+      driver: default
+      config:
+        - subnet: "172.16.239.0/24"

docker/local-metadata/openmetadata-start.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+while ! wget -O /dev/null -o /dev/null localhost:3306; do sleep 5; done
+mv /openmetadata.yaml /openmetadata-*/conf/openmetadata.yaml
+cd /openmetadata-*/
+./bootstrap/bootstrap_storage.sh migrate
+./bin/openmetadata-server-start.sh conf/openmetadata.yaml

docker/local-metadata/openmetadata.yaml

@@ -0,0 +1,138 @@
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License. You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+swagger:
+  resourcePackage: org.openmetadata.catalog.resources
+
+
+server:
+  rootPath: '/api/*'
+  applicationConnectors:
+    - type: http
+      port: 8585
+  adminConnectors:
+    - type: http
+      port: 8586
+
+  # Above configuration for running http is fine for dev and testing.
+  # For production setup, where UI app will hit apis through DPS it
+  # is strongly recommended to run https instead. Note that only
+  # keyStorePath and keyStorePassword are mandatory properties. Values
+  # for other properties are defaults
+  #server:
+  #applicationConnectors:
+  #  - type: https
+  #    port: 8585
+  #    keyStorePath: ./conf/keystore.jks
+  #    keyStorePassword: changeit
+  #    keyStoreType: JKS
+  #    keyStoreProvider:
+  #    trustStorePath: /path/to/file
+  #    trustStorePassword: changeit
+  #    trustStoreType: JKS
+  #    trustStoreProvider:
+  #    keyManagerPassword: changeit
+  #    needClientAuth: false
+  #    wantClientAuth:
+  #    certAlias: <alias>
+  #    crlPath: /path/to/file
+  #    enableCRLDP: false
+  #    enableOCSP: false
+  #    maxCertPathLength: (unlimited)
+  #    ocspResponderUrl: (none)
+  #    jceProvider: (none)
+  #    validateCerts: true
+  #    validatePeers: true
+  #    supportedProtocols: SSLv3
+  #    supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
+  #    allowRenegotiation: true
+  #    endpointIdentificationAlgorithm: (none)
+
+  #adminConnectors:
+  #  - type: https
+  #    port: 8586
+  #    keyStorePath: ./conf/keystore.jks
+  #    keyStorePassword: changeit
+  #    keyStoreType: JKS
+  #    keyStoreProvider:
+  #    trustStorePath: /path/to/file
+  #    trustStorePassword: changeit
+  #    trustStoreType: JKS
+  #    trustStoreProvider:
+  #    keyManagerPassword: changeit
+  #    needClientAuth: false
+  #    wantClientAuth:
+  #    certAlias: <alias>
+  #    crlPath: /path/to/file
+  #    enableCRLDP: false
+  #    enableOCSP: false
+  #    maxCertPathLength: (unlimited)
+  #    ocspResponderUrl: (none)
+  #    jceProvider: (none)
+  #    validateCerts: true
+  #    validatePeers: true
+  #    supportedProtocols: SSLv3
+  #    supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
+  #    allowRenegotiation: true
+  #    endpointIdentificationAlgorithm: (none)
+
+# Logging settings.
+# https://logback.qos.ch/manual/layouts.html#conversionWord
+logging:
+  level: INFO
+  loggers:
+    org.openmetadata.catalog.common: DEBUG
+    io.swagger: ERROR
+  appenders:
+    - type: file
+      threshold: TRACE
+      logFormat: "%level [%d{HH:mm:ss.SSS}] [%t] %logger{5} - %msg %n"
+      currentLogFilename: ./logs/openmetadata.log
+      archivedLogFilenamePattern: ./logs/openmetadata-%d{yyyy-MM-dd}-%i.log.gz
+      archivedFileCount: 7
+      timeZone: UTC
+      maxFileSize: 50MB
+
+database:
+  # the name of the JDBC driver, mysql in our case
+  driverClass: com.mysql.cj.jdbc.Driver
+  # the username and password
+  user: openmetadata_user
+  password: openmetadata_password
+  # the JDBC URL; the database is called openmetadata_db
+  url: jdbc:mysql://localhost/openmetadata_db?allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC
+
+
+elasticsearch:
+  host: elasticsearch
+  port: 9200
+
+eventHandlerConfiguration:
+  eventHandlerClassNames:
+    - "org.openmetadata.catalog.events.AuditEventHandler"
+    - "org.openmetadata.catalog.events.ElasticSearchEventHandler"
+
+health:
+  delayedShutdownHandlerEnabled: true
+  shutdownWaitPeriod: 1s
+  healthCheckUrlPaths: ["/api/v1/health-check"]
+  healthChecks:
+    - name: UserDatabaseCheck
+      critical: true
+      schedule:
+        checkInterval: 2500ms
+        downtimeInterval: 10s
+        failureAttempts: 2
+        successAttempts: 1

docker/metadata/Dockerfile

@@ -7,6 +7,6 @@ RUN apt-get update -y && \
     rm -rf /var/lib/apt/lists/*
 COPY openmetadata-start.sh /
 COPY openmetadata.yaml /
-RUN wget https://github.com/open-metadata/OpenMetadata/releases/download/0.3.0-release/openmetadata-0.3.0.tar.gz
-RUN chmod 777 openmetadata-start.sh && tar zxvf openmetadata-0.3.0.tar.gz
+RUN wget https://github.com/open-metadata/OpenMetadata/releases/download/0.4.0/openmetadata-0.4.0.tar.gz
+RUN chmod 777 openmetadata-start.sh && tar zxvf openmetadata-*.tar.gz
 CMD ["./openmetadata-start.sh"]

docker/metadata/openmetadata-start.sh

@@ -17,7 +17,7 @@
 #
 
 while ! curl -o - localhost:3306; do sleep 5; done
-cp /openmetadata.yaml /openmetadata-0.3.0/conf/openmetadata.yaml
-cd /openmetadata-0.3.0
+cp /openmetadata.yaml /openmetadata-*/conf/openmetadata.yaml
+cd /openmetadata-*/
 ./bootstrap/bootstrap_storage.sh migrate
 ./bin/openmetadata-server-start.sh conf/openmetadata.yaml

docker/run_local_docker.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+mvn -DskipTests clean package
+cd docker/local-metadata/
+docker-compose up

ingestion/Dockerfile_local

@@ -0,0 +1,13 @@
+FROM python:3.8.10
+
+EXPOSE 7777
+
+COPY ./. /openmetadata-ingestion/.
+WORKDIR /openmetadata-ingestion
+
+RUN apt-get update && \
+    apt-get install -y wget gcc libsasl2-dev unixodbc-dev --no-install-recommends && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN chmod 777 local_ingestion_dependency.sh
+CMD ["./local_ingestion_dependency.sh"]

ingestion/local_ingestion_dependency.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pip install --upgrade setuptools '.[sample-dßata, elasticsearch]'
+while ! wget -O /dev/null -o /dev/null localhost:8585; do sleep 5; done
+metadata ingest -c pipelines/sample_data.json
+metadata ingest -c pipelines/sample_users.json
+metadata ingest -c pipelines/sample_usage.json
+metadata ingest -c pipelines/metadata_to_es.json