From 4515044fc5d703904c53e0163c83bfca07a797d2 Mon Sep 17 00:00:00 2001 From: Michael Knapp Date: Mon, 1 May 2017 12:31:44 -0400 Subject: [PATCH] Adding docker support to WhereHows. (#478) --- docker/.gitignore | 3 ++ docker/README.md | 14 +++++++ docker/archive-factory/.gitignore | 2 + docker/archive-factory/build-backend.sh | 22 ++++++++++ docker/archive-factory/build-web.sh | 30 +++++++++++++ docker/backend-service/Dockerfile | 17 ++++++++ docker/docker-compose.yml | 31 ++++++++++++++ docker/mysql/.gitignore | 1 + docker/mysql/Dockerfile | 16 +++++++ docker/mysql/binary/async-init.sh | 10 +++++ .../binary/create_all_tables_wrapper.sql | 32 ++++++++++++++ docker/mysql/binary/init.sql | 20 +++++++++ docker/mysql/binary/run-for-wherehows.sh | 18 ++++++++ docker/mysql/build-docker.sh | 8 ++++ docker/mysql/copy-ddl.sh | 24 +++++++++++ docker/mysql/run-base.sh | 3 ++ docker/mysql/run-mysql-wherehows-docker.sh | 8 ++++ docker/mysql/run-root.sh | 8 ++++ docker/web/Dockerfile | 17 ++++++++ prepare-docker-archives.sh | 42 +++++++++++++++++++ 20 files changed, 326 insertions(+) create mode 100644 docker/.gitignore create mode 100644 docker/README.md create mode 100644 docker/archive-factory/.gitignore create mode 100755 docker/archive-factory/build-backend.sh create mode 100755 docker/archive-factory/build-web.sh create mode 100644 docker/backend-service/Dockerfile create mode 100644 docker/docker-compose.yml create mode 100644 docker/mysql/.gitignore create mode 100644 docker/mysql/Dockerfile create mode 100755 docker/mysql/binary/async-init.sh create mode 100644 docker/mysql/binary/create_all_tables_wrapper.sql create mode 100644 docker/mysql/binary/init.sql create mode 100755 docker/mysql/binary/run-for-wherehows.sh create mode 100755 docker/mysql/build-docker.sh create mode 100755 docker/mysql/copy-ddl.sh create mode 100755 docker/mysql/run-base.sh create mode 100755 docker/mysql/run-mysql-wherehows-docker.sh create mode 100755 docker/mysql/run-root.sh create mode 100644 docker/web/Dockerfile create mode 100755 prepare-docker-archives.sh diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 0000000000..47f607d52b --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1,3 @@ +archives/ +runtime/ +.env diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000..a03577afba --- /dev/null +++ b/docker/README.md @@ -0,0 +1,14 @@ +The docker directory contains all docker related source code. + +Here is how to use it: +1. First ensure that docker and docker-compose are installed, and that the user running this is a member of the docker group, or is root. +The docker compose script uses version 3, so be sure that the version you install supports that. +2. From the WhereHows root, run prepare-docker-archives.sh +3. Edit docker/.env if necessary. +4. From the docker directory: $ docker-compose up +5. In your local browser, open localhost:9000 +6. Also, the backend app is hosted on localhost:9001 + +If any step fails in the script, you can run individual steps in it, the script is pretty intuitive and has comments. + +Hope that helps. diff --git a/docker/archive-factory/.gitignore b/docker/archive-factory/.gitignore new file mode 100644 index 0000000000..eacdd6311c --- /dev/null +++ b/docker/archive-factory/.gitignore @@ -0,0 +1,2 @@ +originals/*.zip +tmp/ diff --git a/docker/archive-factory/build-backend.sh b/docker/archive-factory/build-backend.sh new file mode 100755 index 0000000000..c48d2ab460 --- /dev/null +++ b/docker/archive-factory/build-backend.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +if [ -d "tmp" ]; then + rm -rf tmp +fi + +mkdir tmp +cp originals/backend.zip tmp +cd tmp +unzip backend.zip +mv backend-ser* backend-service + +# Some people may want to remove the documentation of the API. +# rm -rf backend-service/share +rm backend-service/README.md + +# leave the option to change the secret later. +sed -i 's/changeme/$PLAY_CRYPTO_SECRET/g' backend-service/conf/application.conf + +tar zcvf backend-service.tar.gz backend-service + +cp *.tar.gz ../../backend-service/archives \ No newline at end of file diff --git a/docker/archive-factory/build-web.sh b/docker/archive-factory/build-web.sh new file mode 100755 index 0000000000..2a7f6f138b --- /dev/null +++ b/docker/archive-factory/build-web.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +if [ -d "tmp" ]; then + rm -rf tmp +fi + +mkdir tmp +cp originals/web.zip tmp +cd tmp +unzip web.zip +mv wherehows-* wherehows + +# some may wish to remove the docs +# rm -rf wherehows/share + +rm README.md + +# correct a few things that won't work in docker. + +# leave the option to change the play crypto secret later. +sed -i 's/changeme/$PLAY_CRYPTO_SECRET/g' wherehows/conf/application.conf + +# localhost usually does not work within docker, give people the power +# to change that from outside of docker with environment variables. +sed -i 's/localhost:8888/$HDFS_HOST:8888/g' wherehows/conf/application.conf +sed -i 's/localhost/$MYSQL_HOST/g' wherehows/conf/application.conf + +tar zcvf web.tar.gz wherehows + +cp web.tar.gz ../../web/archives \ No newline at end of file diff --git a/docker/backend-service/Dockerfile b/docker/backend-service/Dockerfile new file mode 100644 index 0000000000..5ef3d86d67 --- /dev/null +++ b/docker/backend-service/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:16.10 + +RUN mkdir /application; + +WORKDIR /application + +ADD archives/jdk-8u101-linux-x64.tar.gz /application +ADD archives/backend-service.tar.gz /application + +RUN ln -s jdk1.8.0_101 jdk8; + +WORKDIR /application/backend-service + +ENV JAVA_HOME=/application/jdk8; \ + PATH=/application/jdk8/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +CMD bin/backend-service \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000000..6b77297be2 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,31 @@ +version: "3" +services: + mysql: + image: "wherehows/mysql:1" + ports: + - "3306:3306" + environment: + MYSQL_ROOT_PASSWORD: wherehows + backend: + image: "wherehows/backend:1" + ports: + - "9001:9000" + environment: + WHZ_DB_USERNAME: wherehows + WHZ_DB_PASSWORD: wherehows + WHZ_DB_URL: jdbc:mysql://mysql:3306/wherehows + PLAY_CRYPTO_SECRET: changeme + links: + - "mysql:mysql" + web: + image: "wherehows/web:1" + ports: + - "9000:9000" + environment: + WHZ_DB_USERNAME: wherehows + WHZ_DB_PASSWORD: wherehows + WHZ_DB_URL: jdbc:mysql://mysql:3306/wherehows + MYSQL_HOST: mysql + HDFS_HOST: "127.0.0.1" + links: + - "mysql:mysql" \ No newline at end of file diff --git a/docker/mysql/.gitignore b/docker/mysql/.gitignore new file mode 100644 index 0000000000..d0b81b7eef --- /dev/null +++ b/docker/mysql/.gitignore @@ -0,0 +1 @@ +*_DDL/ diff --git a/docker/mysql/Dockerfile b/docker/mysql/Dockerfile new file mode 100644 index 0000000000..edec9326c9 --- /dev/null +++ b/docker/mysql/Dockerfile @@ -0,0 +1,16 @@ +FROM mysql:5.7.17 + +RUN mkdir /usr/local/wherehows; echo "alias ll=ls -al" > /etc/profile; + +COPY binary/*.sh /usr/local/bin/ +COPY ETL_DDL /usr/local/wherehows/ETL_DDL +COPY WEB_DDL /usr/local/wherehows/WEB_DDL +COPY binary/*.sql /usr/local/wherehows/ + +HEALTHCHECK --interval=5s --timeout=3s CMD mysqladmin -pwherehows ping + +VOLUME /var/lib/mysql +VOLUME /var/log/mysql + +# note that the default command is still 'mysqld' +ENTRYPOINT /usr/local/bin/run-for-wherehows.sh diff --git a/docker/mysql/binary/async-init.sh b/docker/mysql/binary/async-init.sh new file mode 100755 index 0000000000..7a8c6df38d --- /dev/null +++ b/docker/mysql/binary/async-init.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Wait for the server to be completely up before initializing it. +while [ -z "$(mysqladmin --password=$MYSQL_ROOT_PASSWORD ping 2> /dev/null | grep 'alive')" ]; do sleep 1; done + +# must be in the right directory so the create tables script finds the files it sources. +cd /usr/local/wherehows + +mysql --password=$MYSQL_ROOT_PASSWORD < /usr/local/wherehows/init.sql &> /var/log/mysql/create-accounts.log +mysql --password=$MYSQL_ROOT_PASSWORD -hlocalhost -uwherehows -Dwherehows < /usr/local/wherehows/create_all_tables_wrapper.sql &> /var/log/mysql/create-tables.log \ No newline at end of file diff --git a/docker/mysql/binary/create_all_tables_wrapper.sql b/docker/mysql/binary/create_all_tables_wrapper.sql new file mode 100644 index 0000000000..df4adb42a7 --- /dev/null +++ b/docker/mysql/binary/create_all_tables_wrapper.sql @@ -0,0 +1,32 @@ +-- +-- Copyright 2015 LinkedIn Corp. All rights reserved. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- + +/* wrapper sql to call all individual DDL */ +-- use wherehows; + +source ETL_DDL/dataset_metadata.sql; +source ETL_DDL/etl_configure_tables.sql; +source ETL_DDL/executor_metadata.sql; +source ETL_DDL/git_metadata.sql; +source ETL_DDL/lineage_metadata.sql; +source ETL_DDL/metric_metadata.sql; +source ETL_DDL/owner_metadata.sql; +source ETL_DDL/patterns.sql; +source ETL_DDL/kafka_tracking.sql; +source ETL_DDL/dataset_info_metadata.sql; + +source WEB_DDL/track.sql; +source WEB_DDL/users.sql; + +show tables; diff --git a/docker/mysql/binary/init.sql b/docker/mysql/binary/init.sql new file mode 100644 index 0000000000..84d927efd9 --- /dev/null +++ b/docker/mysql/binary/init.sql @@ -0,0 +1,20 @@ + +-- note that this may be resuming from a previous run, only do commands if things do not already exist. +CREATE DATABASE IF NOT EXISTS wherehows + DEFAULT CHARACTER SET utf8 + DEFAULT COLLATE utf8_general_ci; + +CREATE USER IF NOT EXISTS 'wherehows'@'localhost' IDENTIFIED BY 'wherehows'; +CREATE USER IF NOT EXISTS 'wherehows'@'%' IDENTIFIED BY 'wherehows'; + +CREATE USER IF NOT EXISTS 'wherehows_ro'@'localhost' IDENTIFIED BY 'readmetadata'; +CREATE USER IF NOT EXISTS 'wherehows_ro'@'%' IDENTIFIED BY 'readmetadata'; + +FLUSH PRIVILEGES; + +GRANT ALL ON wherehows.* TO 'wherehows'@'localhost'; +GRANT ALL ON wherehows.* TO 'wherehows'@'%'; +GRANT SELECT ON wherehows.* TO 'wherehows_ro'@'localhost'; +GRANT SELECT ON wherehows.* TO 'wherehows_ro'@'%'; + +FLUSH PRIVILEGES; diff --git a/docker/mysql/binary/run-for-wherehows.sh b/docker/mysql/binary/run-for-wherehows.sh new file mode 100755 index 0000000000..2bd1d09849 --- /dev/null +++ b/docker/mysql/binary/run-for-wherehows.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ -z "$MYSQL_ROOT_PASSWORD" ]; then + echo "You must set MYSQL_ROOT_PASSWORD" + exit +fi + +ARG="$@" +if [ -z "$ARG" ]; then + echo "No default argument was provided, using mysqld" + ARG='mysqld' +fi + +./usr/local/bin/async-init.sh & + +# run the normal start up. +# note that mysqld is the default argument, provided by the original docker image. +./entrypoint.sh $ARG | tee /var/log/mysql/entrypoint.log diff --git a/docker/mysql/build-docker.sh b/docker/mysql/build-docker.sh new file mode 100755 index 0000000000..22a39fa43e --- /dev/null +++ b/docker/mysql/build-docker.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +VERSION=$1 +if [ -z "$VERSION" ]; then + VERSION=1 +fi + +docker build -t wherehows/mysql-wherehows:$VERSION . diff --git a/docker/mysql/copy-ddl.sh b/docker/mysql/copy-ddl.sh new file mode 100755 index 0000000000..7bfd97fa71 --- /dev/null +++ b/docker/mysql/copy-ddl.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Copies over the ddl files + +if [ -z "$WORKSPACE" ]; then + echo "You must set the WORKSPACE environment variable for this to work." + exit +fi + +rm -rf *_DDL +rm bin/create_all_tables_wrapper.sql + +DDL_DIR=$WORKSPACE/WhereHows/data-model/DDL +cp $DDL_DIR/create_all_tables_wrapper.sql bin +cp -r $DDL_DIR/*_DDL . + +# Unfortunately these scripts may be executed multiple times. +# The data directory is mounted as a volume, meaning that these scripts could run twice for the +# same directory. Change schema to just create tables if they do not already exist. +sed -i "" -e "s/CREATE TABLE/CREATE TABLE IF NOT EXISTS/g" *_DDL/* + +# In some places we just doubled up on IF NOT EXISTS +sed -i "" -e "s/IF NOT EXISTS IF NOT EXISTS/IF NOT EXISTS/g" *_DDL/* + diff --git a/docker/mysql/run-base.sh b/docker/mysql/run-base.sh new file mode 100755 index 0000000000..d77003f71a --- /dev/null +++ b/docker/mysql/run-base.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker run -e MYSQL_ROOT_PASSWORD=wherehows mysql:8.0.0 diff --git a/docker/mysql/run-mysql-wherehows-docker.sh b/docker/mysql/run-mysql-wherehows-docker.sh new file mode 100755 index 0000000000..7390924521 --- /dev/null +++ b/docker/mysql/run-mysql-wherehows-docker.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +VERSION=$1 +if [ -z "$VERSION" ]; then + VERSION=1 +fi + +docker run --name mysql-wherehows -d -e MYSQL_ROOT_PASSWORD=wherehows -p 3306:3306 wherehows/mysql-wherehows:$VERSION diff --git a/docker/mysql/run-root.sh b/docker/mysql/run-root.sh new file mode 100755 index 0000000000..dbb34d5b6c --- /dev/null +++ b/docker/mysql/run-root.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +VERSION=$1 +if [ -z "$VERSION" ]; then + VERSION=1 +fi + +docker run -it -e MYSQL_ROOT_PASSWORD=wherehows -p 3306:3306 --entrypoint /bin/bash wherehows/mysql-wherehows:$VERSION diff --git a/docker/web/Dockerfile b/docker/web/Dockerfile new file mode 100644 index 0000000000..6f2387cb54 --- /dev/null +++ b/docker/web/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:16.10 + +RUN mkdir /application; + +WORKDIR /application + +ADD archives/jdk-8u101-linux-x64.tar.gz /application +ADD archives/web.tar.gz /application + +RUN ln -s jdk1.8.0_101 jdk8; + +WORKDIR /application/wherehows + +ENV JAVA_HOME=/application/jdk8; \ + PATH=/application/jdk8/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +CMD bin/wherehows \ No newline at end of file diff --git a/prepare-docker-archives.sh b/prepare-docker-archives.sh new file mode 100755 index 0000000000..3ce1ba0fa5 --- /dev/null +++ b/prepare-docker-archives.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# assumption: docker is already installed +# also, you should be in the sudoers + +# ensure the SBT opts are sufficient so the application will build +if [ -z "SBT_OPTS" ]; then + export SBT_OPTS='-Xms1G -Xmx1G -Xss2M' +fi + +if [ -d "web/target" ]; then + # sometimes this directory causes problems if it exists when the build starts. + rm -rf web/target +fi + +# build the application's distribution zip +./gradlew dist + +# move those to a directory where we will work on them more. +mv backend-service/target/universal/*.zip docker/archive-factory/originals/backend.zip +mv web/target/universal/*.zip docker/archive-factory/originals/web.zip + +cd docker/archive-factory + +# unfortunately those zip files are not quite what we want to start with in docker, +# run these other scripts that will revise them. +./build-backend.sh +./build-web.sh + +# build the docker images +cd ../mysql +./copy-ddl.sh +docker build -t wherehows/mysql . +cd ../backend-service +docker build -t wherehows/backend . +cd ../web +docker build -t wherehows/web . + +cd .. +echo "now run this to start the application:" +echo "docker-compose up" +echo "you may need to edit the .env file first."