Adding docker support to WhereHows. (#478)

docker/.gitignore

@@ -0,0 +1,3 @@
+archives/
+runtime/
+.env

docker/README.md

@@ -0,0 +1,14 @@
+The docker directory contains all docker related source code.
+
+Here is how to use it:
+1. First ensure that docker and docker-compose are installed, and that the user running this is a member of the docker group, or is root.
+The docker compose script uses version 3, so be sure that the version you install supports that.
+2. From the WhereHows root, run prepare-docker-archives.sh
+3. Edit docker/.env if necessary.
+4. From the docker directory: $ docker-compose up
+5. In your local browser, open localhost:9000
+6. Also, the backend app is hosted on localhost:9001
+
+If any step fails in the script, you can run individual steps in it, the script is pretty intuitive and has comments.
+
+Hope that helps.

docker/archive-factory/.gitignore

@@ -0,0 +1,2 @@
+originals/*.zip
+tmp/

docker/archive-factory/build-backend.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+if [ -d "tmp" ]; then
+  rm -rf tmp
+fi
+
+mkdir tmp
+cp originals/backend.zip tmp
+cd tmp
+unzip backend.zip
+mv backend-ser* backend-service
+
+# Some people may want to remove the documentation of the API.
+# rm -rf backend-service/share
+rm backend-service/README.md
+
+# leave the option to change the secret later.
+sed -i 's/changeme/$PLAY_CRYPTO_SECRET/g' backend-service/conf/application.conf
+
+tar zcvf backend-service.tar.gz backend-service
+
+cp *.tar.gz ../../backend-service/archives

docker/archive-factory/build-web.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+if [ -d "tmp" ]; then
+  rm -rf tmp
+fi
+
+mkdir tmp
+cp originals/web.zip tmp
+cd tmp
+unzip web.zip
+mv wherehows-* wherehows
+
+# some may wish to remove the docs
+# rm -rf wherehows/share
+
+rm README.md
+
+# correct a few things that won't work in docker.
+
+# leave the option to change the play crypto secret later.
+sed -i 's/changeme/$PLAY_CRYPTO_SECRET/g' wherehows/conf/application.conf
+
+# localhost usually does not work within docker, give people the power
+# to change that from outside of docker with environment variables.
+sed -i 's/localhost:8888/$HDFS_HOST:8888/g' wherehows/conf/application.conf
+sed -i 's/localhost/$MYSQL_HOST/g' wherehows/conf/application.conf
+
+tar zcvf web.tar.gz wherehows
+
+cp web.tar.gz ../../web/archives

docker/backend-service/Dockerfile

@@ -0,0 +1,17 @@
+FROM ubuntu:16.10
+
+RUN mkdir /application;
+
+WORKDIR /application
+
+ADD archives/jdk-8u101-linux-x64.tar.gz /application
+ADD archives/backend-service.tar.gz /application
+
+RUN ln -s jdk1.8.0_101 jdk8;
+
+WORKDIR /application/backend-service
+
+ENV JAVA_HOME=/application/jdk8; \
+    PATH=/application/jdk8/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+CMD bin/backend-service

docker/docker-compose.yml

@@ -0,0 +1,31 @@
+version: "3"
+services:
+  mysql:
+    image: "wherehows/mysql:1"
+    ports:
+      - "3306:3306"
+    environment:
+      MYSQL_ROOT_PASSWORD: wherehows
+  backend:
+    image: "wherehows/backend:1"
+    ports:
+      - "9001:9000"
+    environment:
+      WHZ_DB_USERNAME: wherehows
+      WHZ_DB_PASSWORD: wherehows
+      WHZ_DB_URL: jdbc:mysql://mysql:3306/wherehows
+      PLAY_CRYPTO_SECRET: changeme
+    links:
+      - "mysql:mysql"
+  web:
+    image: "wherehows/web:1"
+    ports:
+      - "9000:9000"
+    environment:
+      WHZ_DB_USERNAME: wherehows
+      WHZ_DB_PASSWORD: wherehows
+      WHZ_DB_URL: jdbc:mysql://mysql:3306/wherehows
+      MYSQL_HOST: mysql
+      HDFS_HOST: "127.0.0.1"
+    links:
+      - "mysql:mysql"

docker/mysql/.gitignore

@@ -0,0 +1 @@
+*_DDL/

docker/mysql/Dockerfile

@@ -0,0 +1,16 @@
+FROM mysql:5.7.17
+
+RUN mkdir /usr/local/wherehows; echo "alias ll=ls -al" > /etc/profile;
+
+COPY binary/*.sh /usr/local/bin/
+COPY ETL_DDL /usr/local/wherehows/ETL_DDL
+COPY WEB_DDL /usr/local/wherehows/WEB_DDL
+COPY binary/*.sql /usr/local/wherehows/
+
+HEALTHCHECK --interval=5s --timeout=3s CMD mysqladmin -pwherehows ping
+
+VOLUME /var/lib/mysql
+VOLUME /var/log/mysql
+
+# note that the default command is still 'mysqld'
+ENTRYPOINT /usr/local/bin/run-for-wherehows.sh

docker/mysql/binary/async-init.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Wait for the server to be completely up before initializing it.
+while [ -z "$(mysqladmin --password=$MYSQL_ROOT_PASSWORD ping 2> /dev/null | grep 'alive')" ]; do sleep 1; done
+
+# must be in the right directory so the create tables script finds the files it sources.
+cd /usr/local/wherehows
+
+mysql --password=$MYSQL_ROOT_PASSWORD < /usr/local/wherehows/init.sql &> /var/log/mysql/create-accounts.log
+mysql --password=$MYSQL_ROOT_PASSWORD -hlocalhost -uwherehows -Dwherehows < /usr/local/wherehows/create_all_tables_wrapper.sql &> /var/log/mysql/create-tables.log

docker/mysql/binary/create_all_tables_wrapper.sql

@@ -0,0 +1,32 @@
+--
+-- Copyright 2015 LinkedIn Corp. All rights reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+--
+
+/* wrapper sql to call all individual DDL */
+-- use wherehows;
+
+source ETL_DDL/dataset_metadata.sql;
+source ETL_DDL/etl_configure_tables.sql;
+source ETL_DDL/executor_metadata.sql;
+source ETL_DDL/git_metadata.sql;
+source ETL_DDL/lineage_metadata.sql;
+source ETL_DDL/metric_metadata.sql;
+source ETL_DDL/owner_metadata.sql;
+source ETL_DDL/patterns.sql;
+source ETL_DDL/kafka_tracking.sql;
+source ETL_DDL/dataset_info_metadata.sql;
+
+source WEB_DDL/track.sql;
+source WEB_DDL/users.sql;
+
+show tables;

docker/mysql/binary/init.sql

@@ -0,0 +1,20 @@
+
+-- note that this may be resuming from a previous run, only do commands if things do not already exist.
+CREATE DATABASE IF NOT EXISTS wherehows
+  DEFAULT CHARACTER SET utf8
+  DEFAULT COLLATE utf8_general_ci;
+
+CREATE USER IF NOT EXISTS 'wherehows'@'localhost' IDENTIFIED BY 'wherehows';
+CREATE USER IF NOT EXISTS 'wherehows'@'%' IDENTIFIED BY 'wherehows';
+
+CREATE USER IF NOT EXISTS 'wherehows_ro'@'localhost' IDENTIFIED BY 'readmetadata';
+CREATE USER IF NOT EXISTS 'wherehows_ro'@'%' IDENTIFIED BY 'readmetadata';
+
+FLUSH PRIVILEGES;
+
+GRANT ALL ON wherehows.* TO 'wherehows'@'localhost';
+GRANT ALL ON wherehows.* TO 'wherehows'@'%';
+GRANT SELECT ON wherehows.* TO 'wherehows_ro'@'localhost';
+GRANT SELECT ON wherehows.* TO 'wherehows_ro'@'%';
+
+FLUSH PRIVILEGES;

docker/mysql/binary/run-for-wherehows.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ -z "$MYSQL_ROOT_PASSWORD" ]; then
+  echo "You must set MYSQL_ROOT_PASSWORD"
+  exit
+fi
+
+ARG="$@"
+if [ -z "$ARG" ]; then
+  echo "No default argument was provided, using mysqld"
+  ARG='mysqld'
+fi
+
+./usr/local/bin/async-init.sh &
+
+# run the normal start up.
+# note that mysqld is the default argument, provided by the original docker image.
+./entrypoint.sh $ARG | tee /var/log/mysql/entrypoint.log

docker/mysql/build-docker.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+VERSION=$1
+if [ -z "$VERSION" ]; then
+  VERSION=1
+fi
+
+docker build -t wherehows/mysql-wherehows:$VERSION .

docker/mysql/copy-ddl.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Copies over the ddl files
+
+if [ -z "$WORKSPACE" ]; then
+  echo "You must set the WORKSPACE environment variable for this to work."
+  exit
+fi
+
+rm -rf *_DDL
+rm bin/create_all_tables_wrapper.sql
+
+DDL_DIR=$WORKSPACE/WhereHows/data-model/DDL
+cp $DDL_DIR/create_all_tables_wrapper.sql bin
+cp -r $DDL_DIR/*_DDL .
+
+# Unfortunately these scripts may be executed multiple times.
+# The data directory is mounted as a volume, meaning that these scripts could run twice for the 
+# same directory.  Change schema to just create tables if they do not already exist.
+sed -i "" -e "s/CREATE TABLE/CREATE TABLE IF NOT EXISTS/g" *_DDL/*
+
+# In some places we just doubled up on IF NOT EXISTS
+sed -i "" -e "s/IF NOT EXISTS IF NOT EXISTS/IF NOT EXISTS/g" *_DDL/*
+

docker/mysql/run-base.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run -e MYSQL_ROOT_PASSWORD=wherehows mysql:8.0.0 

docker/mysql/run-mysql-wherehows-docker.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+VERSION=$1
+if [ -z "$VERSION" ]; then
+  VERSION=1
+fi
+
+docker run --name mysql-wherehows -d -e MYSQL_ROOT_PASSWORD=wherehows -p 3306:3306 wherehows/mysql-wherehows:$VERSION

docker/mysql/run-root.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+VERSION=$1
+if [ -z "$VERSION" ]; then
+  VERSION=1
+fi
+
+docker run -it -e MYSQL_ROOT_PASSWORD=wherehows -p 3306:3306 --entrypoint /bin/bash wherehows/mysql-wherehows:$VERSION

docker/web/Dockerfile

@@ -0,0 +1,17 @@
+FROM ubuntu:16.10
+
+RUN mkdir /application;
+
+WORKDIR /application
+
+ADD archives/jdk-8u101-linux-x64.tar.gz /application
+ADD archives/web.tar.gz /application
+
+RUN ln -s jdk1.8.0_101 jdk8;
+
+WORKDIR /application/wherehows
+
+ENV JAVA_HOME=/application/jdk8; \
+    PATH=/application/jdk8/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+CMD bin/wherehows

prepare-docker-archives.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# assumption: docker is already installed
+# also, you should be in the sudoers
+
+# ensure the SBT opts are sufficient so the application will build
+if [ -z "SBT_OPTS" ]; then
+  export SBT_OPTS='-Xms1G -Xmx1G -Xss2M'
+fi
+
+if [ -d "web/target" ]; then
+  # sometimes this directory causes problems if it exists when the build starts.
+  rm -rf web/target
+fi
+
+# build the application's distribution zip
+./gradlew dist
+
+# move those to a directory where we will work on them more.
+mv backend-service/target/universal/*.zip docker/archive-factory/originals/backend.zip
+mv web/target/universal/*.zip docker/archive-factory/originals/web.zip
+
+cd docker/archive-factory
+
+# unfortunately those zip files are not quite what we want to start with in docker,
+# run these other scripts that will revise them.
+./build-backend.sh
+./build-web.sh
+
+# build the docker images
+cd ../mysql
+./copy-ddl.sh
+docker build -t wherehows/mysql .
+cd ../backend-service
+docker build -t wherehows/backend .
+cd ../web
+docker build -t wherehows/web .
+
+cd ..
+echo "now run this to start the application:"
+echo "docker-compose up"
+echo "you may need to edit the .env file first."