fix(docker): fix volume mounts (#3341)

docker/quickstart.sh

@@ -2,15 +2,12 @@
 
 MONITORING_COMPOSE=""
 if [[ $MONITORING == true ]]; then
-  MONITORING_COMPOSE="-f monitoring/docker-compose.monitoring.yml"
+  MONITORING_COMPOSE="-f quickstart/docker-compose.quickstart.monitoring.yml"
 fi
 
 CONSUMERS_COMPOSE=""
 if [[ $SEPARATE_CONSUMERS == true ]]; then
-  CONSUMERS_COMPOSE="-f docker-compose.consumers.yml -f docker-compose.consumers.dev.yml"
+  CONSUMERS_COMPOSE="-f docker-compose.consumers.yml"
-  if [[ $MONITORING == true ]]; then
-    MONITORING_COMPOSE="-f monitoring/docker-compose.monitoring.yml -f monitoring/docker-compose.consumers.monitoring.yml"
-  fi
 fi
 
 # Quickstarts DataHub by pulling all images from dockerhub and then running the containers locally. No images are
@@ -40,5 +37,5 @@ else
     $MONITORING_COMPOSE $CONSUMERS_COMPOSE pull && \
   docker-compose -p datahub \
     -f quickstart/docker-compose-without-neo4j.quickstart.yml \
-    $MONITORING_COMPOSE $CONSUMERS_COMPOSE up
+    $MONITORING_COMPOSE $CONSUMERS_COMPOSE up $@
 fi

docker/quickstart/docker-compose-without-neo4j.quickstart.yml

@@ -117,7 +117,7 @@ services:
     ports:
     - 3306:3306
     volumes:
-    - ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
+    - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
     - mysqldata:/var/lib/mysql
   mysql-setup:
     container_name: mysql-setup

docker/quickstart/docker-compose.quickstart.monitoring.yml

@@ -0,0 +1,37 @@
+services:
+  datahub-gms:
+    environment:
+    - ENABLE_PROMETHEUS=true
+    - ENABLE_OTEL=true
+    - OTEL_TRACES_EXPORTER=jaeger
+    - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250
+    - OTEL_METRICS_EXPORTER=none
+    - OTEL_SERVICE_NAME=datahub-gms
+    ports:
+    - '4318'
+  grafana:
+    depends_on:
+    - prometheus
+    image: grafana/grafana:latest
+    ports:
+    - 3001:3000
+    volumes:
+    - grafana-storage:/var/lib/grafana
+    - ../monitoring/grafana/datasources:/etc/grafana/provisioning/datasources
+    - ../monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards
+  jaeger-all-in-one:
+    image: jaegertracing/all-in-one:latest
+    ports:
+    - 16686:16686
+    - '14268'
+    - '14250'
+  prometheus:
+    container_name: prometheus
+    image: prom/prometheus:latest
+    ports:
+    - 9089:9090
+    volumes:
+    - ../monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml
+version: '2.3'
+volumes:
+  grafana-storage: null

docker/quickstart/docker-compose.quickstart.yml

@@ -121,7 +121,7 @@ services:
     ports:
     - 3306:3306
     volumes:
-    - ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
+    - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
     - mysqldata:/var/lib/mysql
   mysql-setup:
     container_name: mysql-setup

docker/quickstart/generate_and_compare.sh

@@ -13,6 +13,8 @@ python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.ove
 
 python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml temp-without-neo4j.quickstart.yml
 
+python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml temp.quickstart.monitoring.yml
+
 if cmp docker-compose.quickstart.yml temp.quickstart.yml; then
     printf 'docker-compose.quickstart.yml is up to date.'
 else
@@ -27,3 +29,11 @@ else
     printf 'docker-compose-without-neo4j.quickstart.yml is out of date.'
     exit 1
 fi
+
+if cmp docker-compose.quickstart.monitoring.yml temp.quickstart.monitoring.yml; then
+    printf 'docker-compose.quickstart.monitoring.yml is up to date.'
+    exit 0
+else
+    printf 'docker-compose.quickstart.monitoring.yml is out of date.'
+    exit 1
+fi

docker/quickstart/generate_docker_quickstart.py

@@ -62,6 +62,16 @@ def modify_docker_config(base_path, docker_yaml_config):
         if name in mem_limits:
             service["mem_limit"] = mem_limits[name]
 
+        # 8. Correct relative paths for volume mounts
+        if "volumes" in service:
+            volumes = service["volumes"]
+            for i in range(len(volumes)):
+                ## Quickstart yaml files are located under quickstart. To get correct paths, need to refer to parent directory
+                if volumes[i].startswith("../"):
+                    volumes[i] = "../" + volumes[i]
+                elif volumes[i].startswith("./"):
+                    volumes[i] = "." + volumes[i]
+
     # 8. Set docker compose version to 2.
     # We need at least this version, since we use features like start_period for
     # healthchecks and shell-like variable interpolation.

docker/quickstart/generate_docker_quickstart.sh

@@ -8,8 +8,7 @@ set -euxo pipefail
 python3 -m venv venv
 source venv/bin/activate
 
-cp $(pwd)/../mysql/init.sql $(pwd)/mysql/init.sql
-
 pip install -r requirements.txt
 python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml docker-compose.quickstart.yml
 python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml docker-compose-without-neo4j.quickstart.yml
+python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml docker-compose.quickstart.monitoring.yml

docker/quickstart/mysql/init.sql

@@ -1,43 +0,0 @@
--- create metadata aspect table
-CREATE TABLE metadata_aspect_v2 (
-  urn                           VARCHAR(500) NOT NULL,
-  aspect                        VARCHAR(200) NOT NULL,
-  version                       bigint(20) NOT NULL,
-  metadata                      longtext NOT NULL,
-  systemmetadata                longtext,
-  createdon                     datetime(6) NOT NULL,
-  createdby                     VARCHAR(255) NOT NULL,
-  createdfor                    VARCHAR(255),
-  CONSTRAINT pk_metadata_aspect_v2 PRIMARY KEY (urn,aspect,version)
-);
-
-INSERT INTO metadata_aspect_v2 (urn, aspect, version, metadata, createdon, createdby) VALUES(
-  'urn:li:corpuser:datahub',
-  'corpUserInfo',
-  0,
-  '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}',
-  now(),
-  'urn:li:corpuser:__datahub_system'
-), (
-  'urn:li:corpuser:datahub',
-  'corpUserEditableInfo',
-  0,
-  '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/default_avatar.png"}',
-  now(),
-  'urn:li:corpuser:__datahub_system'
-);
-
--- create metadata index table
-CREATE TABLE metadata_index (
- `id` BIGINT NOT NULL AUTO_INCREMENT,
- `urn` VARCHAR(200) NOT NULL,
- `aspect` VARCHAR(150) NOT NULL,
- `path` VARCHAR(150) NOT NULL,
- `longVal` BIGINT,
- `stringVal` VARCHAR(200),
- `doubleVal` DOUBLE,
- CONSTRAINT id_pk PRIMARY KEY (id),
- INDEX longIndex (`urn`,`aspect`,`path`,`longVal`),
- INDEX stringIndex (`urn`,`aspect`,`path`,`stringVal`),
- INDEX doubleIndex (`urn`,`aspect`,`path`,`doubleVal`)
-);