diff --git a/docker/quickstart.sh b/docker/quickstart.sh index 51d0d38fa8..5543e534b7 100755 --- a/docker/quickstart.sh +++ b/docker/quickstart.sh @@ -2,15 +2,12 @@ MONITORING_COMPOSE="" if [[ $MONITORING == true ]]; then - MONITORING_COMPOSE="-f monitoring/docker-compose.monitoring.yml" + MONITORING_COMPOSE="-f quickstart/docker-compose.quickstart.monitoring.yml" fi CONSUMERS_COMPOSE="" if [[ $SEPARATE_CONSUMERS == true ]]; then - CONSUMERS_COMPOSE="-f docker-compose.consumers.yml -f docker-compose.consumers.dev.yml" - if [[ $MONITORING == true ]]; then - MONITORING_COMPOSE="-f monitoring/docker-compose.monitoring.yml -f monitoring/docker-compose.consumers.monitoring.yml" - fi + CONSUMERS_COMPOSE="-f docker-compose.consumers.yml" fi # Quickstarts DataHub by pulling all images from dockerhub and then running the containers locally. No images are @@ -40,5 +37,5 @@ else $MONITORING_COMPOSE $CONSUMERS_COMPOSE pull && \ docker-compose -p datahub \ -f quickstart/docker-compose-without-neo4j.quickstart.yml \ - $MONITORING_COMPOSE $CONSUMERS_COMPOSE up + $MONITORING_COMPOSE $CONSUMERS_COMPOSE up $@ fi diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 17141df7c8..49fac8983a 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -117,7 +117,7 @@ services: ports: - 3306:3306 volumes: - - ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql + - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - mysqldata:/var/lib/mysql mysql-setup: container_name: mysql-setup diff --git a/docker/quickstart/docker-compose.quickstart.monitoring.yml b/docker/quickstart/docker-compose.quickstart.monitoring.yml new file mode 100644 index 0000000000..0c9e822e2d --- /dev/null +++ b/docker/quickstart/docker-compose.quickstart.monitoring.yml @@ -0,0 +1,37 @@ +services: + datahub-gms: + environment: + - ENABLE_PROMETHEUS=true + - ENABLE_OTEL=true + - OTEL_TRACES_EXPORTER=jaeger + - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250 + - OTEL_METRICS_EXPORTER=none + - OTEL_SERVICE_NAME=datahub-gms + ports: + - '4318' + grafana: + depends_on: + - prometheus + image: grafana/grafana:latest + ports: + - 3001:3000 + volumes: + - grafana-storage:/var/lib/grafana + - ../monitoring/grafana/datasources:/etc/grafana/provisioning/datasources + - ../monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards + jaeger-all-in-one: + image: jaegertracing/all-in-one:latest + ports: + - 16686:16686 + - '14268' + - '14250' + prometheus: + container_name: prometheus + image: prom/prometheus:latest + ports: + - 9089:9090 + volumes: + - ../monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml +version: '2.3' +volumes: + grafana-storage: null diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index d2e7ae6121..6de1c37e97 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -121,7 +121,7 @@ services: ports: - 3306:3306 volumes: - - ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql + - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - mysqldata:/var/lib/mysql mysql-setup: container_name: mysql-setup diff --git a/docker/quickstart/generate_and_compare.sh b/docker/quickstart/generate_and_compare.sh index 3c99c652d8..28ce775836 100755 --- a/docker/quickstart/generate_and_compare.sh +++ b/docker/quickstart/generate_and_compare.sh @@ -13,6 +13,8 @@ python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.ove python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml temp-without-neo4j.quickstart.yml +python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml temp.quickstart.monitoring.yml + if cmp docker-compose.quickstart.yml temp.quickstart.yml; then printf 'docker-compose.quickstart.yml is up to date.' else @@ -27,3 +29,11 @@ else printf 'docker-compose-without-neo4j.quickstart.yml is out of date.' exit 1 fi + +if cmp docker-compose.quickstart.monitoring.yml temp.quickstart.monitoring.yml; then + printf 'docker-compose.quickstart.monitoring.yml is up to date.' + exit 0 +else + printf 'docker-compose.quickstart.monitoring.yml is out of date.' + exit 1 +fi diff --git a/docker/quickstart/generate_docker_quickstart.py b/docker/quickstart/generate_docker_quickstart.py index 41ac48d1c6..7576b1a228 100644 --- a/docker/quickstart/generate_docker_quickstart.py +++ b/docker/quickstart/generate_docker_quickstart.py @@ -62,6 +62,16 @@ def modify_docker_config(base_path, docker_yaml_config): if name in mem_limits: service["mem_limit"] = mem_limits[name] + # 8. Correct relative paths for volume mounts + if "volumes" in service: + volumes = service["volumes"] + for i in range(len(volumes)): + ## Quickstart yaml files are located under quickstart. To get correct paths, need to refer to parent directory + if volumes[i].startswith("../"): + volumes[i] = "../" + volumes[i] + elif volumes[i].startswith("./"): + volumes[i] = "." + volumes[i] + # 8. Set docker compose version to 2. # We need at least this version, since we use features like start_period for # healthchecks and shell-like variable interpolation. diff --git a/docker/quickstart/generate_docker_quickstart.sh b/docker/quickstart/generate_docker_quickstart.sh index 7682eaa760..39c98becc4 100755 --- a/docker/quickstart/generate_docker_quickstart.sh +++ b/docker/quickstart/generate_docker_quickstart.sh @@ -8,8 +8,7 @@ set -euxo pipefail python3 -m venv venv source venv/bin/activate -cp $(pwd)/../mysql/init.sql $(pwd)/mysql/init.sql - pip install -r requirements.txt python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml docker-compose.quickstart.yml python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml docker-compose-without-neo4j.quickstart.yml +python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml docker-compose.quickstart.monitoring.yml diff --git a/docker/quickstart/mysql/init.sql b/docker/quickstart/mysql/init.sql deleted file mode 100644 index d94e6bd9b7..0000000000 --- a/docker/quickstart/mysql/init.sql +++ /dev/null @@ -1,43 +0,0 @@ --- create metadata aspect table -CREATE TABLE metadata_aspect_v2 ( - urn VARCHAR(500) NOT NULL, - aspect VARCHAR(200) NOT NULL, - version bigint(20) NOT NULL, - metadata longtext NOT NULL, - systemmetadata longtext, - createdon datetime(6) NOT NULL, - createdby VARCHAR(255) NOT NULL, - createdfor VARCHAR(255), - CONSTRAINT pk_metadata_aspect_v2 PRIMARY KEY (urn,aspect,version) -); - -INSERT INTO metadata_aspect_v2 (urn, aspect, version, metadata, createdon, createdby) VALUES( - 'urn:li:corpuser:datahub', - 'corpUserInfo', - 0, - '{"displayName":"Data Hub","active":true,"fullName":"Data Hub","email":"datahub@linkedin.com"}', - now(), - 'urn:li:corpuser:__datahub_system' -), ( - 'urn:li:corpuser:datahub', - 'corpUserEditableInfo', - 0, - '{"skills":[],"teams":[],"pictureLink":"https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web-react/src/images/default_avatar.png"}', - now(), - 'urn:li:corpuser:__datahub_system' -); - --- create metadata index table -CREATE TABLE metadata_index ( - `id` BIGINT NOT NULL AUTO_INCREMENT, - `urn` VARCHAR(200) NOT NULL, - `aspect` VARCHAR(150) NOT NULL, - `path` VARCHAR(150) NOT NULL, - `longVal` BIGINT, - `stringVal` VARCHAR(200), - `doubleVal` DOUBLE, - CONSTRAINT id_pk PRIMARY KEY (id), - INDEX longIndex (`urn`,`aspect`,`path`,`longVal`), - INDEX stringIndex (`urn`,`aspect`,`path`,`stringVal`), - INDEX doubleIndex (`urn`,`aspect`,`path`,`doubleVal`) -);