feat(docker): reduce quickstart footprint (#2744)

2025-12-27 09:58:14 +00:00 · 2021-06-23 12:59:49 -07:00 · 2021-06-23 12:59:49 -07:00 · 5a4d194bad
commit 5a4d194bad
parent 96dde2c734
12 changed files with 67 additions and 54 deletions
--- a/docker/broker/env/docker.env
+++ b/docker/broker/env/docker.env
@ -4,4 +4,4 @@ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEX
 KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
-
+KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
--- a/docker/datahub-frontend/env/docker.env
+++ b/docker/datahub-frontend/env/docker.env
@ -3,6 +3,7 @@ DATAHUB_GMS_PORT=8080
 DATAHUB_SECRET=YouKnowNothing
 DATAHUB_APP_VERSION=1.0
 DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
+JAVA_OPTS=-Xms256m -Xmx256m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=true -Dpidfile.path=/dev/null

 # Uncomment and set these to support SSL connection to GMS
 # NOTE: Currently GMS itself does not offer SSL support, these settings are intended for when there is a proxy in front
--- a/docker/datahub-gms/env/docker-without-neo4j.env
+++ b/docker/datahub-gms/env/docker-without-neo4j.env
@ -9,6 +9,7 @@ KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
 ELASTICSEARCH_HOST=elasticsearch
 ELASTICSEARCH_PORT=9200
 GRAPH_SERVICE_IMPL=elasticsearch
+JAVA_OPTS=-Xms256m -Xmx256m

 MAE_CONSUMER_ENABLED=true
 MCE_CONSUMER_ENABLED=true
--- a/docker/datahub-gms/env/docker.env
+++ b/docker/datahub-gms/env/docker.env
@ -12,6 +12,7 @@ NEO4J_HOST=http://neo4j:7474
 NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=datahub
+JAVA_OPTS=-Xms256m -Xmx256m
 GRAPH_SERVICE_IMPL=neo4j

 MAE_CONSUMER_ENABLED=true
--- a/docker/datahub-gms/env/docker.mariadb.env
+++ b/docker/datahub-gms/env/docker.mariadb.env
@ -13,3 +13,5 @@ NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=datahub
 MAE_CONSUMER_ENABLED=true
 MCE_CONSUMER_ENABLED=true
+JAVA_OPTS=-Xms256m -Xmx256m
+
--- a/docker/datahub-gms/env/docker.postgres.env
+++ b/docker/datahub-gms/env/docker.postgres.env
@ -13,4 +13,6 @@ NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=datahub
 MAE_CONSUMER_ENABLED=true
 MCE_CONSUMER_ENABLED=true
+JAVA_OPTS=-Xms256m -Xmx256m
+

--- a/docker/docker-compose.tools.yml
+++ b/docker/docker-compose.tools.yml
@ -0,0 +1,38 @@
+# Tools useful for operating & debugging DataHub.
+---
+version: '3.8'
+services:
+  kafka-rest-proxy:
+    image: confluentinc/cp-kafka-rest:5.4.0
+    env_file: kafka-rest-proxy/env/docker.env
+    hostname: kafka-rest-proxy
+    container_name: kafka-rest-proxy
+    ports:
+      - "8082:8082"
+    depends_on:
+      - zookeeper
+      - broker
+      - schema-registry
+
+  kafka-topics-ui:
+    image: landoop/kafka-topics-ui:0.9.4
+    env_file: kafka-topics-ui/env/docker.env
+    hostname: kafka-topics-ui
+    container_name: kafka-topics-ui
+    ports:
+      - "18000:8000"
+    depends_on:
+      - zookeeper
+      - broker
+      - schema-registry
+      - kafka-rest-proxy
+
+  kibana:
+    image: kibana:7.9.3
+    env_file: kibana/env/docker.env
+    container_name: kibana
+    hostname: kibana
+    ports:
+      - "5601:5601"
+    depends_on:
+      - elasticsearch
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@ -27,31 +27,6 @@ services:
      - "29092:29092"
      - "9092:9092"

-  kafka-rest-proxy:
-    image: confluentinc/cp-kafka-rest:5.4.0
-    env_file: kafka-rest-proxy/env/docker.env
-    hostname: kafka-rest-proxy
-    container_name: kafka-rest-proxy
-    ports:
-      - "8082:8082"
-    depends_on:
-      - zookeeper
-      - broker
-      - schema-registry
-
-  kafka-topics-ui:
-    image: landoop/kafka-topics-ui:0.9.4
-    env_file: kafka-topics-ui/env/docker.env
-    hostname: kafka-topics-ui
-    container_name: kafka-topics-ui
-    ports:
-      - "18000:8000"
-    depends_on:
-      - zookeeper
-      - broker
-      - schema-registry
-      - kafka-rest-proxy
-
  # This "container" is a workaround to pre-create topics
  kafka-setup:
    build:
@ -75,16 +50,6 @@ services:
    ports:
      - "8081:8081"

-  schema-registry-ui:
-    image: landoop/schema-registry-ui:latest
-    env_file: schema-registry-ui/env/docker.env
-    container_name: schema-registry-ui
-    hostname: schema-registry-ui
-    ports:
-      - "8000:8000"
-    depends_on:
-      - schema-registry
-
  elasticsearch:
    image: elasticsearch:7.9.3
    env_file: elasticsearch/env/docker.env
@ -99,16 +64,6 @@ services:
        start_period: 2m
        retries: 4

-  kibana:
-    image: kibana:7.9.3
-    env_file: kibana/env/docker.env
-    container_name: kibana
-    hostname: kibana
-    ports:
-      - "5601:5601"
-    depends_on:
-      - elasticsearch
-
  neo4j:
    image: neo4j:4.0.6
    env_file: neo4j/env/docker.env
--- a/docker/elasticsearch/env/docker.env
+++ b/docker/elasticsearch/env/docker.env
@ -1,3 +1,3 @@
 discovery.type=single-node
 xpack.security.enabled=false
-ES_JAVA_OPTS=-Xms512m -Xmx512m
+ES_JAVA_OPTS=-Xms256m -Xmx256m
--- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
+++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
@ -13,6 +13,7 @@ services:
    - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
    - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
    - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
+    - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
    hostname: broker
    image: confluentinc/cp-kafka:5.4.0
    ports:
@ -28,12 +29,16 @@ services:
    - DATAHUB_SECRET=YouKnowNothing
    - DATAHUB_APP_VERSION=1.0
    - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
+    - JAVA_OPTS=-Xms256m -Xmx256m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf
+      -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml
+      -Dlogback.debug=true -Dpidfile.path=/dev/null
    - KAFKA_BOOTSTRAP_SERVER=broker:29092
    - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
    - ELASTIC_CLIENT_HOST=elasticsearch
    - ELASTIC_CLIENT_PORT=9200
    hostname: datahub-frontend-react
    image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-latest}
+    mem_limit: 512m
    ports:
    - 9002:9002
  datahub-gms:
@ -52,11 +57,12 @@ services:
    - ELASTICSEARCH_HOST=elasticsearch
    - ELASTICSEARCH_PORT=9200
    - GRAPH_SERVICE_IMPL=elasticsearch
+    - JAVA_OPTS=-Xms256m -Xmx256m
    - MAE_CONSUMER_ENABLED=true
    - MCE_CONSUMER_ENABLED=true
    hostname: datahub-gms
    image: linkedin/datahub-gms:${DATAHUB_VERSION:-latest}
-    mem_limit: 850m
+    mem_limit: 512m
    ports:
    - 8080:8080
  elasticsearch:
@ -64,7 +70,7 @@ services:
    environment:
    - discovery.type=single-node
    - xpack.security.enabled=false
-    - ES_JAVA_OPTS=-Xms512m -Xmx512m
+    - ES_JAVA_OPTS=-Xms256m -Xmx256m
    healthcheck:
      retries: 4
      start_period: 2m
--- a/docker/quickstart/docker-compose.quickstart.yml
+++ b/docker/quickstart/docker-compose.quickstart.yml
@ -13,6 +13,7 @@ services:
    - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
    - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
    - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
+    - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
    hostname: broker
    image: confluentinc/cp-kafka:5.4.0
    ports:
@ -28,12 +29,16 @@ services:
    - DATAHUB_SECRET=YouKnowNothing
    - DATAHUB_APP_VERSION=1.0
    - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
+    - JAVA_OPTS=-Xms256m -Xmx256m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf
+      -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml
+      -Dlogback.debug=true -Dpidfile.path=/dev/null
    - KAFKA_BOOTSTRAP_SERVER=broker:29092
    - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
    - ELASTIC_CLIENT_HOST=elasticsearch
    - ELASTIC_CLIENT_PORT=9200
    hostname: datahub-frontend-react
    image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head}
+    mem_limit: 512m
    ports:
    - 9002:9002
  datahub-gms:
@ -55,12 +60,13 @@ services:
    - NEO4J_URI=bolt://neo4j
    - NEO4J_USERNAME=neo4j
    - NEO4J_PASSWORD=datahub
+    - JAVA_OPTS=-Xms256m -Xmx256m
    - GRAPH_SERVICE_IMPL=neo4j
    - MAE_CONSUMER_ENABLED=true
    - MCE_CONSUMER_ENABLED=true
    hostname: datahub-gms
    image: linkedin/datahub-gms:${DATAHUB_VERSION:-head}
-    mem_limit: 850m
+    mem_limit: 512m
    ports:
    - 8080:8080
  elasticsearch:
@ -68,7 +74,7 @@ services:
    environment:
    - discovery.type=single-node
    - xpack.security.enabled=false
-    - ES_JAVA_OPTS=-Xms512m -Xmx512m
+    - ES_JAVA_OPTS=-Xms256m -Xmx256m
    healthcheck:
      retries: 4
      start_period: 2m
--- a/docker/quickstart/generate_docker_quickstart.py
+++ b/docker/quickstart/generate_docker_quickstart.py
@ -15,10 +15,11 @@ omitted_services = [
    "schema-registry-ui",
    "kibana",
 ]
+# Note that these are upper bounds on memory usage. Once exceeded, the container is killed.
+# Each service will be configured to use much less Java heap space than allocated here.
 mem_limits = {
-    "datahub-gms": "850m",
-    "datahub-mae-consumer": "256m",
-    "datahub-mce-consumer": "384m",
+    "datahub-gms": "512m",
+    "datahub-frontend-react": "512m",
    "elasticsearch": "1g",
 }