diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml index 1af05dfae0..b216f00528 100644 --- a/docker/quickstart/docker-compose-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-m1.quickstart.yml @@ -48,9 +48,7 @@ services: - DATAHUB_SECRET=YouKnowNothing - DATAHUB_APP_VERSION=1.0 - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB - - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf - -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml - -Dlogback.debug=false -Dpidfile.path=/dev/null + - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null - KAFKA_BOOTSTRAP_SERVER=broker:29092 - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 - ELASTIC_CLIENT_HOST=elasticsearch @@ -140,8 +138,7 @@ services: start_period: 2m test: - CMD-SHELL - - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' - || exit 1 + - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1 hostname: elasticsearch image: elasticsearch:7.10.1 mem_limit: 1g diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index 19fede733d..39edef2b9f 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -46,9 +46,7 @@ services: - DATAHUB_SECRET=YouKnowNothing - DATAHUB_APP_VERSION=1.0 - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB - - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf - -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml - -Dlogback.debug=false -Dpidfile.path=/dev/null + - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null - KAFKA_BOOTSTRAP_SERVER=broker:29092 - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 - ELASTIC_CLIENT_HOST=elasticsearch @@ -95,8 +93,6 @@ services: volumes: - ${HOME}/.datahub/plugins:/etc/datahub/plugins datahub-upgrade: - labels: - datahub_setup_job: true command: - -u - SystemUpdate @@ -133,8 +129,7 @@ services: start_period: 2m test: - CMD-SHELL - - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' - || exit 1 + - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1 hostname: elasticsearch image: elasticsearch:7.10.1 mem_limit: 1g @@ -143,8 +138,6 @@ services: volumes: - esdata:/usr/share/elasticsearch/data elasticsearch-setup: - labels: - datahub_setup_job: true container_name: elasticsearch-setup depends_on: - elasticsearch @@ -157,8 +150,6 @@ services: labels: datahub_setup_job: true kafka-setup: - labels: - datahub_setup_job: true container_name: kafka-setup depends_on: - broker @@ -187,8 +178,6 @@ services: - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - mysqldata:/var/lib/mysql mysql-setup: - labels: - datahub_setup_job: true container_name: mysql-setup depends_on: - mysql diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 27f17d5619..cfb57d8c0a 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -5,115 +5,115 @@ services: broker: container_name: broker depends_on: - - zookeeper + - zookeeper environment: - - KAFKA_BROKER_ID=1 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m + - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false hostname: broker image: confluentinc/cp-kafka:7.2.2 ports: - - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 + - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 datahub-actions: depends_on: - - datahub-gms + - datahub-gms environment: - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - DATAHUB_GMS_PROTOCOL=http - - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT - - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 - - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - - SCHEMA_REGISTRY_URL=http://schema-registry:8081 + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - DATAHUB_GMS_PROTOCOL=http + - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system + - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT + - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 + - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 + - SCHEMA_REGISTRY_URL=http://schema-registry:8081 hostname: actions image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} restart: on-failure:5 datahub-frontend-react: container_name: datahub-frontend-react depends_on: - - datahub-gms + - datahub-gms environment: - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - DATAHUB_SECRET=YouKnowNothing - - DATAHUB_APP_VERSION=1.0 - - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB - - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 - - ELASTIC_CLIENT_HOST=elasticsearch - - ELASTIC_CLIENT_PORT=9200 + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - DATAHUB_SECRET=YouKnowNothing + - DATAHUB_APP_VERSION=1.0 + - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB + - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 + - ELASTIC_CLIENT_HOST=elasticsearch + - ELASTIC_CLIENT_PORT=9200 hostname: datahub-frontend-react image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} ports: - - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 + - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - ${HOME}/.datahub/plugins:/etc/datahub/plugins datahub-gms: container_name: datahub-gms depends_on: - - mysql + - mysql environment: - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_PASSWORD=datahub - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - EBEAN_DATASOURCE_USERNAME=datahub - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_PORT=9200 - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - ENTITY_SERVICE_ENABLE_RETENTION=true - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - GRAPH_SERVICE_IMPL=elasticsearch - - JAVA_OPTS=-Xms1g -Xmx1g - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - MAE_CONSUMER_ENABLED=true - - MCE_CONSUMER_ENABLED=true - - PE_CONSUMER_ENABLED=true - - UI_INGESTION_ENABLED=true + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_PASSWORD=datahub + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - EBEAN_DATASOURCE_USERNAME=datahub + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - ELASTICSEARCH_PORT=9200 + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - ENTITY_SERVICE_ENABLE_RETENTION=true + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - GRAPH_SERVICE_IMPL=elasticsearch + - JAVA_OPTS=-Xms1g -Xmx1g + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - MAE_CONSUMER_ENABLED=true + - MCE_CONSUMER_ENABLED=true + - PE_CONSUMER_ENABLED=true + - UI_INGESTION_ENABLED=true hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: - - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 + - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - ${HOME}/.datahub/plugins:/etc/datahub/plugins datahub-upgrade: command: - - -u - - SystemUpdate + - -u + - SystemUpdate container_name: datahub-upgrade environment: - - EBEAN_DATASOURCE_USERNAME=datahub - - EBEAN_DATASOURCE_PASSWORD=datahub - - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false - - GRAPH_SERVICE_IMPL=elasticsearch - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - EBEAN_DATASOURCE_USERNAME=datahub + - EBEAN_DATASOURCE_PASSWORD=datahub + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false + - GRAPH_SERVICE_IMPL=elasticsearch + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml hostname: datahub-upgrade image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head} labels: @@ -121,30 +121,30 @@ services: elasticsearch: container_name: elasticsearch environment: - - discovery.type=single-node - - xpack.security.enabled=false - - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true + - discovery.type=single-node + - xpack.security.enabled=false + - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true healthcheck: retries: 4 start_period: 2m test: - - CMD-SHELL - - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1 + - CMD-SHELL + - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1 hostname: elasticsearch image: elasticsearch:7.10.1 mem_limit: 1g ports: - - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 + - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 volumes: - - esdata:/usr/share/elasticsearch/data + - esdata:/usr/share/elasticsearch/data elasticsearch-setup: container_name: elasticsearch-setup depends_on: - - elasticsearch + - elasticsearch environment: - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_PROTOCOL=http + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_PROTOCOL=http hostname: elasticsearch-setup image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} labels: @@ -152,12 +152,12 @@ services: kafka-setup: container_name: kafka-setup depends_on: - - broker - - schema-registry + - broker + - schema-registry environment: - - DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false} - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false} + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 hostname: kafka-setup image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} labels: @@ -166,55 +166,55 @@ services: command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password container_name: mysql environment: - - MYSQL_DATABASE=datahub - - MYSQL_USER=datahub - - MYSQL_PASSWORD=datahub - - MYSQL_ROOT_PASSWORD=datahub + - MYSQL_DATABASE=datahub + - MYSQL_USER=datahub + - MYSQL_PASSWORD=datahub + - MYSQL_ROOT_PASSWORD=datahub hostname: mysql image: mysql:5.7 ports: - - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 + - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 volumes: - - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - - mysqldata:/var/lib/mysql + - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql + - mysqldata:/var/lib/mysql mysql-setup: - labels: - datahub_setup_job: true container_name: mysql-setup depends_on: - - mysql + - mysql environment: - - MYSQL_HOST=mysql - - MYSQL_PORT=3306 - - MYSQL_USERNAME=datahub - - MYSQL_PASSWORD=datahub - - DATAHUB_DB_NAME=datahub + - MYSQL_HOST=mysql + - MYSQL_PORT=3306 + - MYSQL_USERNAME=datahub + - MYSQL_PASSWORD=datahub + - DATAHUB_DB_NAME=datahub hostname: mysql-setup image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head} + labels: + datahub_setup_job: true schema-registry: container_name: schema-registry depends_on: - - broker + - broker environment: - - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT - - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 + - SCHEMA_REGISTRY_HOST_NAME=schemaregistry + - SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT + - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 hostname: schema-registry image: confluentinc/cp-schema-registry:7.2.2 ports: - - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 + - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: container_name: zookeeper environment: - - ZOOKEEPER_CLIENT_PORT=2181 - - ZOOKEEPER_TICK_TIME=2000 + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 hostname: zookeeper image: confluentinc/cp-zookeeper:7.2.2 ports: - - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 + - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: - - zkdata:/var/lib/zookeeper -version: "2.3" + - zkdata:/var/lib/zookeeper +version: '2.3' volumes: esdata: null mysqldata: null diff --git a/docker/quickstart/docker-compose.monitoring.quickstart.yml b/docker/quickstart/docker-compose.monitoring.quickstart.yml index 37ccd057a0..0796692605 100644 --- a/docker/quickstart/docker-compose.monitoring.quickstart.yml +++ b/docker/quickstart/docker-compose.monitoring.quickstart.yml @@ -1,47 +1,47 @@ services: datahub-frontend-react: environment: - - ENABLE_PROMETHEUS=true - - ENABLE_OTEL=true - - OTEL_TRACES_EXPORTER=jaeger - - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250 - - OTEL_METRICS_EXPORTER=none - - OTEL_SERVICE_NAME=datahub-gms + - ENABLE_PROMETHEUS=true + - ENABLE_OTEL=true + - OTEL_TRACES_EXPORTER=jaeger + - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250 + - OTEL_METRICS_EXPORTER=none + - OTEL_SERVICE_NAME=datahub-gms ports: - - "4318" + - '4318' datahub-gms: environment: - - ENABLE_PROMETHEUS=true - - ENABLE_OTEL=true - - OTEL_TRACES_EXPORTER=jaeger - - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250 - - OTEL_METRICS_EXPORTER=none - - OTEL_SERVICE_NAME=datahub-gms + - ENABLE_PROMETHEUS=true + - ENABLE_OTEL=true + - OTEL_TRACES_EXPORTER=jaeger + - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250 + - OTEL_METRICS_EXPORTER=none + - OTEL_SERVICE_NAME=datahub-gms ports: - - "4318" + - '4318' grafana: depends_on: - - prometheus + - prometheus image: grafana/grafana:9.1.4 ports: - - 3001:3000 + - 3001:3000 volumes: - - grafana-storage:/var/lib/grafana - - ../monitoring/grafana/datasources:/etc/grafana/provisioning/datasources - - ../monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards + - grafana-storage:/var/lib/grafana + - ../monitoring/grafana/datasources:/etc/grafana/provisioning/datasources + - ../monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards jaeger-all-in-one: image: jaegertracing/all-in-one:latest ports: - - 16686:16686 - - "14268" - - "14250" + - 16686:16686 + - '14268' + - '14250' prometheus: container_name: prometheus image: prom/prometheus:latest ports: - - 9089:9090 + - 9089:9090 volumes: - - ../monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml -version: "2.3" + - ../monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml +version: '2.3' volumes: grafana-storage: null diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index ee9efac705..3b6765cdfa 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -5,124 +5,124 @@ services: broker: container_name: broker depends_on: - - zookeeper + - zookeeper environment: - - KAFKA_BROKER_ID=1 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m + - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false hostname: broker image: confluentinc/cp-kafka:7.2.2 ports: - - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 + - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 volumes: - - broker:/var/lib/kafka/data/ + - broker:/var/lib/kafka/data/ datahub-actions: depends_on: - - datahub-gms + - datahub-gms environment: - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - DATAHUB_GMS_PROTOCOL=http - - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT - - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 - - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - - SCHEMA_REGISTRY_URL=http://schema-registry:8081 + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - DATAHUB_GMS_PROTOCOL=http + - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system + - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT + - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 + - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 + - SCHEMA_REGISTRY_URL=http://schema-registry:8081 hostname: actions image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} restart: on-failure:5 datahub-frontend-react: container_name: datahub-frontend-react depends_on: - - datahub-gms + - datahub-gms environment: - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - DATAHUB_SECRET=YouKnowNothing - - DATAHUB_APP_VERSION=1.0 - - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB - - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 - - ELASTIC_CLIENT_HOST=elasticsearch - - ELASTIC_CLIENT_PORT=9200 + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - DATAHUB_SECRET=YouKnowNothing + - DATAHUB_APP_VERSION=1.0 + - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB + - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 + - ELASTIC_CLIENT_HOST=elasticsearch + - ELASTIC_CLIENT_PORT=9200 hostname: datahub-frontend-react image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} ports: - - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 + - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - ${HOME}/.datahub/plugins:/etc/datahub/plugins datahub-gms: container_name: datahub-gms depends_on: - - mysql - - neo4j + - mysql + - neo4j environment: - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms - - EBEAN_DATASOURCE_USERNAME=datahub - - EBEAN_DATASOURCE_PASSWORD=datahub - - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - NEO4J_HOST=http://neo4j:7474 - - NEO4J_URI=bolt://neo4j - - NEO4J_USERNAME=neo4j - - NEO4J_PASSWORD=datahub - - JAVA_OPTS=-Xms1g -Xmx1g - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - GRAPH_SERVICE_IMPL=neo4j - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - ENTITY_SERVICE_ENABLE_RETENTION=true - - MAE_CONSUMER_ENABLED=true - - MCE_CONSUMER_ENABLED=true - - PE_CONSUMER_ENABLED=true - - UI_INGESTION_ENABLED=true - - METADATA_SERVICE_AUTH_ENABLED=false + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms + - EBEAN_DATASOURCE_USERNAME=datahub + - EBEAN_DATASOURCE_PASSWORD=datahub + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - NEO4J_HOST=http://neo4j:7474 + - NEO4J_URI=bolt://neo4j + - NEO4J_USERNAME=neo4j + - NEO4J_PASSWORD=datahub + - JAVA_OPTS=-Xms1g -Xmx1g + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - GRAPH_SERVICE_IMPL=neo4j + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - ENTITY_SERVICE_ENABLE_RETENTION=true + - MAE_CONSUMER_ENABLED=true + - MCE_CONSUMER_ENABLED=true + - PE_CONSUMER_ENABLED=true + - UI_INGESTION_ENABLED=true + - METADATA_SERVICE_AUTH_ENABLED=false hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: - - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 + - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 volumes: - - ${HOME}/.datahub/plugins/:/etc/datahub/plugins - - ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources + - ${HOME}/.datahub/plugins/:/etc/datahub/plugins + - ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources datahub-upgrade: command: - - -u - - SystemUpdate + - -u + - SystemUpdate container_name: datahub-upgrade environment: - - EBEAN_DATASOURCE_USERNAME=datahub - - EBEAN_DATASOURCE_PASSWORD=datahub - - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false - - GRAPH_SERVICE_IMPL=elasticsearch - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - EBEAN_DATASOURCE_USERNAME=datahub + - EBEAN_DATASOURCE_PASSWORD=datahub + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false + - GRAPH_SERVICE_IMPL=elasticsearch + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml hostname: datahub-upgrade image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head} labels: @@ -130,30 +130,30 @@ services: elasticsearch: container_name: elasticsearch environment: - - discovery.type=single-node - - xpack.security.enabled=false - - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true + - discovery.type=single-node + - xpack.security.enabled=false + - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true healthcheck: retries: 4 start_period: 2m test: - - CMD-SHELL - - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1 + - CMD-SHELL + - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1 hostname: elasticsearch image: elasticsearch:7.10.1 mem_limit: 1g ports: - - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 + - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 volumes: - - esdata:/usr/share/elasticsearch/data + - esdata:/usr/share/elasticsearch/data elasticsearch-setup: container_name: elasticsearch-setup depends_on: - - elasticsearch + - elasticsearch environment: - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_PROTOCOL=http + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_PROTOCOL=http hostname: elasticsearch-setup image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} labels: @@ -161,12 +161,12 @@ services: kafka-setup: container_name: kafka-setup depends_on: - - broker - - schema-registry + - broker + - schema-registry environment: - - DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false} - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false} + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 hostname: kafka-setup image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} labels: @@ -175,27 +175,27 @@ services: command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password container_name: mysql environment: - - MYSQL_DATABASE=datahub - - MYSQL_USER=datahub - - MYSQL_PASSWORD=datahub - - MYSQL_ROOT_PASSWORD=datahub + - MYSQL_DATABASE=datahub + - MYSQL_USER=datahub + - MYSQL_PASSWORD=datahub + - MYSQL_ROOT_PASSWORD=datahub hostname: mysql image: mysql:5.7 ports: - - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 + - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 volumes: - - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - - mysqldata:/var/lib/mysql + - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql + - mysqldata:/var/lib/mysql mysql-setup: container_name: mysql-setup depends_on: - - mysql + - mysql environment: - - MYSQL_HOST=mysql - - MYSQL_PORT=3306 - - MYSQL_USERNAME=datahub - - MYSQL_PASSWORD=datahub - - DATAHUB_DB_NAME=datahub + - MYSQL_HOST=mysql + - MYSQL_PORT=3306 + - MYSQL_USERNAME=datahub + - MYSQL_PASSWORD=datahub + - DATAHUB_DB_NAME=datahub hostname: mysql-setup image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head} labels: @@ -203,40 +203,40 @@ services: neo4j: container_name: neo4j environment: - - NEO4J_AUTH=neo4j/datahub - - NEO4J_dbms_default__database=graph.db - - NEO4J_dbms_allow__upgrade=true + - NEO4J_AUTH=neo4j/datahub + - NEO4J_dbms_default__database=graph.db + - NEO4J_dbms_allow__upgrade=true hostname: neo4j image: neo4j:4.4.9-community ports: - - ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474 - - ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687 + - ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474 + - ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687 volumes: - - neo4jdata:/data + - neo4jdata:/data schema-registry: container_name: schema-registry depends_on: - - broker + - broker environment: - - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT - - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 + - SCHEMA_REGISTRY_HOST_NAME=schemaregistry + - SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT + - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 hostname: schema-registry image: confluentinc/cp-schema-registry:7.2.2 ports: - - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 + - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: container_name: zookeeper environment: - - ZOOKEEPER_CLIENT_PORT=2181 - - ZOOKEEPER_TICK_TIME=2000 + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 hostname: zookeeper image: confluentinc/cp-zookeeper:7.2.2 ports: - - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 + - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: - - zkdata:/var/lib/zookeeper -version: "2.3" + - zkdata:/var/lib/zookeeper +version: '2.3' volumes: broker: null esdata: null diff --git a/docker/quickstart/generate_and_compare.sh b/docker/quickstart/generate_and_compare.sh index 770c07c5e3..e34abeb982 100755 --- a/docker/quickstart/generate_and_compare.sh +++ b/docker/quickstart/generate_and_compare.sh @@ -1,8 +1,5 @@ #!/bin/bash -# this scripts checks if docker-compose$flavour.quickstart.yml is up to date for these 'flavours': -FLAVOURS=("" "-without-neo4j" ".monitoring") - DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "$DIR" @@ -12,21 +9,4 @@ python3 -m venv venv source venv/bin/activate pip install -r requirements.txt -python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml temp.quickstart.yml -python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml temp-without-neo4j.quickstart.yml -python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml ../docker-compose.m1.yml temp-m1.quickstart.yml -python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml ../docker-compose-without-neo4j.m1.yml temp-without-neo4j-m1.quickstart.yml -python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml temp.monitoring.quickstart.yml -python generate_docker_quickstart.py ../docker-compose.consumers.yml temp.consumers.quickstart.yml -python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml temp.consumers-without-neo4j.quickstart.yml - -for flavour in "${FLAVOURS[@]}" -do - - if cmp <(yq -i -P 'sort_keys(..)' docker-compose$flavour.quickstart.yml) <(yq -i -P 'sort_keys(..)' temp$flavour.quickstart.yml); then - echo "docker-compose$flavour.quickstart.yml is up to date." - else - echo "docker-compose$flavour.quickstart.yml is out of date." - exit 1 - fi -done +python generate_docker_quickstart.py check-all diff --git a/docker/quickstart/generate_docker_quickstart.py b/docker/quickstart/generate_docker_quickstart.py index 74353c1e8b..d2c22e4234 100644 --- a/docker/quickstart/generate_docker_quickstart.py +++ b/docker/quickstart/generate_docker_quickstart.py @@ -1,4 +1,8 @@ +from io import StringIO +from typing import List import os +import sys +import pathlib from collections.abc import Mapping import click @@ -6,8 +10,38 @@ import yaml from dotenv import dotenv_values from yaml import Loader -# Generates a merged docker-compose file with env variables inlined. -# Usage: python3 docker_compose_cli_gen.py ../docker-compose.yml ../docker-compose.override.yml ../docker-compose-gen.yml +COMPOSE_SPECS = { + "docker-compose.quickstart.yml": [ + "../docker-compose.yml", + "../docker-compose.override.yml", + ], + "docker-compose-m1.quickstart.yml": [ + "../docker-compose.yml", + "../docker-compose.override.yml", + "../docker-compose.m1.yml", + ], + "docker-compose-without-neo4j.quickstart.yml": [ + "../docker-compose-without-neo4j.yml", + "../docker-compose-without-neo4j.override.yml", + ], + "docker-compose-without-neo4j-m1.quickstart.yml": [ + "../docker-compose-without-neo4j.yml", + "../docker-compose-without-neo4j.override.yml", + "../docker-compose-without-neo4j.m1.yml", + ], + "docker-compose.monitoring.quickstart.yml": [ + "../monitoring/docker-compose.monitoring.yml", + ], + "docker-compose.consumers.quickstart.yml": [ + "../docker-compose.consumers.yml", + ], + "docker-compose.consumers-without-neo4j.quickstart.yml": [ + "../docker-compose.consumers-without-neo4j.yml", + ], + "docker-compose.kafka-setup.quickstart.yml": [ + "../docker-compose.kafka-setup.yml", + ], +} omitted_services = [ "kafka-rest-proxy", @@ -34,6 +68,7 @@ def dict_merge(dct, merge_dct): else: dct[k] = merge_dct[k] + def modify_docker_config(base_path, docker_yaml_config): if not docker_yaml_config["services"]: docker_yaml_config["services"] = {} @@ -91,17 +126,35 @@ def modify_docker_config(base_path, docker_yaml_config): docker_yaml_config["version"] = "2.3" -@click.command() -@click.argument( - "compose-files", - nargs=-1, - type=click.Path( - exists=True, - dir_okay=False, - ), -) -@click.argument("output-file", type=click.Path()) -def generate(compose_files, output_file) -> None: +def dedup_env_vars(merged_docker_config): + for service in merged_docker_config["services"]: + if "environment" in merged_docker_config["services"][service]: + lst = merged_docker_config["services"][service]["environment"] + if lst is not None: + # use a set to cache duplicates + caches = set() + results = {} + for item in lst: + partitions = item.rpartition("=") + prefix = partitions[0] + suffix = partitions[1] + # check whether prefix already exists + if prefix not in caches and suffix != "": + results[prefix] = item + caches.add(prefix) + if set(lst) != set([v for k, v in results.items()]): + sorted_vars = sorted([k for k in results]) + merged_docker_config["services"][service]["environment"] = [ + results[var] for var in sorted_vars + ] + + +def merge_files(compose_files: List[str]) -> str: + """ + Generates a merged docker-compose file with env variables inlined. + + Example Usage: python3 generate_docker_quickstart.py generate-one ../docker-compose.yml ../docker-compose.override.yml ../docker-compose-gen.yml + """ # Resolve .env files to inlined vars modified_files = [] @@ -121,40 +174,77 @@ def generate(compose_files, output_file) -> None: # Dedup env vars, last wins dedup_env_vars(merged_docker_config) + # Generate yaml to string. + out = StringIO() + yaml.dump( + merged_docker_config, + out, + default_flow_style=False, + width=1000, + ) + return out.getvalue() + + +@click.group() +def main_cmd() -> None: + pass + + +@main_cmd.command() +@click.argument( + "compose-files", + nargs=-1, + type=click.Path( + exists=True, + dir_okay=False, + ), +) +@click.argument("output-file", type=click.Path()) +def generate_one(compose_files, output_file) -> None: + """ + Generates a merged docker-compose file with env variables inlined. + + Example Usage: python3 generate_docker_quickstart.py generate-one ../docker-compose.yml ../docker-compose.override.yml ../docker-compose-gen.yml + """ + + merged_contents = merge_files(compose_files) + # Write output file - output_dir = os.path.dirname(output_file) - if len(output_dir) and not os.path.exists(output_dir): - os.makedirs(output_dir) - with open(output_file, "w") as new_conf_file: - yaml.dump( - merged_docker_config, - new_conf_file, - default_flow_style=False, - ) + pathlib.Path(output_file).parent.mkdir(parents=True, exist_ok=True) + pathlib.Path(output_file).write_text(merged_contents) print(f"Successfully generated {output_file}.") -def dedup_env_vars(merged_docker_config): - for service in merged_docker_config['services']: - if 'environment' in merged_docker_config['services'][service]: - lst = merged_docker_config['services'][service]['environment'] - if lst is not None: - # use a set to cache duplicates - caches = set() - results = {} - for item in lst: - partitions = item.rpartition('=') - prefix = partitions[0] - suffix = partitions[1] - # check whether prefix already exists - if prefix not in caches and suffix != "": - results[prefix] = item - caches.add(prefix) - if set(lst) != set([v for k,v in results.items()]): - sorted_vars = sorted([k for k in results]) - merged_docker_config['services'][service]['environment'] = [results[var] for var in sorted_vars] +@main_cmd.command() +@click.pass_context +def generate_all(ctx: click.Context) -> None: + """ + Generates all merged docker-compose files with env variables inlined. + """ + + for output_compose_file, inputs in COMPOSE_SPECS.items(): + ctx.invoke(generate_one, compose_files=inputs, output_file=output_compose_file) + + +@main_cmd.command() +def check_all() -> None: + """ + Checks that the generated docker-compose files are up to date. + """ + + for output_compose_file, inputs in COMPOSE_SPECS.items(): + expected = merge_files(inputs) + + # Check that the files match. + current = pathlib.Path(output_compose_file).read_text() + + if expected != current: + print( + f"File {output_compose_file} is out of date. Please run `python3 generate_docker_quickstart.py generate-all`." + ) + sys.exit(1) if __name__ == "__main__": - generate() + main_cmd() diff --git a/docker/quickstart/generate_docker_quickstart.sh b/docker/quickstart/generate_docker_quickstart.sh index 3d8ac7de9a..ee040a72bb 100755 --- a/docker/quickstart/generate_docker_quickstart.sh +++ b/docker/quickstart/generate_docker_quickstart.sh @@ -9,11 +9,4 @@ python3 -m venv venv source venv/bin/activate pip install -r requirements.txt -python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml docker-compose.quickstart.yml -python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml docker-compose-without-neo4j.quickstart.yml -python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml ../docker-compose.m1.yml docker-compose-m1.quickstart.yml -python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml ../docker-compose-without-neo4j.m1.yml docker-compose-without-neo4j-m1.quickstart.yml -python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml docker-compose.monitoring.quickstart.yml -python generate_docker_quickstart.py ../docker-compose.consumers.yml docker-compose.consumers.quickstart.yml -python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml docker-compose.consumers-without-neo4j.quickstart.yml -python generate_docker_quickstart.py ../docker-compose.kafka-setup.yml docker-compose.kafka-setup.quickstart.yml +python generate_docker_quickstart.py generate-all