yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-28 10:28:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(Kubernetes): Add JMX exporter containers to all DataHub components (#2251)

Browse Source 
Co-authored-by: Ricardo Bernardino <ricardo.bernardino@definedcrowd.com>
This commit is contained in:
bernardino 2021-03-31 06:03:27 +01:00 committed by GitHub
parent a5b206a38d
commit 9a947948f0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 355 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
contrib/kubernetes/datahub/charts/datahub-frontend/README.md
View File

@ -10,6 +10,7 @@ Current chart version is `0.2.0`
|-----|------|---------|-------------|
| affinity | object | `{}` | |
| datahub.play.mem.buffer.size | string | `"10MB"` | |
| exporters.jmx.enabled | boolean | false | |
| extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` |
| extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" |
| extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" |

17
contrib/kubernetes/datahub/charts/datahub-frontend/templates/config-jmx-exporter.yaml Normal file
View File

@ -0,0 +1,17 @@
{{- if .Values.exporters.jmx.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "datahub-frontend.fullname" . }}-config-jmx-exporter
labels:
{{- include "datahub-frontend.labels" . | nindent 4 }}
data:
config.yml: |-
hostPort: localhost:{{ .Values.env.JMXPORT }}
lowercaseOutputName: {{ .Values.exporters.jmx.config.lowercaseOutputName }}
lowercaseOutputLabelNames: {{ .Values.exporters.jmx.config.lowercaseOutputLabelNames }}
rules:
{{ .Values.exporters.jmx.config.rules | toYaml | indent 6 }}
ssl: false
startDelaySeconds: {{ .Values.exporters.jmx.config.startDelaySeconds }}
{{- end }}

8
contrib/kubernetes/datahub/charts/datahub-frontend/templates/deployment.yaml
View File

@ -29,6 +29,11 @@ spec:
{{- if .Values.extraVolumes }}
{{ toYaml .Values.extraVolumes | nindent 8 }}
{{- end }}
{{- if .Values.exporters.jmx.enabled }}
- name: config-jmx-exporter
configMap:
name: {{ include "datahub-frontend.fullname" . }}-config-jmx-exporter
{{- end }}
initContainers:
{{- if .Values.extraInitContainers }}
{{- .Values.extraInitContainers | toYaml | nindent 6 }}
@ -74,6 +79,9 @@ spec:
{{- end }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- include "datahub-jmxexporter.container" . }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}

46
contrib/kubernetes/datahub/charts/datahub-frontend/values.yaml
View File

@ -89,6 +89,52 @@ tolerations: []
affinity: {}
env:
JMXPORT: 1099
exporters:
jmx:
enabled: false
# image:
# repository: bitnami/jmx-exporter
# tag: 0.15.0
# pullPolicy: IfNotPresent
# config:
# lowercaseOutputName: true
# lowercaseOutputLabelNames: true
# rules:
# - pattern: ".*"
# startDelaySeconds: 30
# env: {}
# resources: {}
# path: /metrics
# ports:
# jmxxp:
# containerPort: 5556
# protocol: TCP
# livenessProbe:
# httpGet:
# path: /metrics
# port: jmxxp
# initialDelaySeconds: 30
# periodSeconds: 15
# timeoutSeconds: 60
# failureThreshold: 8
# successThreshold: 1
# readinessProbe:
# httpGet:
# path: /metrics
# port: jmxxp
# initialDelaySeconds: 30
# periodSeconds: 15
# timeoutSeconds: 60
# failureThreshold: 8
# successThreshold: 1
# serviceMonitor:
# interval: 30s
# scrapeTimeout: 30s
# scheme: http
datahub:
play:
mem:

1
contrib/kubernetes/datahub/charts/datahub-gms/README.md
View File

@ -9,6 +9,7 @@ Current chart version is `0.2.0`
| Key | Type | Default | Description |
|-----|------|---------|-------------|
| affinity | object | `{}` | |
| exporters.jmx.enabled | boolean | false | |
| extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` |
| extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" |
| extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" |

17
contrib/kubernetes/datahub/charts/datahub-gms/templates/config-jmx-exporter.yaml Normal file
View File

@ -0,0 +1,17 @@
{{- if .Values.exporters.jmx.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "datahub-gms.fullname" . }}-config-jmx-exporter
labels:
{{- include "datahub-gms.labels" . | nindent 4 }}
data:
config.yml: |-
hostPort: localhost:{{ .Values.env.JMXPORT }}
lowercaseOutputName: {{ .Values.exporters.jmx.config.lowercaseOutputName }}
lowercaseOutputLabelNames: {{ .Values.exporters.jmx.config.lowercaseOutputLabelNames }}
rules:
{{ .Values.exporters.jmx.config.rules | toYaml | indent 6 }}
ssl: false
startDelaySeconds: {{ .Values.exporters.jmx.config.startDelaySeconds }}
{{- end }}

8
contrib/kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml
View File

@ -36,6 +36,11 @@ spec:
defaultMode: 256
secretName: {{ .Values.global.credentialsAndCertsSecrets.name }}
{{- end }}
{{- if .Values.exporters.jmx.enabled }}
- name: config-jmx-exporter
configMap:
name: {{ include "datahub-gms.fullname" . }}-config-jmx-exporter
{{- end }}
{{- if .Values.extraVolumes }}
{{ toYaml .Values.extraVolumes | nindent 8 }}
{{- end }}
@ -114,6 +119,9 @@ spec:
{{- end }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- include "datahub-jmxexporter.container" . }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}

45
contrib/kubernetes/datahub/charts/datahub-gms/values.yaml
View File

@ -88,6 +88,51 @@ tolerations: []
affinity: {}
env:
JMXPORT: 1099
exporters:
jmx:
enabled: false
# image:
# repository: bitnami/jmx-exporter
# tag: 0.15.0
# pullPolicy: IfNotPresent
# config:
# lowercaseOutputName: true
# rules:
# - pattern: ".*"
# startDelaySeconds: 30
# env: {}
# resources: {}
# path: /metrics
# ports:
# jmxxp:
# containerPort: 5556
# protocol: TCP
# livenessProbe:
# httpGet:
# path: /metrics
# port: jmxxp
# initialDelaySeconds: 30
# periodSeconds: 15
# timeoutSeconds: 60
# failureThreshold: 8
# successThreshold: 1
# readinessProbe:
# httpGet:
# path: /metrics
# port: jmxxp
# initialDelaySeconds: 30
# periodSeconds: 15
# timeoutSeconds: 60
# failureThreshold: 8
# successThreshold: 1
# serviceMonitor:
# interval: 30s
# scrapeTimeout: 30s
# scheme: http
#This section is useful if we are installing this chart separately for testing
# helm install datahub-gms datahub-gms/
global:

21
contrib/kubernetes/datahub/charts/datahub-jmxexporter/Chart.yaml Normal file
View File

@ -0,0 +1,21 @@
apiVersion: v2
name: datahub-jmxexporter
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: library
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
version: 0.2.1
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application.
appVersion: 0.3.1

32
contrib/kubernetes/datahub/charts/datahub-jmxexporter/templates/_container.tpl Normal file
View File

@ -0,0 +1,32 @@
{{- define "datahub-jmxexporter.container" -}}
{{- if .Values.exporters.jmx.enabled }}
- name: jmx-exporter
image: "{{ .Values.exporters.jmx.image.repository }}:{{ .Values.exporters.jmx.image.tag }}"
imagePullPolicy: {{ .Values.exporters.jmx.image.pullPolicy }}
args: ["{{ .Values.exporters.jmx.ports.jmxxp.containerPort}}", "/opt/jmx_exporter/config.yml"]
ports:
{{- range $key, $port := .Values.exporters.jmx.ports }}
- name: {{ $key }}
{{ toYaml $port | indent 14 }}
{{- end }}
livenessProbe:
{{ toYaml .Values.exporters.jmx.livenessProbe | indent 12 }}
readinessProbe:
{{ toYaml .Values.exporters.jmx.readinessProbe | indent 12 }}
env:
- name: SERVICE_PORT
value: {{ .Values.exporters.jmx.ports.jmxxp.containerPort | quote }}
{{- with .Values.exporters.jmx.env }}
{{- range $key, $value := . }}
- name: {{ $key | upper | replace "." "_" }}
value: {{ $value | quote }}
{{- end }}
{{- end }}
resources:
{{ toYaml .Values.exporters.jmx.resources | indent 12 }}
volumeMounts:
- name: config-jmx-exporter
mountPath: /opt/jmx_exporter/config.yml
subPath: config.yml
{{- end }}
{{- end }}

1
contrib/kubernetes/datahub/charts/datahub-mae-consumer/README.md
View File

@ -9,6 +9,7 @@ Current chart version is `0.2.0`
| Key | Type | Default | Description |
|-----|------|---------|-------------|
| affinity | object | `{}` | |
| exporters.jmx.enabled | boolean | false | |
| extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` |
| extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" |
| extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" |

17
contrib/kubernetes/datahub/charts/datahub-mae-consumer/templates/config-jmx-exporter.yaml Normal file
View File

@ -0,0 +1,17 @@
{{- if .Values.exporters.jmx.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "datahub-mae-consumer.fullname" . }}-config-jmx-exporter
labels:
{{- include "datahub-mae-consumer.labels" . | nindent 4 }}
data:
config.yml: |-
hostPort: localhost:{{ .Values.env.JMXPORT }}
lowercaseOutputName: {{ .Values.exporters.jmx.config.lowercaseOutputName }}
lowercaseOutputLabelNames: {{ .Values.exporters.jmx.config.lowercaseOutputLabelNames }}
rules:
{{ .Values.exporters.jmx.config.rules | toYaml | indent 6 }}
ssl: false
startDelaySeconds: {{ .Values.exporters.jmx.config.startDelaySeconds }}
{{- end }}

8
contrib/kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml
View File

@ -36,6 +36,11 @@ spec:
defaultMode: 256
secretName: {{ .Values.global.credentialsAndCertsSecrets.name }}
{{- end }}
{{- if .Values.exporters.jmx.enabled }}
- name: config-jmx-exporter
configMap:
name: {{ include "datahub-mae-consumer.fullname" . }}-config-jmx-exporter
{{- end }}
{{- if .Values.extraVolumes }}
{{ toYaml .Values.extraVolumes | nindent 8 }}
{{- end }}
@ -97,6 +102,9 @@ spec:
{{- end }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- include "datahub-jmxexporter.container" . }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}

45
contrib/kubernetes/datahub/charts/datahub-mae-consumer/values.yaml
View File

@ -89,6 +89,51 @@ tolerations: []
affinity: {}
env:
JMXPORT: 1099
exporters:
jmx:
enabled: false
# image:
# repository: bitnami/jmx-exporter
# tag: 0.15.0
# pullPolicy: IfNotPresent
# config:
# lowercaseOutputName: true
# rules:
# - pattern: ".*"
# startDelaySeconds: 30
# env: {}
# resources: {}
# path: /metrics
# ports:
# jmxxp:
# containerPort: 5556
# protocol: TCP
# livenessProbe:
# httpGet:
# path: /metrics
# port: jmxxp
# initialDelaySeconds: 30
# periodSeconds: 15
# timeoutSeconds: 60
# failureThreshold: 8
# successThreshold: 1
# readinessProbe:
# httpGet:
# path: /metrics
# port: jmxxp
# initialDelaySeconds: 30
# periodSeconds: 15
# timeoutSeconds: 60
# failureThreshold: 8
# successThreshold: 1
# serviceMonitor:
# interval: 30s
# scrapeTimeout: 30s
# scheme: http
global:
elasticsearch:
host: "elasticsearch"

1
contrib/kubernetes/datahub/charts/datahub-mce-consumer/README.md
View File

@ -9,6 +9,7 @@ Current chart version is `0.2.0`
| Key | Type | Default | Description |
|-----|------|---------|-------------|
| affinity | object | `{}` | |
| exporters.jmx.enabled | boolean | false | |
| extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` |
| extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" |
| extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" |

17
contrib/kubernetes/datahub/charts/datahub-mce-consumer/templates/config-jmx-exporter.yaml Normal file
View File

@ -0,0 +1,17 @@
{{- if .Values.exporters.jmx.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "datahub-mce-consumer.fullname" . }}-config-jmx-exporter
labels:
{{- include "datahub-mce-consumer.labels" . | nindent 4 }}
data:
config.yml: |-
hostPort: localhost:{{ .Values.env.JMXPORT }}
lowercaseOutputName: {{ .Values.exporters.jmx.config.lowercaseOutputName }}
lowercaseOutputLabelNames: {{ .Values.exporters.jmx.config.lowercaseOutputLabelNames }}
rules:
{{ .Values.exporters.jmx.config.rules | toYaml | indent 6 }}
ssl: false
startDelaySeconds: {{ .Values.exporters.jmx.config.startDelaySeconds }}
{{- end }}

8
contrib/kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml
View File

@ -36,6 +36,11 @@ spec:
defaultMode: 256
secretName: {{ .Values.global.credentialsAndCertsSecrets.name }}
{{- end }}
{{- if .Values.exporters.jmx.enabled }}
- name: config-jmx-exporter
configMap:
name: {{ include "datahub-mce-consumer.fullname" . }}-config-jmx-exporter
{{- end }}
{{- if .Values.extraVolumes }}
{{ toYaml .Values.extraVolumes | nindent 8 }}
{{- end }}
@ -86,6 +91,9 @@ spec:
{{- end }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- include "datahub-jmxexporter.container" . }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}

45
contrib/kubernetes/datahub/charts/datahub-mce-consumer/values.yaml
View File

@ -89,6 +89,51 @@ tolerations: []
affinity: {}
env:
JMXPORT: 1099
exporters:
jmx:
enabled: false
# image:
# repository: bitnami/jmx-exporter
# tag: 0.15.0
# pullPolicy: IfNotPresent
# config:
# lowercaseOutputName: true
# rules:
# - pattern: ".*"
# startDelaySeconds: 30
# env: {}
# resources: {}
# path: /metrics
# ports:
# jmxxp:
# containerPort: 5556
# protocol: TCP
# livenessProbe:
# httpGet:
# path: /metrics
# port: jmxxp
# initialDelaySeconds: 30
# periodSeconds: 15
# timeoutSeconds: 60
# failureThreshold: 8
# successThreshold: 1
# readinessProbe:
# httpGet:
# path: /metrics
# port: jmxxp
# initialDelaySeconds: 30
# periodSeconds: 15
# timeoutSeconds: 60
# failureThreshold: 8
# successThreshold: 1
# serviceMonitor:
# interval: 30s
# scrapeTimeout: 30s
# scheme: http
global:
kafka:
bootstrap:

5
docker/datahub-gms/Dockerfile
View File

@ -5,6 +5,8 @@ FROM openjdk:8-jre-alpine as base
ENV DOCKERIZE_VERSION v0.6.1
RUN apk --no-cache add curl tar \
&& curl https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.4.20.v20190813/jetty-runner-9.4.20.v20190813.jar --output jetty-runner.jar \
&& curl https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-jmx/9.4.20.v20190813/jetty-jmx-9.4.20.v20190813.jar --output jetty-jmx.jar \
&& curl https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util/9.4.20.v20190813/jetty-util-9.4.20.v20190813.jar --output jetty-util.jar \
&& curl -L https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz | tar -C /usr/local/bin -xzv
FROM openjdk:8 as prod-build
@ -26,6 +28,9 @@ FROM ${APP_ENV}-install as final
RUN addgroup -S datahub && adduser -S datahub -G datahub
USER datahub
ENV JMX_OPTS=""
ENV JAVA_OPTS=""
EXPOSE 8080
HEALTHCHECK --start-period=2m --retries=4 CMD curl --fail http://localhost:8080/health || exit 1

5
docker/datahub-gms/start.sh
View File

@ -23,4 +23,7 @@ dockerize \
-wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT \
-wait $NEO4J_HOST \
-timeout 240s \
java -jar /jetty-runner.jar /datahub/datahub-gms/bin/war.war
java $JAVA_OPTS $JMX_OPTS \
--jar jetty-util.jar \
--jar jetty-jmx.jar \
-jar /jetty-runner.jar /datahub/datahub-gms/bin/war.war

3
docker/datahub-mae-consumer/Dockerfile
View File

@ -32,6 +32,9 @@ FROM ${APP_ENV}-install as final
RUN addgroup -S datahub && adduser -S datahub -G datahub
USER datahub
ENV JMX_OPTS=""
ENV JAVA_OPTS=""
EXPOSE 9090
HEALTHCHECK --start-period=2m --retries=4 CMD curl --fail http://localhost:9091/actuator/health || exit 1

2
docker/datahub-mae-consumer/start.sh
View File

@ -22,4 +22,4 @@ dockerize \
-wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT \
-wait $NEO4J_HOST \
-timeout 240s \
java -jar /datahub/datahub-mae-consumer/bin/mae-consumer-job.jar
java $JAVA_OPTS $JMX_OPTS -jar /datahub/datahub-mae-consumer/bin/mae-consumer-job.jar

3
docker/datahub-mce-consumer/Dockerfile
View File

@ -32,6 +32,9 @@ FROM ${APP_ENV}-install as final
RUN addgroup -S datahub && adduser -S datahub -G datahub
USER datahub
ENV JMX_OPTS=""
ENV JAVA_OPTS=""
EXPOSE 9090
HEALTHCHECK --start-period=2m --retries=4 CMD curl --fail http://localhost:9090/actuator/health || exit 1

3
docker/datahub-mce-consumer/start.sh
View File

@ -1,7 +1,6 @@
#!/bin/sh
# -wait tcp://GMS_HOST:$GMS_PORT \
dockerize \
-wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \
-timeout 240s \
java -jar /datahub/datahub-mce-consumer/bin/mce-consumer-job.jar
java $JAVA_OPTS $JMX_OPTS -jar /datahub/datahub-mce-consumer/bin/mce-consumer-job.jar