yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-25 08:58:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(k8s): Update pods with correct probes and remove unnecessary dependencies (#2362)

Browse Source
This commit is contained in:
Dexter Lee 2021-04-09 17:49:12 -07:00 committed by GitHub
parent 6610666496
commit f293add5b1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 256 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/kubernetes/datahub/Chart.yaml
View File

@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
version: 0.1.1
version: 0.1.2
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application.
appVersion: latest #0.3.1

22
contrib/kubernetes/datahub/README.md
View File

@ -2,17 +2,7 @@ datahub
=======
A Helm chart for LinkedIn DataHub
Current chart version is `0.1.1`
## Chart Requirements
| Repository | Name | Version |
|------------|------|---------|
| file://./charts/datahub-frontend | datahub-frontend | 0.2.1 |
| file://./charts/datahub-gms | datahub-gms | 0.2.1 |
| file://./charts/datahub-mae-consumer | datahub-mae-consumer | 0.2.1 |
| file://./charts/datahub-mce-consumer | datahub-mce-consumer | 0.2.1 |
| file://./charts/datahub-ingestion-cron | datahub-ingestion-cron | 0.2.1 |
Current chart version is `0.1.2`
#### Chart Values
@ -34,6 +24,12 @@ Current chart version is `0.1.1`
| elasticsearchSetupJob.enabled | bool | `true` | |
| elasticsearchSetupJob.image.repository | string | `"linkedin/datahub-elasticsearch-setup"` | |
| elasticsearchSetupJob.image.tag | string | `"latest"` | |
| kafkaSetupJob.enabled | bool | `true` | |
| kafkaSetupJob.image.repository | string | `"linkedin/datahub-kafka-setup"` | |
| kafkaSetupJob.image.tag | string | `"latest"` | |
| mysqlSetupJob.enabled | bool | `false` | |
| mysqlSetupJob.image.repository | string | `""` | |
| mysqlSetupJob.image.tag | string | `""` | |
| global.datahub.appVersion | string | `"1.0"` | |
| global.datahub.gms.port | string | `"8080"` | |
| global.elasticsearch.host | string | `"elasticsearch"` | |
@ -44,6 +40,7 @@ Current chart version is `0.1.1`
| global.hostAliases[0].hostnames[3] | string | `"neo4j"` | |
| global.hostAliases[0].ip | string | `"192.168.0.104"` | |
| global.kafka.bootstrap.server | string | `"broker:29092"` | |
| global.kafka.zookeeper.server | string | `"zookeeper:2181"` | |
| global.kafka.schemaregistry.url | string | `"http://schema-registry:8081"` | |
| global.neo4j.host | string | `"neo4j:7474"` | |
| global.neo4j.uri | string | `"bolt://neo4j"` | |
@ -51,7 +48,8 @@ Current chart version is `0.1.1`
| global.neo4j.password.secretRef | string | `"neo4j-secrets"` | |
| global.neo4j.password.secretKey | string | `"neo4j-password"` | |
| global.sql.datasource.driver | string | `"com.mysql.jdbc.Driver"` | |
| global.sql.datasource.host | string | `"mysql"` | |
| global.sql.datasource.host | string | `"mysql:3306"` | |
| global.sql.datasource.hostForMysqlClient | string | `"mysql"` | |
| global.sql.datasource.url | string | `"jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false\u0026useSSL=true"` | |
| global.sql.datasource.username | string | `"datahub"` | |
| global.sql.datasource.password.secretRef | string | `"mysql-secrets"` | |

8
contrib/kubernetes/datahub/charts/datahub-frontend/README.md
View File

@ -10,6 +10,7 @@ Current chart version is `0.2.0`
|-----|------|---------|-------------|
| affinity | object | `{}` | |
| datahub.play.mem.buffer.size | string | `"10MB"` | |
| existingGmsSecret | object | {} | Reference to GMS secret if already exists |
| exporters.jmx.enabled | boolean | false | |
| extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` |
| extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" |
@ -24,11 +25,18 @@ Current chart version is `0.2.0`
| ingress.enabled | bool | `false` | |
| ingress.hosts[0].host | string | `"chart-example.local"` | |
| ingress.hosts[0].paths | list | `[]` | |
| ingress.hosts[0].redirectPaths | list | `[]` | |
| ingress.tls | list | `[]` | |
| livenessProbe.initialDelaySeconds | int | `60` | |
| livenessProbe.periodSeconds | int | `30` | |
| livenessProbe.failureThreshold | int | `4` | |
| nameOverride | string | `""` | |
| nodeSelector | object | `{}` | |
| podAnnotations | object | `{}` | |
| podSecurityContext | object | `{}` | |
| readinessProbe.initialDelaySeconds | int | `60` | |
| readinessProbe.periodSeconds | int | `30` | |
| readinessProbe.failureThreshold | int | `4` | |
| replicaCount | int | `1` | |
| resources | object | `{}` | |
| securityContext | object | `{}` | |

14
contrib/kubernetes/datahub/charts/datahub-frontend/templates/deployment.yaml
View File

@ -46,16 +46,22 @@ spec:
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- name: http
containerPort: 9001
containerPort: 9002
protocol: TCP
livenessProbe:
httpGet:
path: /admin
port: http
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
readinessProbe:
httpGet:
path: /admin
port: http
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
env:
- name: DATAHUB_GMS_HOST
value: {{ printf "%s-%s" .Release.Name "datahub-gms" }}
@ -63,9 +69,15 @@ spec:
value: "{{ .Values.global.datahub.gms.port }}"
- name: DATAHUB_SECRET
valueFrom:
{{- if .Values.existingGmsSecret }}
secretKeyRef:
name: "{{ .Values.existingGmsSecret.name }}"
key: "{{ .Values.existingGmsSecret.key }}"
{{- else }}
secretKeyRef:
name: {{ printf "%s-gms-secret" .Release.Name }}
key: datahub.gms.secret
{{- end }}
- name: DATAHUB_APP_VERSION
value: "{{ .Values.global.datahub.appVersion }}"
- name: DATAHUB_PLAY_MEM_BUFFER_SIZE

6
contrib/kubernetes/datahub/charts/datahub-frontend/templates/ingress.yaml
View File

@ -31,6 +31,12 @@ spec:
- host: {{ .host | quote }}
http:
paths:
{{- range .redirectPaths }}
- path: {{ .path }}
backend:
serviceName: {{ .name }}
servicePort: {{ .port }}
{{- end }}
{{- range .paths }}
- path: {{ . }}
backend:

11
contrib/kubernetes/datahub/charts/datahub-frontend/templates/secrets.yaml Normal file
View File

@ -0,0 +1,11 @@
{{- if not .Values.existingGmsSecret -}}
apiVersion: v1
kind: Secret
metadata:
name: {{ printf "%s-gms-secret" .Release.Name }}
labels:
{{- include "datahub-frontend.labels" . | nindent 4 }}
type: Opaque
data:
datahub.gms.secret: {{ randAlphaNum 10 | b64enc | quote }}
{{- end -}}

13
contrib/kubernetes/datahub/charts/datahub-frontend/values.yaml
View File

@ -38,7 +38,7 @@ securityContext: {}
service:
type: LoadBalancer
port: 9001
port: 9002
ingress:
enabled: false
@ -48,6 +48,7 @@ ingress:
hosts:
- host: chart-example.local
paths: []
redirectPaths: []
tls: []
# - secretName: chart-example-tls
# hosts:
@ -135,6 +136,16 @@ exporters:
# scrapeTimeout: 30s
# scheme: http
livenessProbe:
initialDelaySeconds: 60
periodSeconds: 30
failureThreshold: 4
readinessProbe:
initialDelaySeconds: 60
periodSeconds: 30
failureThreshold: 4
datahub:
play:
mem:

6
contrib/kubernetes/datahub/charts/datahub-gms/README.md
View File

@ -45,10 +45,16 @@ Current chart version is `0.2.0`
| ingress.hosts[0].host | string | `"chart-example.local"` | |
| ingress.hosts[0].paths | list | `[]` | |
| ingress.tls | list | `[]` | |
| livenessProbe.initialDelaySeconds | int | `60` | |
| livenessProbe.periodSeconds | int | `30` | |
| livenessProbe.failureThreshold | int | `8` | |
| nameOverride | string | `""` | |
| nodeSelector | object | `{}` | |
| podAnnotations | object | `{}` | |
| podSecurityContext | object | `{}` | |
| readinessProbe.initialDelaySeconds | int | `60` | |
| readinessProbe.periodSeconds | int | `30` | |
| readinessProbe.failureThreshold | int | `8` | |
| replicaCount | int | `1` | |
| resources | object | `{}` | |
| securityContext | object | `{}` | |

14
contrib/kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml
View File

@ -58,6 +58,20 @@ spec:
- name: http
containerPort: 8080
protocol: TCP
livenessProbe:
httpGet:
path: /health
port: http
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
readinessProbe:
httpGet:
path: /health
port: http
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
env:
- name: EBEAN_DATASOURCE_USERNAME
value: "{{ .Values.global.sql.datasource.username }}"

9
contrib/kubernetes/datahub/charts/datahub-gms/templates/secrets.yaml
View File

@ -1,9 +0,0 @@
apiVersion: v1
kind: Secret
metadata:
name: {{ printf "%s-gms-secret" .Release.Name }}
labels:
{{- include "datahub-gms.labels" . | nindent 4 }}
type: Opaque
data:
datahub.gms.secret: {{ randAlphaNum 10 | b64enc | quote }}

10
contrib/kubernetes/datahub/charts/datahub-gms/values.yaml
View File

@ -133,6 +133,16 @@ exporters:
# scrapeTimeout: 30s
# scheme: http
livenessProbe:
initialDelaySeconds: 60
periodSeconds: 30
failureThreshold: 8
readinessProbe:
initialDelaySeconds: 60
periodSeconds: 30
failureThreshold: 8
#This section is useful if we are installing this chart separately for testing
# helm install datahub-gms datahub-gms/
global:

6
contrib/kubernetes/datahub/charts/datahub-mae-consumer/README.md
View File

@ -37,10 +37,16 @@ Current chart version is `0.2.0`
| ingress.hosts[0].host | string | `"chart-example.local"` | |
| ingress.hosts[0].paths | list | `[]` | |
| ingress.tls | list | `[]` | |
| livenessProbe.initialDelaySeconds | int | `60` | |
| livenessProbe.periodSeconds | int | `30` | |
| livenessProbe.failureThreshold | int | `8` | |
| nameOverride | string | `""` | |
| nodeSelector | object | `{}` | |
| podAnnotations | object | `{}` | |
| podSecurityContext | object | `{}` | |
| readinessProbe.initialDelaySeconds | int | `60` | |
| readinessProbe.periodSeconds | int | `30` | |
| readinessProbe.failureThreshold | int | `8` | |
| replicaCount | int | `1` | |
| resources | object | `{}` | |
| securityContext | object | `{}` | |

14
contrib/kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml
View File

@ -54,6 +54,20 @@ spec:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
livenessProbe:
httpGet:
path: /actuator/health
port: 9091
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
readinessProbe:
httpGet:
path: /actuator/health
port: 9091
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
env:
- name: KAFKA_BOOTSTRAP_SERVER
value: "{{ .Values.global.kafka.bootstrap.server }}"

10
contrib/kubernetes/datahub/charts/datahub-mae-consumer/values.yaml
View File

@ -134,6 +134,16 @@ exporters:
# scrapeTimeout: 30s
# scheme: http
livenessProbe:
initialDelaySeconds: 60
periodSeconds: 30
failureThreshold: 8
readinessProbe:
initialDelaySeconds: 60
periodSeconds: 30
failureThreshold: 8
global:
elasticsearch:
host: "elasticsearch"

6
contrib/kubernetes/datahub/charts/datahub-mce-consumer/README.md
View File

@ -31,10 +31,16 @@ Current chart version is `0.2.0`
| ingress.hosts[0].host | string | `"chart-example.local"` | |
| ingress.hosts[0].paths | list | `[]` | |
| ingress.tls | list | `[]` | |
| livenessProbe.initialDelaySeconds | int | `60` | |
| livenessProbe.periodSeconds | int | `30` | |
| livenessProbe.failureThreshold | int | `4` | |
| nameOverride | string | `""` | |
| nodeSelector | object | `{}` | |
| podAnnotations | object | `{}` | |
| podSecurityContext | object | `{}` | |
| readinessProbe.initialDelaySeconds | int | `60` | |
| readinessProbe.periodSeconds | int | `30` | |
| readinessProbe.failureThreshold | int | `4` | |
| replicaCount | int | `1` | |
| resources | object | `{}` | |
| securityContext | object | `{}` | |

14
contrib/kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml
View File

@ -54,6 +54,20 @@ spec:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
livenessProbe:
httpGet:
path: /actuator/health
port: 9090
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
readinessProbe:
httpGet:
path: /actuator/health
port: 9090
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
env:
- name: KAFKA_BOOTSTRAP_SERVER
value: "{{ .Values.global.kafka.bootstrap.server }}"

10
contrib/kubernetes/datahub/charts/datahub-mce-consumer/values.yaml
View File

@ -134,6 +134,16 @@ exporters:
# scrapeTimeout: 30s
# scheme: http
livenessProbe:
initialDelaySeconds: 60
periodSeconds: 30
failureThreshold: 4
readinessProbe:
initialDelaySeconds: 60
periodSeconds: 30
failureThreshold: 4
global:
kafka:
bootstrap:

43
contrib/kubernetes/datahub/templates/kafka-setup-job.yml Normal file
View File

@ -0,0 +1,43 @@
{{- if .Values.kafkaSetupJob.enabled -}}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ .Release.Name }}-kafka-setup-job
labels:
app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
app.kubernetes.io/instance: {{ .Release.Name | quote }}
app.kubernetes.io/version: {{ .Chart.AppVersion }}
helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
annotations:
# This is what defines this resource as a hook. Without this line, the
# job is considered part of the release.
"helm.sh/hook": pre-install,pre-upgrade
"helm.sh/hook-weight": "-5"
"helm.sh/hook-delete-policy": before-hook-creation
spec:
template:
spec:
{{- with .Values.global.hostAliases }}
hostAliases:
{{- toYaml . | nindent 8 }}
{{- end }}
restartPolicy: Never
securityContext:
runAsUser: 1000
fsGroup: 1000
containers:
- name: kafka-setup-job
image: "{{ .Values.kafkaSetupJob.image.repository }}:{{ .Values.kafkaSetupJob.image.tag }}"
env:
- name: KAFKA_ZOOKEEPER_CONNECT
value: {{ .Values.global.kafka.zookeeper.server | quote }}
- name: KAFKA_BOOTSTRAP_SERVER
value: {{ .Values.global.kafka.bootstrap.server | quote }}
resources:
limits:
cpu: 500m
memory: 512Mi
requests:
cpu: 300m
memory: 256Mi
{{- end -}}

50
contrib/kubernetes/datahub/templates/mysql-setup-job.yml Normal file
View File

@ -0,0 +1,50 @@
{{- if .Values.mysqlSetupJob.enabled -}}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ .Release.Name }}-mysql-setup-job
labels:
app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
app.kubernetes.io/instance: {{ .Release.Name | quote }}
app.kubernetes.io/version: {{ .Chart.AppVersion }}
helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
annotations:
# This is what defines this resource as a hook. Without this line, the
# job is considered part of the release.
"helm.sh/hook": pre-install,pre-upgrade
"helm.sh/hook-weight": "-5"
"helm.sh/hook-delete-policy": before-hook-creation
spec:
template:
spec:
{{- with .Values.global.hostAliases }}
hostAliases:
{{- toYaml . | nindent 8 }}
{{- end }}
restartPolicy: Never
securityContext:
runAsUser: 1000
fsGroup: 1000
containers:
- name: mysql-setup-job
image: "{{ .Values.mysqlSetupJob.image.repository }}:{{ .Values.mysqlSetupJob.image.tag }}"
env:
- name: MYSQL_USERNAME
value: {{ .Values.global.sql.datasource.username | quote }}
- name: MYSQL_PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Values.global.sql.datasource.password.secretRef }}"
key: "{{ .Values.global.sql.datasource.password.secretKey }}"
- name: MYSQL_HOST
value: {{ .Values.global.sql.datasource.hostForMysqlClient | quote }}
- name: MYSQL_PORT
value: {{ .Values.global.sql.datasource.port | quote }}
resources:
limits:
cpu: 500m
memory: 512Mi
requests:
cpu: 300m
memory: 256Mi
{{- end -}}

12
contrib/kubernetes/datahub/values.yaml
View File

@ -38,6 +38,15 @@ elasticsearchSetupJob:
repository: linkedin/datahub-elasticsearch-setup
tag: "latest"
kafkaSetupJob:
enabled: true
image:
repository: linkedin/datahub-kafka-setup
tag: "latest"
mysqlSetupJob:
enabled: false
global:
elasticsearch:
host: "elasticsearch"
@ -46,6 +55,8 @@ global:
kafka:
bootstrap:
server: "broker:29092"
zookeeper:
server: "zookeeper:2181"
schemaregistry:
url: "http://schema-registry:8081"
@ -60,6 +71,7 @@ global:
sql:
datasource:
host: "mysql:3306"
hostForMysqlClient: "mysql"
url: "jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true"
driver: "com.mysql.jdbc.Driver"
username: "datahub"