mirror of
https://github.com/datahub-project/datahub.git
synced 2025-11-02 19:58:59 +00:00
Bump up kafkaAvroSerde to support SSL for Schema Registry (#1898)
* Bump up kafkaAvroSerde to support security config for Confluent Schema Registry * Support certs in secrets * Extra Spring config (e.g. security) * Optional values * Clarify log warnings * Update faq.md Co-authored-by: Lars Nielsen <Lars.Nielsen@kindredgroup.com> Co-authored-by: Mars Lan <mars.th.lan@gmail.com>
This commit is contained in:
Lars Nielsen
GitHub
parent
9bcf273661
commit
b26d6fe880
@ -51,7 +51,7 @@ project.ext.externalDependency = [
|
||||
'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
|
||||
'junit': 'junit:junit:4.12',
|
||||
// avro-serde includes dependencies for `kafka-avro-serializer` `kafka-schema-registry-client` and `avro`
|
||||
'kafkaAvroSerde': 'io.confluent:kafka-streams-avro-serde:5.2.2',
|
||||
'kafkaAvroSerde': 'io.confluent:kafka-streams-avro-serde:5.5.1',
|
||||
'kafkaClients': 'org.apache.kafka:kafka-clients:2.3.0',
|
||||
'logbackClassic': 'ch.qos.logback:logback-classic:1.2.3',
|
||||
'lombok': 'org.projectlombok:lombok:1.18.12',
|
||||
|
||||
@ -51,3 +51,10 @@ Current chart version is `0.1.0`
|
||||
| global.sql.datasource.username | string | `"datahub"` | |
|
||||
| global.sql.datasource.password.secretRef | string | `"mysql-secrets"` | |
|
||||
| global.sql.datasource.password.secretKey | string | `"mysql-password"` | |
|
||||
|
||||
#### Optional Chart Values
|
||||
|
||||
| global.credentialsAndCertsSecretPath | string | `"/mnt/certs"` | |
|
||||
| global.credentialsAndCertsSecrets.name | string | `""` | |
|
||||
| global.credentialsAndCertsSecrets.secureEnv | string | `""` | |
|
||||
| global.springKafkaConfigurationOverrides | string | `""` | |
|
||||
|
||||
@ -26,6 +26,12 @@ spec:
|
||||
securityContext:
|
||||
{{- toYaml .Values.podSecurityContext | nindent 8 }}
|
||||
volumes:
|
||||
{{- if .Values.global.credentialsAndCertsSecrets }}
|
||||
- name: datahub-certs-dir
|
||||
secret:
|
||||
defaultMode: 256
|
||||
secretName: {{ .Values.global.credentialsAndCertsSecrets.name }}
|
||||
{{- end }}
|
||||
{{- if .Values.extraVolumes }}
|
||||
{{ toYaml .Values.extraVolumes | indent 8 }}
|
||||
{{- end }}
|
||||
@ -72,10 +78,29 @@ spec:
|
||||
secretKeyRef:
|
||||
name: "{{ .Values.global.neo4j.password.secretRef }}"
|
||||
key: "{{ .Values.global.neo4j.password.secretKey }}"
|
||||
{{- if .Values.global.springKafkaConfigurationOverrides }}
|
||||
{{- range $configName, $configValue := .Values.global.springKafkaConfigurationOverrides }}
|
||||
- name: SPRING_KAFKA_PROPERTIES_{{ $configName | replace "." "_" | upper }}
|
||||
value: {{ $configValue }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.credentialsAndCertsSecrets }}
|
||||
{{- range $envVarName, $envVarValue := .Values.global.credentialsAndCertsSecrets.secureEnv }}
|
||||
- name: SPRING_KAFKA_PROPERTIES_{{ $envVarName | replace "." "_" | upper }}
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ $.Values.global.credentialsAndCertsSecrets.name }}
|
||||
key: {{ $envVarValue }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.extraEnvs }}
|
||||
{{ toYaml .Values.extraEnvs | indent 10 }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- if .Values.global.credentialsAndCertsSecrets }}
|
||||
- name: datahub-certs-dir
|
||||
mountPath: {{ .Values.global.credentialsAndCertsSecretPath | default "/mnt/certs" }}
|
||||
{{- end }}
|
||||
{{- if .Values.extraVolumeMounts }}
|
||||
{{ toYaml .Values.extraVolumeMounts | indent 10 }}
|
||||
{{- end }}
|
||||
|
||||
@ -26,6 +26,12 @@ spec:
|
||||
securityContext:
|
||||
{{- toYaml .Values.podSecurityContext | nindent 8 }}
|
||||
volumes:
|
||||
{{- if .Values.global.credentialsAndCertsSecrets }}
|
||||
- name: datahub-certs-dir
|
||||
secret:
|
||||
defaultMode: 256
|
||||
secretName: {{ .Values.global.credentialsAndCertsSecrets.name }}
|
||||
{{- end }}
|
||||
{{- if .Values.extraVolumes }}
|
||||
{{ toYaml .Values.extraVolumes | indent 8 }}
|
||||
{{- end }}
|
||||
@ -55,10 +61,29 @@ spec:
|
||||
secretKeyRef:
|
||||
name: "{{ .Values.global.neo4j.password.secretRef }}"
|
||||
key: "{{ .Values.global.neo4j.password.secretKey }}"
|
||||
{{- if .Values.global.springKafkaConfigurationOverrides }}
|
||||
{{- range $configName, $configValue := .Values.global.springKafkaConfigurationOverrides }}
|
||||
- name: SPRING_KAFKA_PROPERTIES_{{ $configName | replace "." "_" | upper }}
|
||||
value: {{ $configValue }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.credentialsAndCertsSecrets }}
|
||||
{{- range $envVarName, $envVarValue := .Values.global.credentialsAndCertsSecrets.secureEnv }}
|
||||
- name: SPRING_KAFKA_PROPERTIES_{{ $envVarName | replace "." "_" | upper }}
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ $.Values.global.credentialsAndCertsSecrets.name }}
|
||||
key: {{ $envVarValue }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.extraEnvs }}
|
||||
{{ toYaml .Values.extraEnvs | indent 10 }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- if .Values.global.credentialsAndCertsSecrets }}
|
||||
- name: datahub-certs-dir
|
||||
mountPath: {{ .Values.global.credentialsAndCertsSecretPath | default "/mnt/certs" }}
|
||||
{{- end }}
|
||||
{{- if .Values.extraVolumeMounts }}
|
||||
{{ toYaml .Values.extraVolumeMounts | indent 10 }}
|
||||
{{- end }}
|
||||
|
||||
@ -26,6 +26,12 @@ spec:
|
||||
securityContext:
|
||||
{{- toYaml .Values.podSecurityContext | nindent 8 }}
|
||||
volumes:
|
||||
{{- if .Values.global.credentialsAndCertsSecrets }}
|
||||
- name: datahub-certs-dir
|
||||
secret:
|
||||
defaultMode: 256
|
||||
secretName: {{ .Values.global.credentialsAndCertsSecrets.name }}
|
||||
{{- end }}
|
||||
{{- if .Values.extraVolumes }}
|
||||
{{ toYaml .Values.extraVolumes | indent 8 }}
|
||||
{{- end }}
|
||||
@ -44,10 +50,29 @@ spec:
|
||||
value: {{ printf "%s-%s" .Release.Name "datahub-gms" }}
|
||||
- name: GMS_PORT
|
||||
value: "{{ .Values.global.datahub.gms.port }}"
|
||||
{{- if .Values.global.springKafkaConfigurationOverrides }}
|
||||
{{- range $configName, $configValue := .Values.global.springKafkaConfigurationOverrides }}
|
||||
- name: SPRING_KAFKA_PROPERTIES_{{ $configName | replace "." "_" | upper }}
|
||||
value: {{ $configValue }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.credentialsAndCertsSecrets }}
|
||||
{{- range $envVarName, $envVarValue := .Values.global.credentialsAndCertsSecrets.secureEnv }}
|
||||
- name: SPRING_KAFKA_PROPERTIES_{{ $envVarName | replace "." "_" | upper }}
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ $.Values.global.credentialsAndCertsSecrets.name }}
|
||||
key: {{ $envVarValue }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.extraEnvs }}
|
||||
{{ toYaml .Values.extraEnvs | indent 10 }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- if .Values.global.credentialsAndCertsSecrets }}
|
||||
- name: datahub-certs-dir
|
||||
mountPath: {{ .Values.global.credentialsAndCertsSecretPath | default "/mnt/certs" }}
|
||||
{{- end }}
|
||||
{{- if .Values.extraVolumeMounts }}
|
||||
{{ toYaml .Values.extraVolumeMounts | indent 10 }}
|
||||
{{- end }}
|
||||
|
||||
@ -66,4 +66,24 @@ global:
|
||||
- "broker"
|
||||
- "mysql"
|
||||
- "elasticsearch"
|
||||
- "neo4j"
|
||||
- "neo4j"
|
||||
|
||||
# credentialsAndCertsSecretPath: /mnt/datahub/certs
|
||||
# credentialsAndCertsSecrets:
|
||||
# name: datahub-certs
|
||||
# secureEnv:
|
||||
# ssl.key.password: datahub.linkedin.com.KeyPass
|
||||
# ssl.keystore.password: datahub.linkedin.com.KeyStorePass
|
||||
# ssl.truststore.password: datahub.linkedin.com.TrustStorePass
|
||||
# kafkastore.ssl.truststore.password: datahub.linkedin.com.TrustStorePass
|
||||
|
||||
# springKafkaConfigurationOverrides:
|
||||
# ssl.keystore.location: /mnt/datahub/certs/datahub.linkedin.com.keystore.jks
|
||||
# ssl.truststore.location: /mnt/datahub/certs/datahub.linkedin.com.truststore.jks
|
||||
# kafkastore.ssl.truststore.location: /mnt/datahub/certs/datahub.linkedin.com.truststore.jks
|
||||
# security.protocol: SSL
|
||||
# kafkastore.security.protocol: SSL
|
||||
# ssl.keystore.type: JKS
|
||||
# ssl.truststore.type: JKS
|
||||
# ssl.protocol: TLS
|
||||
# ssl.endpoint.identification.algorithm:
|
||||
|
||||
@ -116,3 +116,7 @@ You can call the [rest.li](https://github.com/linkedin/rest.li) API to ingest me
|
||||
## Does Kafka support SSL? If so, how?
|
||||
|
||||
Yes. We are using the Spring Boot framework to start our apps, including setting up Kafka. You can [use environment variables to set system properties](https://docs.spring.io/spring-boot/docs/current/reference/html/spring-boot-features.html#boot-features-external-config-relaxed-binding-from-environment-variables), including [Kafka properties](https://docs.spring.io/spring-boot/docs/current/reference/html/appendix-application-properties.html#integration-properties). From there you can set your SSL configuration for Kafka.
|
||||
|
||||
If Schema Registry is configured to use security (SSL), then you also need to set the following config: https://docs.confluent.io/current/kafka/encryption.html#encryption-ssl-schema-registry.
|
||||
|
||||
> **Note** In the logs you might see something like `The configuration 'kafkastore.ssl.truststore.password' was supplied but isn't a known config.` The configuration is not a configuration required for the producer. These WARN message can be safely ignored. Each of Datahub services are passed a full set of configuration but may not require all the configurations that are passed to them. These warn messages indicate that the service was passed a configuration that is not relevant to it and can be safely ignored.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user