Bump up kafkaAvroSerde to support SSL for Schema Registry (#1898)

* Bump up kafkaAvroSerde to support security config for Confluent Schema Registry

* Support certs in secrets

* Extra Spring config (e.g. security)

* Optional values

* Clarify log warnings

* Update faq.md

Co-authored-by: Lars Nielsen <Lars.Nielsen@kindredgroup.com>
Co-authored-by: Mars Lan <mars.th.lan@gmail.com>

build.gradle

@@ -51,7 +51,7 @@ project.ext.externalDependency = [
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'junit': 'junit:junit:4.12',
     // avro-serde includes dependencies for `kafka-avro-serializer` `kafka-schema-registry-client` and `avro`
-    'kafkaAvroSerde': 'io.confluent:kafka-streams-avro-serde:5.2.2',
+    'kafkaAvroSerde': 'io.confluent:kafka-streams-avro-serde:5.5.1',
     'kafkaClients': 'org.apache.kafka:kafka-clients:2.3.0',
     'logbackClassic': 'ch.qos.logback:logback-classic:1.2.3',
     'lombok': 'org.projectlombok:lombok:1.18.12',

contrib/kubernetes/datahub/README.md

@@ -51,3 +51,10 @@ Current chart version is `0.1.0`
 | global.sql.datasource.username | string | `"datahub"` |  |
 | global.sql.datasource.password.secretRef | string | `"mysql-secrets"` |  |
 | global.sql.datasource.password.secretKey | string | `"mysql-password"` |  |
+
+#### Optional Chart Values
+
+| global.credentialsAndCertsSecretPath | string | `"/mnt/certs"` |  |
+| global.credentialsAndCertsSecrets.name | string | `""` |  |
+| global.credentialsAndCertsSecrets.secureEnv | string | `""` |  |
+| global.springKafkaConfigurationOverrides | string | `""` |  |

contrib/kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml

@@ -26,6 +26,12 @@ spec:
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       volumes:
+        {{- if .Values.global.credentialsAndCertsSecrets }}
+        - name: datahub-certs-dir
+          secret:
+            defaultMode: 256
+            secretName: {{ .Values.global.credentialsAndCertsSecrets.name }}
+        {{- end }}
       {{- if .Values.extraVolumes }}
         {{ toYaml .Values.extraVolumes | indent 8 }}
       {{- end }}
@@ -72,10 +78,29 @@ spec:
                 secretKeyRef:
                   name: "{{ .Values.global.neo4j.password.secretRef }}"
                   key: "{{ .Values.global.neo4j.password.secretKey }}"
+            {{- if .Values.global.springKafkaConfigurationOverrides }}
+            {{- range $configName, $configValue := .Values.global.springKafkaConfigurationOverrides }}
+            - name: SPRING_KAFKA_PROPERTIES_{{ $configName | replace "." "_" | upper }}
+              value: {{ $configValue }}
+            {{- end }}
+            {{- end }}
+            {{- if .Values.global.credentialsAndCertsSecrets }}
+            {{- range $envVarName, $envVarValue := .Values.global.credentialsAndCertsSecrets.secureEnv }}
+            - name: SPRING_KAFKA_PROPERTIES_{{ $envVarName | replace "." "_" | upper }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $.Values.global.credentialsAndCertsSecrets.name }}
+                  key: {{ $envVarValue }}
+            {{- end }}
+            {{- end }}
           {{- if .Values.extraEnvs }}
             {{ toYaml .Values.extraEnvs | indent 10 }}
           {{- end }}
           volumeMounts:
+          {{- if .Values.global.credentialsAndCertsSecrets }}
+            - name: datahub-certs-dir
+              mountPath: {{ .Values.global.credentialsAndCertsSecretPath | default "/mnt/certs" }}
+          {{- end }}
           {{- if .Values.extraVolumeMounts }}
             {{ toYaml .Values.extraVolumeMounts | indent 10 }}
           {{- end }}

contrib/kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml

@@ -26,6 +26,12 @@ spec:
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       volumes:
+        {{- if .Values.global.credentialsAndCertsSecrets }}
+        - name: datahub-certs-dir
+          secret:
+            defaultMode: 256
+            secretName: {{ .Values.global.credentialsAndCertsSecrets.name }}
+        {{- end }}
       {{- if .Values.extraVolumes }}
         {{ toYaml .Values.extraVolumes | indent 8 }}
       {{- end }}
@@ -55,10 +61,29 @@ spec:
                 secretKeyRef:
                   name: "{{ .Values.global.neo4j.password.secretRef }}"
                   key: "{{ .Values.global.neo4j.password.secretKey }}"
+            {{- if .Values.global.springKafkaConfigurationOverrides }}
+            {{- range $configName, $configValue := .Values.global.springKafkaConfigurationOverrides }}
+            - name: SPRING_KAFKA_PROPERTIES_{{ $configName | replace "." "_" | upper }}
+              value: {{ $configValue }}
+            {{- end }}
+            {{- end }}
+            {{- if .Values.global.credentialsAndCertsSecrets }}
+            {{- range $envVarName, $envVarValue := .Values.global.credentialsAndCertsSecrets.secureEnv }}
+            - name: SPRING_KAFKA_PROPERTIES_{{ $envVarName | replace "." "_" | upper }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $.Values.global.credentialsAndCertsSecrets.name }}
+                  key: {{ $envVarValue }}
+            {{- end }}
+            {{- end }}
           {{- if .Values.extraEnvs }}
             {{ toYaml .Values.extraEnvs | indent 10 }}
           {{- end }}
           volumeMounts:
+          {{- if .Values.global.credentialsAndCertsSecrets }}
+            - name: datahub-certs-dir
+              mountPath: {{ .Values.global.credentialsAndCertsSecretPath | default "/mnt/certs" }}
+          {{- end }}
           {{- if .Values.extraVolumeMounts }}
             {{ toYaml .Values.extraVolumeMounts | indent 10 }}
           {{- end }}

contrib/kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml

@@ -26,6 +26,12 @@ spec:
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       volumes:
+        {{- if .Values.global.credentialsAndCertsSecrets }}
+        - name: datahub-certs-dir
+          secret:
+            defaultMode: 256
+            secretName: {{ .Values.global.credentialsAndCertsSecrets.name }}
+        {{- end }}
       {{- if .Values.extraVolumes }}
         {{ toYaml .Values.extraVolumes | indent 8 }}
       {{- end }}
@@ -44,10 +50,29 @@ spec:
               value: {{ printf "%s-%s" .Release.Name "datahub-gms" }}
             - name: GMS_PORT
               value: "{{ .Values.global.datahub.gms.port }}"
+            {{- if .Values.global.springKafkaConfigurationOverrides }}
+            {{- range $configName, $configValue := .Values.global.springKafkaConfigurationOverrides }}
+            - name: SPRING_KAFKA_PROPERTIES_{{ $configName | replace "." "_" | upper }}
+              value: {{ $configValue }}
+            {{- end }}
+            {{- end }}
+            {{- if .Values.global.credentialsAndCertsSecrets }}
+            {{- range $envVarName, $envVarValue := .Values.global.credentialsAndCertsSecrets.secureEnv }}
+            - name: SPRING_KAFKA_PROPERTIES_{{ $envVarName | replace "." "_" | upper }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $.Values.global.credentialsAndCertsSecrets.name }}
+                  key: {{ $envVarValue }}
+            {{- end }}
+            {{- end }}
           {{- if .Values.extraEnvs }}
             {{ toYaml .Values.extraEnvs | indent 10 }}
           {{- end }}
           volumeMounts:
+          {{- if .Values.global.credentialsAndCertsSecrets }}
+            - name: datahub-certs-dir
+              mountPath: {{ .Values.global.credentialsAndCertsSecretPath | default "/mnt/certs" }}
+          {{- end }}
           {{- if .Values.extraVolumeMounts }}
             {{ toYaml .Values.extraVolumeMounts | indent 10 }}
           {{- end }}

contrib/kubernetes/datahub/values.yaml

@@ -66,4 +66,24 @@ global:
         - "broker"
         - "mysql"
         - "elasticsearch"
-        - "neo4j"
+        - "neo4j"
+
+  # credentialsAndCertsSecretPath: /mnt/datahub/certs
+  # credentialsAndCertsSecrets:
+  #   name: datahub-certs
+  #   secureEnv:
+  #     ssl.key.password: datahub.linkedin.com.KeyPass
+  #     ssl.keystore.password: datahub.linkedin.com.KeyStorePass
+  #     ssl.truststore.password: datahub.linkedin.com.TrustStorePass
+  #     kafkastore.ssl.truststore.password: datahub.linkedin.com.TrustStorePass
+
+  # springKafkaConfigurationOverrides:
+  #   ssl.keystore.location: /mnt/datahub/certs/datahub.linkedin.com.keystore.jks
+  #   ssl.truststore.location: /mnt/datahub/certs/datahub.linkedin.com.truststore.jks
+  #   kafkastore.ssl.truststore.location: /mnt/datahub/certs/datahub.linkedin.com.truststore.jks
+  #   security.protocol: SSL
+  #   kafkastore.security.protocol: SSL
+  #   ssl.keystore.type: JKS
+  #   ssl.truststore.type: JKS
+  #   ssl.protocol: TLS
+  #   ssl.endpoint.identification.algorithm: 

docs/faq.md

@@ -116,3 +116,7 @@ You can call the [rest.li](https://github.com/linkedin/rest.li) API to ingest me
 ## Does Kafka support SSL? If so, how?
 
 Yes. We are using the Spring Boot framework to start our apps, including setting up Kafka. You can [use environment variables to set system properties](https://docs.spring.io/spring-boot/docs/current/reference/html/spring-boot-features.html#boot-features-external-config-relaxed-binding-from-environment-variables), including [Kafka properties](https://docs.spring.io/spring-boot/docs/current/reference/html/appendix-application-properties.html#integration-properties). From there you can set your SSL configuration for Kafka.
+
+If Schema Registry is configured to use security (SSL), then you also need to set the following config: https://docs.confluent.io/current/kafka/encryption.html#encryption-ssl-schema-registry.
+
+> **Note** In the logs you might see something like `The configuration 'kafkastore.ssl.truststore.password' was supplied but isn't a known config.` The configuration is not a configuration required for the producer. These WARN message can be safely ignored. Each of Datahub services are passed a full set of configuration but may not require all the configurations that are passed to them. These warn messages indicate that the service was passed a configuration that is not relevant to it and can be safely ignored.