From ce8d638cdb829e1e17aab91dcdb62669171de43a Mon Sep 17 00:00:00 2001 From: Joao Alexandre Prado Tavares Cunha Date: Sun, 9 Oct 2022 17:20:06 +0200 Subject: [PATCH 1/2] Makes TLS communication between KAS pods possible Relies on the existing internal TLS mechanism that adds support to the chart to share and mount trusted CAs across GitLab chart components. Adds support to gitlab.kas.privateApi.tls.enabled and gitlab.kas.privateApi.tls.secretName chart values, so that we can: - Decide when to mount the TLS secrets and configure KAS with with them through gitlab.kas.privateApi.tls.enabled. - Mount the Kubernetes TLS secret based on the gitlab.kas.privateApi.tls.secretName. - Allow the user to set the OWN_PRIVATE_API_HOST env through gitlab.kas.privateApi.host. Changelog: added --- .../charts/kas/templates/_default-config.yaml | 6 ++ .../charts/kas/templates/deployment.yaml | 13 +++- charts/gitlab/charts/kas/values.yaml | 7 +- doc/charts/gitlab/kas/index.md | 45 +++++++++++- spec/configuration/kas_spec.rb | 68 +++++++++++++++++++ 5 files changed, 135 insertions(+), 4 deletions(-) diff --git a/charts/gitlab/charts/kas/templates/_default-config.yaml b/charts/gitlab/charts/kas/templates/_default-config.yaml index bce17440c2..c56de9b762 100644 --- a/charts/gitlab/charts/kas/templates/_default-config.yaml +++ b/charts/gitlab/charts/kas/templates/_default-config.yaml @@ -22,3 +22,9 @@ private_api: listen: address: ":{{ .Values.service.privateApiPort }}" authentication_secret_file: "/etc/kas/.gitlab_kas_private_api_secret" + {{- if .Values.privateApi.tls.enabled }} + certificate_file: "/etc/kas/tls.crt" + key_file: "/etc/kas/tls.key" + ca_certificate_file: "/etc/ssl/certs/ca-certificates.crt" + {{- end }} + diff --git a/charts/gitlab/charts/kas/templates/deployment.yaml b/charts/gitlab/charts/kas/templates/deployment.yaml index cb7b7dd7e1..99a9b76509 100644 --- a/charts/gitlab/charts/kas/templates/deployment.yaml +++ b/charts/gitlab/charts/kas/templates/deployment.yaml @@ -84,12 +84,14 @@ spec: args: - "--configuration-file=/etc/kas/config.yaml" env: + - name: OWN_PRIVATE_API_HOST + value: "{{ .Values.privateApi.host }}" - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: OWN_PRIVATE_API_URL - value: "grpc://$(POD_IP):{{ .Values.service.privateApiPort }}" + value: "{{ ternary "grpcs" "grpc" .Values.privateApi.tls.enabled }}://$(POD_IP):{{ .Values.service.privateApiPort }}" ports: - containerPort: {{ .Values.service.internalPort }} name: {{ template "name" . }} @@ -138,6 +140,15 @@ spec: items: - key: {{ template "gitlab.kas.privateApi.key" . }} path: .gitlab_kas_private_api_secret + {{- if $.Values.privateApi.tls.enabled }} + - secret: + name: {{ .Values.privateApi.tls.secretName }} + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + {{- end }} {{- if .Values.redis.enabled -}} {{- include "gitlab.redis.secrets" . | nindent 12 }} {{- end }} diff --git a/charts/gitlab/charts/kas/values.yaml b/charts/gitlab/charts/kas/values.yaml index 0adde4d2a5..c9ae49f137 100644 --- a/charts/gitlab/charts/kas/values.yaml +++ b/charts/gitlab/charts/kas/values.yaml @@ -93,10 +93,13 @@ workhorse: {} # merged with the default kas config customConfig: {} -privateApi: {} +privateApi: + host: private-api.test # secret: # key: - + tls: + enabled: false + # secretName: deployment: strategy: {} # minReadySeconds: 0 diff --git a/doc/charts/gitlab/kas/index.md b/doc/charts/gitlab/kas/index.md index c152e1b0c5..895ab5e24a 100644 --- a/doc/charts/gitlab/kas/index.md +++ b/doc/charts/gitlab/kas/index.md @@ -62,7 +62,7 @@ specified in `global.hosts.domain`. You can pass these parameters to the `helm install` command by using the `--set` flags. | Parameter | Default | Description | -| ----------------------------------------- | ----------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +|-------------------------------------------|-------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `annotations` | `{}` | Pod annotations. | | `common.labels` | `{}` | Supplemental labels that are applied to all objects created by this chart. | | `extraContainers` | | List of extra containers to include. | @@ -109,6 +109,9 @@ You can pass these parameters to the `helm install` command by using the `--set` | `service.privateApiPort` | `8155` | Internal port to expose `kas`' private API on (for `kas` -> `kas` communication). | | `privateApi.secret` | Autogenerated | The name of the secret to use for authenticating with the database. | | `privateApi.key` | Autogenerated | The name of the key in `privateApi.secret` to use. | +| `privateApi.tls` | `false` | Enable `kas` pods to communicate with each other using TLS. | +| `privateApi.tls.secretName` | `nil` | Name of the [Kubernetes TLS secret](https://kubernetes.io/docs/concepts/configuration/secret/#tls-secrets) which contains the certificate and its associated key. Required if `privateApi.tls` is `true`. | +| `privateApi.host` | `private-api.test` | Name of the host added to your certificate's SAN. Required if `privateApi.tls` is `true`. | | `global.kas.service.apiExternalPort` | `8153` | External port for the internal API (for GitLab backend). | | `service.type` | `ClusterIP` | Service type. | | `tolerations` | `[]` | Toleration labels for pod assignment. | @@ -116,6 +119,46 @@ You can pass these parameters to the `helm install` command by using the `--set` | `deployment.minReadySeconds` | `0` | Minimum number of seconds that must pass before a `kas` pod is considered ready. | | `deployment.strategy` | `{}` | Allows one to configure the update strategy utilized by the deployment. | +## Enable TLS communication between `kas` pods + +Enable communication when you want your `kas` pods to communicate with each other over TLS: + +1. Create the certificate authority and certificates that your `kas` pods will trust. +1. Configure your chart to use the trusted certificates. + +### Create your certificates + +To create a certificate authority (CA) and the required certificates, follow the steps in +[Use TLS between components of the GitLab chart](../../../advanced/internal-tls/index.md). + +When configuring your `CERT_SANS`, you must add a host to enable TLS communication. The host does not need to be resolvable. Any dummy host will work, including the default `private-api.test`. + +### Configure chart values + +For `kas` to use the certificates you created, set the value of `global.certificates.customCAs`, `gitlab.kas.privateApi.host` and `gitlab.kas.privateApi.tls`. + +For example, you could use this `values.yaml` file to deploy your chart: + + ```yaml + .internal-ca: &internal-ca gitlab-internal-tls-ca # The secret name you used to share your TLS CA. + .internal-tls: &internal-tls gitlab-internal-tls # The secret name you used to share your TLS certificate. + + global: + certificates: + customCAs: + - secret: *internal-ca + hosts: + domain: gitlab.example.com # Your gitlab domain + + gitlab: + kas: + host: private-api.test # The host you've added to your certificate's SAN on the previous step. + privateApi: + tls: + enabled: true + secretName: *internal-tls + ``` + ## Test the `kas` chart To install the chart: diff --git a/spec/configuration/kas_spec.rb b/spec/configuration/kas_spec.rb index 35b3a63558..c6002ef42e 100644 --- a/spec/configuration/kas_spec.rb +++ b/spec/configuration/kas_spec.rb @@ -170,6 +170,32 @@ describe 'kas configuration' do expect(config_yaml_data['gitlab']).not_to be_nil end + context 'when privateApi.tls is enabled' do + let(:kas_values) do + default_kas_values.deep_merge!(YAML.safe_load(%( + gitlab: + kas: + privateApi: + tls: + enabled: true + ))) + end + + it 'does not have redis config' do + expected_config = { + "listen" => { + "address" => :"8155", + "authentication_secret_file" => "/etc/kas/.gitlab_kas_private_api_secret", + "ca_certificate_file" => "/etc/ssl/certs/ca-certificates.crt", + "certificate_file" => "/etc/kas/tls.crt", + "key_file" => "/etc/kas/tls.key" + } + } + + expect(config_yaml_data['private_api']).to eq(expected_config) + end + end + context 'when customConfig is given' do let(:custom_config) do YAML.safe_load(%( @@ -416,6 +442,12 @@ describe 'kas configuration' do expect(deployment['spec']).not_to have_key('minReadySeconds') end + it 'sets OWN_PRIVATE_API_URL to use grpc' do + expect(deployment['spec']['template']['spec']['containers'].first['env']).to include( + { "name" => "OWN_PRIVATE_API_URL", "value" => "grpc://$(POD_IP):8155" } + ) + end + context 'when deployment.minReadySeconds is given' do let(:deployment_values) { { 'minReadySeconds' => 60 } } @@ -423,6 +455,42 @@ describe 'kas configuration' do expect(deployment['spec']).to include('minReadySeconds' => 60) end end + + context 'when privateApi.tls is enabled' do + let(:kas_values) do + default_kas_values.deep_merge!(YAML.safe_load(%( + gitlab: + kas: + privateApi: + tls: + enabled: true + ))) + end + + it 'sets OWN_PRIVATE_API_URL to use grpcs' do + expect(deployment['spec']['template']['spec']['containers'].first['env']).to include( + { "name" => "OWN_PRIVATE_API_URL", "value" => "grpcs://$(POD_IP):8155" } + ) + end + + it 'creates the TLS secret volume' do + init_etc_kas_volume = deployment['spec']['template']['spec']['volumes'].find do |volume| + volume['name'] == 'init-etc-kas' + end + + expect(init_etc_kas_volume['projected']['sources']).to include( + { + "secret" => { + "name" => nil, + "items" => [ + { "key" => "tls.crt", "path" => "tls.crt" }, + { "key" => "tls.key", "path" => "tls.key" } + ] + } + } + ) + end + end end end -- GitLab From d4c80dfead371d5b8c26430026f493127044f9a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Cunha?= Date: Fri, 14 Oct 2022 12:48:31 +0200 Subject: [PATCH 2/2] Point KAS OWN_PRIVATE_API_HOST to its service name This makes things simpler, since we don't need to ask users to add a specific host for kas to other kas internally, just for the sake of being able to create an SSL certificate. The kas service already has a certificate that KAS pods can use to talk to each other. --- charts/gitlab/charts/kas/templates/deployment.yaml | 2 +- charts/gitlab/charts/kas/values.yaml | 1 - charts/gitlab/templates/_kas.tpl | 12 ++++++++++-- doc/charts/gitlab/kas/index.md | 5 +---- spec/configuration/kas_spec.rb | 6 ++++++ 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/charts/gitlab/charts/kas/templates/deployment.yaml b/charts/gitlab/charts/kas/templates/deployment.yaml index 99a9b76509..7cf9cfc9e6 100644 --- a/charts/gitlab/charts/kas/templates/deployment.yaml +++ b/charts/gitlab/charts/kas/templates/deployment.yaml @@ -85,7 +85,7 @@ spec: - "--configuration-file=/etc/kas/config.yaml" env: - name: OWN_PRIVATE_API_HOST - value: "{{ .Values.privateApi.host }}" + value: {{ include "gitlab.kas.serviceHost" . }} - name: POD_IP valueFrom: fieldRef: diff --git a/charts/gitlab/charts/kas/values.yaml b/charts/gitlab/charts/kas/values.yaml index c9ae49f137..aba13a9f76 100644 --- a/charts/gitlab/charts/kas/values.yaml +++ b/charts/gitlab/charts/kas/values.yaml @@ -94,7 +94,6 @@ workhorse: {} customConfig: {} privateApi: - host: private-api.test # secret: # key: tls: diff --git a/charts/gitlab/templates/_kas.tpl b/charts/gitlab/templates/_kas.tpl index 11d470ccb7..ba879107fa 100644 --- a/charts/gitlab/templates/_kas.tpl +++ b/charts/gitlab/templates/_kas.tpl @@ -43,12 +43,20 @@ Returns the KAS internal URL (for GitLab backend connections) {{- if .Values.global.appConfig.gitlab_kas.internalUrl -}} {{- .Values.global.appConfig.gitlab_kas.internalUrl -}} {{- else -}} -{{- $serviceName := include "gitlab.kas.serviceName" . -}} +{{- $serviceHost := include "gitlab.kas.serviceHost" . -}} {{- $port := .Values.global.kas.service.apiExternalPort -}} -{{- printf "grpc://%s.%s.svc:%s" $serviceName $.Release.Namespace (toString $port) -}} +{{- printf "grpc://%s:%s" $serviceHost (toString $port) -}} {{- end -}} {{- end -}} +{{/* +Return the KAS service host +*/}} +{{- define "gitlab.kas.serviceHost" -}} +{{- $serviceName := include "gitlab.kas.serviceName" . -}} +{{- printf "%s.%s.svc" $serviceName $.Release.Namespace -}} +{{- end -}} + {{/* Return the KAS service name */}} diff --git a/doc/charts/gitlab/kas/index.md b/doc/charts/gitlab/kas/index.md index 895ab5e24a..2a222cb385 100644 --- a/doc/charts/gitlab/kas/index.md +++ b/doc/charts/gitlab/kas/index.md @@ -131,11 +131,9 @@ Enable communication when you want your `kas` pods to communicate with each othe To create a certificate authority (CA) and the required certificates, follow the steps in [Use TLS between components of the GitLab chart](../../../advanced/internal-tls/index.md). -When configuring your `CERT_SANS`, you must add a host to enable TLS communication. The host does not need to be resolvable. Any dummy host will work, including the default `private-api.test`. - ### Configure chart values -For `kas` to use the certificates you created, set the value of `global.certificates.customCAs`, `gitlab.kas.privateApi.host` and `gitlab.kas.privateApi.tls`. +For `kas` to use the certificates you created, set the value of `global.certificates.customCAs` and `gitlab.kas.privateApi.tls`. For example, you could use this `values.yaml` file to deploy your chart: @@ -152,7 +150,6 @@ For example, you could use this `values.yaml` file to deploy your chart: gitlab: kas: - host: private-api.test # The host you've added to your certificate's SAN on the previous step. privateApi: tls: enabled: true diff --git a/spec/configuration/kas_spec.rb b/spec/configuration/kas_spec.rb index c6002ef42e..fe55af5298 100644 --- a/spec/configuration/kas_spec.rb +++ b/spec/configuration/kas_spec.rb @@ -448,6 +448,12 @@ describe 'kas configuration' do ) end + it 'sets OWN_PRIVATE_API_HOST to use its service host' do + expect(deployment['spec']['template']['spec']['containers'].first['env']).to include( + { "name" => "OWN_PRIVATE_API_HOST", "value" => "test-kas.default.svc" } + ) + end + context 'when deployment.minReadySeconds is given' do let(:deployment_values) { { 'minReadySeconds' => 60 } } -- GitLab