From c2308762621fb193e8a5d0697e993d2b07d9ecfd Mon Sep 17 00:00:00 2001 From: Mikhail Mazurskiy Date: Wed, 24 Jul 2024 20:58:36 +1000 Subject: [PATCH 1/7] Set terminationGracePeriodSeconds to 2 hours to avoid kas pod kills Changelog: fixed --- charts/gitlab/charts/kas/values.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/gitlab/charts/kas/values.yaml b/charts/gitlab/charts/kas/values.yaml index 685af6d3e8..eebe50391e 100644 --- a/charts/gitlab/charts/kas/values.yaml +++ b/charts/gitlab/charts/kas/values.yaml @@ -133,7 +133,9 @@ privateApi: {} # secret: # key: deployment: - terminationGracePeriodSeconds: 300 + # By default, kas is configured with max connection age of two hours. + # Kubernetes needs to wait for kas to drain/kill those connections and exit rather than killing kas pods earlier. + terminationGracePeriodSeconds: 7260 strategy: {} # minReadySeconds: 0 -- GitLab From 69f5a0c012442d724b073c22cc217f2e56ba1777 Mon Sep 17 00:00:00 2001 From: Mikhail Mazurskiy Date: Fri, 26 Jul 2024 16:26:33 +1000 Subject: [PATCH 2/7] Set default maxSurge and maxUnavailable for kas Changelog: changed --- charts/gitlab/charts/kas/values.yaml | 7 ++++++- spec/configuration/kas_spec.rb | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/charts/gitlab/charts/kas/values.yaml b/charts/gitlab/charts/kas/values.yaml index eebe50391e..0ac8a180a3 100644 --- a/charts/gitlab/charts/kas/values.yaml +++ b/charts/gitlab/charts/kas/values.yaml @@ -136,7 +136,12 @@ deployment: # By default, kas is configured with max connection age of two hours. # Kubernetes needs to wait for kas to drain/kill those connections and exit rather than killing kas pods earlier. terminationGracePeriodSeconds: 7260 - strategy: {} + strategy: + rollingUpdate: + # This allows Chart to finish deploying even if old Pods are still around + # e.g. due to long-running connections. + maxSurge: 100% + maxUnavailable: 0 # minReadySeconds: 0 ## Allow to overwrite under which User and Group the Pod will be running. diff --git a/spec/configuration/kas_spec.rb b/spec/configuration/kas_spec.rb index 787a72fc55..49a68c7eda 100644 --- a/spec/configuration/kas_spec.rb +++ b/spec/configuration/kas_spec.rb @@ -107,6 +107,24 @@ describe 'kas configuration' do end end + context 'when kas is enabled with default values' do + let(:kas_enabled_template) do + HelmTemplate.new(default_values) + end + + describe 'templates/deployment.yaml' do + subject(:deployment) { kas_enabled_template.resources_by_kind('Deployment')['Deployment/test-kas'] } + + it 'does specify terminationGracePeriodSeconds' do + expect(deployment['spec']['template']['spec']['terminationGracePeriodSeconds']).to eq(7260) + end + + it 'does specify strategy with 100% max surge' do + expect(deployment['spec']['strategy']).to eq({ 'rollingUpdate' => { 'maxSurge' => '100%', 'maxUnavailable' => 0 } }) + end + end + end + context 'when kas is enabled with custom values' do let(:kas_enabled_template) do HelmTemplate.new(default_values.merge(kas_values)) -- GitLab From cba22faccfe1e42c6f7c766c9228a74fb4f24a8a Mon Sep 17 00:00:00 2001 From: Timo Furrer Date: Mon, 5 Aug 2024 09:00:33 +0200 Subject: [PATCH 3/7] Ignore KAS deployment for default strategy spec --- spec/configuration/strategy_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/configuration/strategy_spec.rb b/spec/configuration/strategy_spec.rb index 6bcfa82dc4..df68bf92d5 100644 --- a/spec/configuration/strategy_spec.rb +++ b/spec/configuration/strategy_spec.rb @@ -105,6 +105,7 @@ describe 'Strategy configuration' do 'Deployment/test-toolbox', 'Deployment/test-minio', 'Deployment/test-gitlab-runner', + 'Deployment/test-kas', 'StatefulSet/test-redis-master', 'StatefulSet/test-postgresql' ] -- GitLab From bf47acc9224435749b00f57f12bb93a5b7070c4a Mon Sep 17 00:00:00 2001 From: Mikhail Mazurskiy Date: Mon, 12 Aug 2024 14:37:38 +1000 Subject: [PATCH 4/7] Guard nginx Ingress annotations with a check Changelog: changed --- charts/gitlab/charts/kas/templates/ingress.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/gitlab/charts/kas/templates/ingress.yaml b/charts/gitlab/charts/kas/templates/ingress.yaml index ee5fc6edee..fab6c0ed62 100644 --- a/charts/gitlab/charts/kas/templates/ingress.yaml +++ b/charts/gitlab/charts/kas/templates/ingress.yaml @@ -17,14 +17,14 @@ metadata: {{- if eq "nginx" .Values.global.ingress.provider }} nginx.ingress.kubernetes.io/proxy-buffering: "off" nginx.ingress.kubernetes.io/custom-http-errors: "" - {{- end }} - {{- if $.Values.global.kas.tls.enabled }} + {{- if $.Values.global.kas.tls.enabled }} nginx.ingress.kubernetes.io/backend-protocol: https - {{- if $.Values.global.kas.tls.verify }} + {{- if $.Values.global.kas.tls.verify }} nginx.ingress.kubernetes.io/proxy-ssl-verify: 'on' nginx.ingress.kubernetes.io/proxy-ssl-name: {{ include "fullname" . }}.{{ $.Release.Namespace }}.svc - {{- if $.Values.global.kas.tls.caSecretName }} + {{- if $.Values.global.kas.tls.caSecretName }} nginx.ingress.kubernetes.io/proxy-ssl-secret: {{ $.Release.Namespace }}/{{ $.Values.global.kas.tls.caSecretName }} + {{- end }} {{- end }} {{- end }} {{- end }} -- GitLab From e80fcdeb2303c555329a8a8913d2cb0b071ae807 Mon Sep 17 00:00:00 2001 From: Mikhail Mazurskiy Date: Thu, 12 Sep 2024 12:52:05 +1000 Subject: [PATCH 5/7] Doc tweaks --- doc/charts/gitlab/kas/index.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/doc/charts/gitlab/kas/index.md b/doc/charts/gitlab/kas/index.md index b210ddf20a..dc721ee51f 100644 --- a/doc/charts/gitlab/kas/index.md +++ b/doc/charts/gitlab/kas/index.md @@ -12,18 +12,16 @@ DETAILS: The `kas` sub-chart provides a configurable deployment of the [GitLab agent server (KAS)](https://docs.gitlab.com/ee/administration/clusters/kas.html). -The agent server is a component you install together with GitLab. It is required to -manage the [GitLab agent for Kubernetes](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent). +The agent server is a component you install together with GitLab. It is the component +[GitLab Agent for Kubernetes](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent) connects to. -This chart depends on access to the GitLab API and the Gitaly Servers. +This chart depends on access to the GitLab API, the Gitaly Servers, and Redis. When you enable this chart, an Ingress is deployed. To consume minimal resources, the `kas` container uses a distroless image. -The deployed services are exposed by an Ingress, which uses -[WebSocket proxying](https://nginx.org/en/docs/http/websocket.html) for communication. -This proxy allows long-lived connections with the external component, -[`agentk`](https://docs.gitlab.com/ee/user/clusters/agent/install/index.html). -`agentk` is the Kubernetes cluster-side agent counterpart. +The deployed services are exposed by an Ingress, which accepts +[WebSocket](https://nginx.org/en/docs/http/websocket.html) connections from agents. +These connections are long-lived and, hence, Ingress should be configured appropriately. The route to access the service depends on your [Ingress configuration](#specify-an-ingress). -- GitLab From 5add56a0bc7fb8b741e056e6f9e240d46435aeb4 Mon Sep 17 00:00:00 2001 From: Taka Nishida Date: Wed, 25 Sep 2024 16:26:46 +0900 Subject: [PATCH 6/7] Docs: Update defaults and add reference link --- doc/charts/gitlab/kas/index.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/charts/gitlab/kas/index.md b/doc/charts/gitlab/kas/index.md index dc721ee51f..31abe81d87 100644 --- a/doc/charts/gitlab/kas/index.md +++ b/doc/charts/gitlab/kas/index.md @@ -65,7 +65,7 @@ You can pass these parameters to the `helm install` command by using the `--set` | Parameter | Default | Description | |----------------------------------------------------------|-------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `affinity` | `{}` | [Affinity rules](../index.md#affinity) for pod assignment | +| `affinity` | `{}` | [Affinity rules](../index.md#affinity) for pod assignment | | `annotations` | `{}` | Pod annotations. | | `common.labels` | `{}` | Supplemental labels that are applied to all objects created by this chart. | | `securityContext.runAsUser` | `65532` | User ID under which the pod should be started | @@ -140,11 +140,11 @@ You can pass these parameters to the `helm install` command by using the `--set` | `service.loadBalancerSourceRanges` | `nil` | A list of custom load balancer source ranges when `service.type` is `LoadBalancer`. | | `service.kubernetesApiPort` | `8154` | External port to expose proxied Kubernetes API on. | | `service.privateApiPort` | `8155` | Internal port to expose `kas`' private API on (for `kas` -> `kas` communication). | -| `serviceAccount.annotations` | `{}` | ServiceAccount annotations. | -| `serviceAccount.automountServiceAccountToken`| `false` | Indicates whether or not the default ServiceAccount access token should be mounted in pods. | -| `serviceAccount.create` | `false` | Indicates whether or not a ServiceAccount should be created. | -| `serviceAccount.enabled` | `false` | Indicates whether or not to use a ServiceAccount. | -| `serviceAccount.name` | | Name of the ServiceAccount. If not set, the full chart name is used. | +| `serviceAccount.annotations` | `{}` | ServiceAccount annotations. | +| `serviceAccount.automountServiceAccountToken` | `false` | Indicates whether or not the default ServiceAccount access token should be mounted in pods. | +| `serviceAccount.create` | `false` | Indicates whether or not a ServiceAccount should be created. | +| `serviceAccount.enabled` | `false` | Indicates whether or not to use a ServiceAccount. | +| `serviceAccount.name` | | Name of the ServiceAccount. If not set, the full chart name is used. | | `privateApi.secret` | Autogenerated | The name of the secret to use for authenticating with the database. | | `privateApi.key` | Autogenerated | The name of the key in `privateApi.secret` to use. | | `global.kas.service.apiExternalPort` | `8153` | External port for the internal API (for GitLab backend). | @@ -152,8 +152,8 @@ You can pass these parameters to the `helm install` command by using the `--set` | `tolerations` | `[]` | Toleration labels for pod assignment. | | `customConfig` | `{}` | When given, merges the default `kas` configuration with these values giving precedence to those defined here. | | `deployment.minReadySeconds` | `0` | Minimum number of seconds that must pass before a `kas` pod is considered ready. | -| `deployment.strategy` | `{}` | Allows one to configure the update strategy utilized by the deployment. | -| `deployment.terminationGracePeriodSeconds` | `300` | How much time in seconds a Pod is allowed to spend shutting down after receiving SIGTERM. | +| `deployment.strategy` | `rollingUpdate{}` | Allows one to configure the update strategy utilized by the deployment. | +| `deployment.terminationGracePeriodSeconds` | `7260` | How much time in seconds a Pod is allowed to spend shutting down after receiving SIGTERM. It needs to be greater than the [maximum connection age](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/shutdown.md?ref_type=heads#maximum-connection-age) | | `priorityClassName` | | [Priority class](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) assigned to pods. | ## Enable TLS communication -- GitLab From e5108c83f5e99c5276b7af5ae2cf1b041a75ab90 Mon Sep 17 00:00:00 2001 From: Mikhail Mazurskiy Date: Wed, 2 Oct 2024 16:12:15 +1000 Subject: [PATCH 7/7] Document long-running connections config for kas --- doc/charts/gitlab/kas/index.md | 216 +++++++++++++++++---------------- 1 file changed, 114 insertions(+), 102 deletions(-) diff --git a/doc/charts/gitlab/kas/index.md b/doc/charts/gitlab/kas/index.md index 31abe81d87..dcc28ee3f2 100644 --- a/doc/charts/gitlab/kas/index.md +++ b/doc/charts/gitlab/kas/index.md @@ -16,17 +16,7 @@ The agent server is a component you install together with GitLab. It is the comp [GitLab Agent for Kubernetes](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent) connects to. This chart depends on access to the GitLab API, the Gitaly Servers, and Redis. -When you enable this chart, an Ingress is deployed. - To consume minimal resources, the `kas` container uses a distroless image. -The deployed services are exposed by an Ingress, which accepts -[WebSocket](https://nginx.org/en/docs/http/websocket.html) connections from agents. -These connections are long-lived and, hence, Ingress should be configured appropriately. - -The route to access the service depends on your [Ingress configuration](#specify-an-ingress). - -For more information, see the -[GitLab agent for Kubernetes architecture](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/architecture.md). ## Disable the agent server @@ -39,6 +29,28 @@ For example: helm upgrade --install kas --set global.kas.enabled=false ``` +## Long-running connections and Ingress + +When you enable this chart, an Ingress is deployed. +The Ingress: + +- Accepts [WebSocket](https://nginx.org/en/docs/http/websocket.html) connections from agents. +- Exposes Kubernetes API proxy for [user access](https://docs.gitlab.com/ee/user/clusters/agent/user_access.html), + [access from CI](https://docs.gitlab.com/ee/user/clusters/agent/ci_cd_workflow.html), and other features. + +Connections for several features (e.g. using `kubectl logs` in a CI job) are long-lived and, hence, Ingress should be configured appropriately. +The configuration is Ingress-specific. +You need to ensure the Ingress allows the connections to stay up for at least 2 hours (in default Chart configuration). +For the Ingress controllers that are deployed into the cluster itself, such as the nginx Ingress controller, +make sure that the Pods of the Ingress controller themselves have at least 2 hours of +[termination grace period](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-termination). +This is controlled by the `terminationGracePeriodSeconds` of the Pods. +Overall, `kas` should be allowed to terminate the connections before the Ingress terminates them. +The same applies to any other network load balancer or proxy that might be in front of the Ingress. +To achieve that, the configuration for Ingress should allow slightly longer maximum connection age than `kas` configuration. + +The route to access the service depends on your [Ingress configuration](#specify-an-ingress). + ### Specify an Ingress When you use the chart's Ingress with the default configuration, @@ -55,7 +67,7 @@ Set `global.hosts.kas.name`, for example: global.hosts.kas.name: kas.my-other-domain.com ``` -This example uses `kas.my-other-domain.com` as the host for the KAS Ingress alone. +This example uses `kas.my-other-domain.com` as the host for the `kas` Ingress alone. The rest of the services (including GitLab, Registry, MinIO, etc.) use the domain specified in `global.hosts.domain`. @@ -63,98 +75,98 @@ specified in `global.hosts.domain`. You can pass these parameters to the `helm install` command by using the `--set` flags. -| Parameter | Default | Description | -|----------------------------------------------------------|-------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `affinity` | `{}` | [Affinity rules](../index.md#affinity) for pod assignment | -| `annotations` | `{}` | Pod annotations. | -| `common.labels` | `{}` | Supplemental labels that are applied to all objects created by this chart. | -| `securityContext.runAsUser` | `65532` | User ID under which the pod should be started | -| `securityContext.runAsGroup` | `65534` | Group ID under which the pod should be started | -| `securityContext.fsGroup` | `65532` | Group ID under which the pod should be started | -| `securityContext.fsGroupChangePolicy` | | Policy for changing ownership and permission of the volume (requires Kubernetes 1.23) | -| `securityContext.seccompProfile.type` | `RuntimeDefault` | Seccomp profile to use | -| `containerSecurityContext.runAsUser` | `65532` | Override container [securityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#securitycontext-v1-core) user ID under which the container is started | -| `containerSecurityContext.allowPrivilegeEscalation` | `false` | Controls whether a process of the container can gain more privileges than its parent process | -| `containerSecurityContext.runAsNonRoot` | `true` | Controls whether the container runs with a non-root user | -| `containerSecurityContext.capabilities.drop` | `[ "ALL" ]` | Removes [Linux capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) for the Gitaly container | -| `extraContainers` | | List of extra containers to include. | -| `extraEnv` | | List of extra environment variables to expose | -| `extraEnvFrom` | | List of extra environment variables from other data sources to expose | -| `init.containerSecurityContext` | | init container securityContext overrides | -| `init.containerSecurityContext.allowPrivilegeEscalation` | `false` | initContainer specific: Controls whether a process can gain more privileges than its parent process | -| `init.containerSecurityContext.runAsNonRoot` | `true` | initContainer specific: Controls whether the container runs with a non-root user | -| `init.containerSecurityContext.capabilities.drop` | `[ "ALL" ]` | initContainer specific: Removes [Linux capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) for the container | -| `image.repository` | `registry.gitlab.com/gitlab-org/build/cng/gitlab-kas` | Image repository. | -| `image.tag` | `v13.7.0` | Image tag. | -| `hpa.behavior` | `{scaleDown: {stabilizationWindowSeconds: 300 }}` | Behavior contains the specifications for up- and downscaling behavior (requires `autoscaling/v2beta2` or higher). | -| `hpa.customMetrics` | `[]` | Custom metrics contains the specifications for which to use to calculate the desired replica count (overrides the default use of Average CPU Utilization configured in `targetAverageUtilization`). | -| `hpa.cpu.targetType` | `AverageValue` | Set the autoscaling CPU target type, must be either `Utilization` or `AverageValue`. | -| `hpa.cpu.targetAverageValue` | `100m` | Set the autoscaling CPU target value. | -| `hpa.cpu.targetAverageUtilization` | | Set the autoscaling CPU target utilization. | -| `hpa.memory.targetType` | | Set the autoscaling memory target type, must be either `Utilization` or `AverageValue`. | -| `hpa.memory.targetAverageValue` | | Set the autoscaling memory target value. | -| `hpa.memory.targetAverageUtilization` | | Set the autoscaling memory target utilization. | -| `hpa.targetAverageValue` | | **DEPRECATED** Set the autoscaling CPU target value | -| `ingress.enabled` | `true` if `global.kas.enabled=true` | You can use `kas.ingress.enabled` to explicitly turn it on or off. If not set, you can optionally use `global.ingress.enabled` for the same purpose. | -| `ingress.apiVersion` | | Value to use in the `apiVersion` field. | -| `ingress.annotations` | `{}` | Ingress annotations. | -| `ingress.tls` | `{}` | Ingress TLS configuration. | -| `ingress.agentPath` | `/` | Ingress path for the agent API endpoint. | -| `ingress.k8sApiPath` | `/k8s-proxy` | Ingress path for Kubernetes API endpoint. | -| `keda.enabled` | `false` | Use [KEDA](https://keda.sh/) `ScaledObjects` instead of `HorizontalPodAutoscalers` | -| `keda.pollingInterval` | `30` | The interval to check each trigger on | -| `keda.cooldownPeriod` | `300` | The period to wait after the last trigger reported active before scaling the resource back to 0 | -| `keda.minReplicaCount` | | Minimum number of replicas KEDA will scale the resource down to, defaults to `minReplicas` | -| `keda.maxReplicaCount` | | Maximum number of replicas KEDA will scale the resource up to, defaults to `maxReplicas` | -| `keda.fallback` | | KEDA fallback configuration, see the [documentation](https://keda.sh/docs/2.10/concepts/scaling-deployments/#fallback) | -| `keda.hpaName` | | The name of the HPA resource KEDA will create, defaults to `keda-hpa-{scaled-object-name}` | -| `keda.restoreToOriginalReplicaCount` | | Specifies whether the target resource should be scaled back to original replicas count after the `ScaledObject` is deleted | -| `keda.behavior` | | The specifications for up- and downscaling behavior, defaults to `hpa.behavior` | -| `keda.triggers` | | List of triggers to activate scaling of the target resource, defaults to triggers computed from `hpa.cpu` and `hpa.memory` | -| `metrics.enabled` | `true` | If a metrics endpoint should be made available for scraping. | -| `metrics.path` | `/metrics` | Metrics endpoint path. | -| `metrics.serviceMonitor.enabled` | `false` | If a ServiceMonitor should be created to enable Prometheus Operator to manage the metrics scraping. Enabling removes the `prometheus.io` scrape annotations. It cannot be enabled together with `metrics.podMonitor.enabled`. | -| `metrics.serviceMonitor.additionalLabels` | `{}` | Additional labels to add to the ServiceMonitor. | -| `metrics.serviceMonitor.endpointConfig` | `{}` | Additional endpoint configuration for the ServiceMonitor. | -| `metrics.podMonitor.enabled` | `false` | If a PodMonitor should be created to enable Prometheus Operator to manage the metrics scraping. Enabling removes the `prometheus.io` scrape annotations. It cannot be enabled together with `metrics.serviceMonitor.enabled`. | -| `metrics.podMonitor.additionalLabels` | `{}` | Additional labels to add to the PodMonitor. | -| `metrics.podMonitor.endpointConfig` | `{}` | Additional endpoint configuration for the PodMonitor. | -| `maxReplicas` | `10` | HPA `maxReplicas`. | -| `maxUnavailable` | `1` | HPA `maxUnavailable`. | -| `minReplicas` | `2` | HPA `maxReplicas`. | -| `nodeSelector` | | Define a [nodeSelector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for the `Pod`s of this `Deployment`, if present. | -| `observability.port` | `8151` | Observability endpoint port. Used for metrics and probe endpoints. | -| `observability.livenessProbe.path` | `/liveness` | URI for the liveness probe endpoint. This value has to match the `observability.liveness_probe.url_path` value from the KAS service configuration. | -| `observability.readinessProbe.path` | `/readiness` | URI for the readiness probe endpoint. This value has to match the `observability.readiness_probe.url_path` value from the KAS service configuration. | -| `serviceAccount.annotations` | `{}` | Service account annotations. | -| `podLabels` | `{}` | Supplemental Pod labels. Not used for selectors. | -| `serviceLabels` | `{}` | Supplemental service labels. | -| `common.labels` | | Supplemental labels that are applied to all objects created by this chart. | -| `redis.enabled` | `true` | Allows opting-out of using Redis for KAS features. Warnings: Redis will become a hard dependency soon, so this key is already deprecated. | -| `resources.requests.cpu` | `75m` | GitLab Exporter minimum CPU. | -| `resources.requests.memory` | `100M` | GitLab Exporter minimum memory. | -| `service.externalPort` | `8150` | External port (for `agentk` connections). | -| `service.internalPort` | `8150` | Internal port (for `agentk` connections). | -| `service.apiInternalPort` | `8153` | Internal port for the internal API (for GitLab backend). | -| `service.loadBalancerIP` | `nil` | A custom load balancer IP when `service.type` is `LoadBalancer`. | -| `service.loadBalancerSourceRanges` | `nil` | A list of custom load balancer source ranges when `service.type` is `LoadBalancer`. | -| `service.kubernetesApiPort` | `8154` | External port to expose proxied Kubernetes API on. | -| `service.privateApiPort` | `8155` | Internal port to expose `kas`' private API on (for `kas` -> `kas` communication). | -| `serviceAccount.annotations` | `{}` | ServiceAccount annotations. | -| `serviceAccount.automountServiceAccountToken` | `false` | Indicates whether or not the default ServiceAccount access token should be mounted in pods. | -| `serviceAccount.create` | `false` | Indicates whether or not a ServiceAccount should be created. | -| `serviceAccount.enabled` | `false` | Indicates whether or not to use a ServiceAccount. | -| `serviceAccount.name` | | Name of the ServiceAccount. If not set, the full chart name is used. | -| `privateApi.secret` | Autogenerated | The name of the secret to use for authenticating with the database. | -| `privateApi.key` | Autogenerated | The name of the key in `privateApi.secret` to use. | -| `global.kas.service.apiExternalPort` | `8153` | External port for the internal API (for GitLab backend). | -| `service.type` | `ClusterIP` | Service type. | -| `tolerations` | `[]` | Toleration labels for pod assignment. | -| `customConfig` | `{}` | When given, merges the default `kas` configuration with these values giving precedence to those defined here. | -| `deployment.minReadySeconds` | `0` | Minimum number of seconds that must pass before a `kas` pod is considered ready. | -| `deployment.strategy` | `rollingUpdate{}` | Allows one to configure the update strategy utilized by the deployment. | +| Parameter | Default | Description | +|----------------------------------------------------------|-------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `affinity` | `{}` | [Affinity rules](../index.md#affinity) for pod assignment | +| `annotations` | `{}` | Pod annotations. | +| `common.labels` | `{}` | Supplemental labels that are applied to all objects created by this chart. | +| `securityContext.runAsUser` | `65532` | User ID under which the pod should be started | +| `securityContext.runAsGroup` | `65534` | Group ID under which the pod should be started | +| `securityContext.fsGroup` | `65532` | Group ID under which the pod should be started | +| `securityContext.fsGroupChangePolicy` | | Policy for changing ownership and permission of the volume (requires Kubernetes 1.23) | +| `securityContext.seccompProfile.type` | `RuntimeDefault` | Seccomp profile to use | +| `containerSecurityContext.runAsUser` | `65532` | Override container [securityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#securitycontext-v1-core) user ID under which the container is started | +| `containerSecurityContext.allowPrivilegeEscalation` | `false` | Controls whether a process of the container can gain more privileges than its parent process | +| `containerSecurityContext.runAsNonRoot` | `true` | Controls whether the container runs with a non-root user | +| `containerSecurityContext.capabilities.drop` | `[ "ALL" ]` | Removes [Linux capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) for the Gitaly container | +| `extraContainers` | | List of extra containers to include. | +| `extraEnv` | | List of extra environment variables to expose | +| `extraEnvFrom` | | List of extra environment variables from other data sources to expose | +| `init.containerSecurityContext` | | init container securityContext overrides | +| `init.containerSecurityContext.allowPrivilegeEscalation` | `false` | initContainer specific: Controls whether a process can gain more privileges than its parent process | +| `init.containerSecurityContext.runAsNonRoot` | `true` | initContainer specific: Controls whether the container runs with a non-root user | +| `init.containerSecurityContext.capabilities.drop` | `[ "ALL" ]` | initContainer specific: Removes [Linux capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) for the container | +| `image.repository` | `registry.gitlab.com/gitlab-org/build/cng/gitlab-kas` | Image repository. | +| `image.tag` | `v13.7.0` | Image tag. | +| `hpa.behavior` | `{scaleDown: {stabilizationWindowSeconds: 300 }}` | Behavior contains the specifications for up- and downscaling behavior (requires `autoscaling/v2beta2` or higher). | +| `hpa.customMetrics` | `[]` | Custom metrics contains the specifications for which to use to calculate the desired replica count (overrides the default use of Average CPU Utilization configured in `targetAverageUtilization`). | +| `hpa.cpu.targetType` | `AverageValue` | Set the autoscaling CPU target type, must be either `Utilization` or `AverageValue`. | +| `hpa.cpu.targetAverageValue` | `100m` | Set the autoscaling CPU target value. | +| `hpa.cpu.targetAverageUtilization` | | Set the autoscaling CPU target utilization. | +| `hpa.memory.targetType` | | Set the autoscaling memory target type, must be either `Utilization` or `AverageValue`. | +| `hpa.memory.targetAverageValue` | | Set the autoscaling memory target value. | +| `hpa.memory.targetAverageUtilization` | | Set the autoscaling memory target utilization. | +| `hpa.targetAverageValue` | | **DEPRECATED** Set the autoscaling CPU target value | +| `ingress.enabled` | `true` if `global.kas.enabled=true` | You can use `kas.ingress.enabled` to explicitly turn it on or off. If not set, you can optionally use `global.ingress.enabled` for the same purpose. | +| `ingress.apiVersion` | | Value to use in the `apiVersion` field. | +| `ingress.annotations` | `{}` | Ingress annotations. | +| `ingress.tls` | `{}` | Ingress TLS configuration. | +| `ingress.agentPath` | `/` | Ingress path for the agent API endpoint. | +| `ingress.k8sApiPath` | `/k8s-proxy` | Ingress path for Kubernetes API endpoint. | +| `keda.enabled` | `false` | Use [KEDA](https://keda.sh/) `ScaledObjects` instead of `HorizontalPodAutoscalers` | +| `keda.pollingInterval` | `30` | The interval to check each trigger on | +| `keda.cooldownPeriod` | `300` | The period to wait after the last trigger reported active before scaling the resource back to 0 | +| `keda.minReplicaCount` | | Minimum number of replicas KEDA will scale the resource down to, defaults to `minReplicas` | +| `keda.maxReplicaCount` | | Maximum number of replicas KEDA will scale the resource up to, defaults to `maxReplicas` | +| `keda.fallback` | | KEDA fallback configuration, see the [documentation](https://keda.sh/docs/2.10/concepts/scaling-deployments/#fallback) | +| `keda.hpaName` | | The name of the HPA resource KEDA will create, defaults to `keda-hpa-{scaled-object-name}` | +| `keda.restoreToOriginalReplicaCount` | | Specifies whether the target resource should be scaled back to original replicas count after the `ScaledObject` is deleted | +| `keda.behavior` | | The specifications for up- and downscaling behavior, defaults to `hpa.behavior` | +| `keda.triggers` | | List of triggers to activate scaling of the target resource, defaults to triggers computed from `hpa.cpu` and `hpa.memory` | +| `metrics.enabled` | `true` | If a metrics endpoint should be made available for scraping. | +| `metrics.path` | `/metrics` | Metrics endpoint path. | +| `metrics.serviceMonitor.enabled` | `false` | If a ServiceMonitor should be created to enable Prometheus Operator to manage the metrics scraping. Enabling removes the `prometheus.io` scrape annotations. It cannot be enabled together with `metrics.podMonitor.enabled`. | +| `metrics.serviceMonitor.additionalLabels` | `{}` | Additional labels to add to the ServiceMonitor. | +| `metrics.serviceMonitor.endpointConfig` | `{}` | Additional endpoint configuration for the ServiceMonitor. | +| `metrics.podMonitor.enabled` | `false` | If a PodMonitor should be created to enable Prometheus Operator to manage the metrics scraping. Enabling removes the `prometheus.io` scrape annotations. It cannot be enabled together with `metrics.serviceMonitor.enabled`. | +| `metrics.podMonitor.additionalLabels` | `{}` | Additional labels to add to the PodMonitor. | +| `metrics.podMonitor.endpointConfig` | `{}` | Additional endpoint configuration for the PodMonitor. | +| `maxReplicas` | `10` | HPA `maxReplicas`. | +| `maxUnavailable` | `1` | HPA `maxUnavailable`. | +| `minReplicas` | `2` | HPA `maxReplicas`. | +| `nodeSelector` | | Define a [nodeSelector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for the `Pod`s of this `Deployment`, if present. | +| `observability.port` | `8151` | Observability endpoint port. Used for metrics and probe endpoints. | +| `observability.livenessProbe.path` | `/liveness` | URI for the liveness probe endpoint. This value has to match the `observability.liveness_probe.url_path` value from the `kas` service configuration. | +| `observability.readinessProbe.path` | `/readiness` | URI for the readiness probe endpoint. This value has to match the `observability.readiness_probe.url_path` value from the `kas` service configuration. | +| `serviceAccount.annotations` | `{}` | Service account annotations. | +| `podLabels` | `{}` | Supplemental Pod labels. Not used for selectors. | +| `serviceLabels` | `{}` | Supplemental service labels. | +| `common.labels` | | Supplemental labels that are applied to all objects created by this chart. | +| `redis.enabled` | `true` | Allows opting-out of using Redis for `kas` features. Warnings: Redis will become a hard dependency soon, so this key is already deprecated. | +| `resources.requests.cpu` | `75m` | GitLab Exporter minimum CPU. | +| `resources.requests.memory` | `100M` | GitLab Exporter minimum memory. | +| `service.externalPort` | `8150` | External port (for `agentk` connections). | +| `service.internalPort` | `8150` | Internal port (for `agentk` connections). | +| `service.apiInternalPort` | `8153` | Internal port for the internal API (for GitLab backend). | +| `service.loadBalancerIP` | `nil` | A custom load balancer IP when `service.type` is `LoadBalancer`. | +| `service.loadBalancerSourceRanges` | `nil` | A list of custom load balancer source ranges when `service.type` is `LoadBalancer`. | +| `service.kubernetesApiPort` | `8154` | External port to expose proxied Kubernetes API on. | +| `service.privateApiPort` | `8155` | Internal port to expose `kas`' private API on (for `kas` -> `kas` communication). | +| `serviceAccount.annotations` | `{}` | ServiceAccount annotations. | +| `serviceAccount.automountServiceAccountToken` | `false` | Indicates whether or not the default ServiceAccount access token should be mounted in pods. | +| `serviceAccount.create` | `false` | Indicates whether or not a ServiceAccount should be created. | +| `serviceAccount.enabled` | `false` | Indicates whether or not to use a ServiceAccount. | +| `serviceAccount.name` | | Name of the ServiceAccount. If not set, the full chart name is used. | +| `privateApi.secret` | Autogenerated | The name of the secret to use for authenticating with the database. | +| `privateApi.key` | Autogenerated | The name of the key in `privateApi.secret` to use. | +| `global.kas.service.apiExternalPort` | `8153` | External port for the internal API (for GitLab backend). | +| `service.type` | `ClusterIP` | Service type. | +| `tolerations` | `[]` | Toleration labels for pod assignment. | +| `customConfig` | `{}` | When given, merges the default `kas` configuration with these values giving precedence to those defined here. | +| `deployment.minReadySeconds` | `0` | Minimum number of seconds that must pass before a `kas` pod is considered ready. | +| `deployment.strategy` | `rollingUpdate{}` | Allows one to configure the update strategy utilized by the deployment. | | `deployment.terminationGracePeriodSeconds` | `7260` | How much time in seconds a Pod is allowed to spend shutting down after receiving SIGTERM. It needs to be greater than the [maximum connection age](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/shutdown.md?ref_type=heads#maximum-connection-age) | -| `priorityClassName` | | [Priority class](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) assigned to pods. | +| `priorityClassName` | | [Priority class](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) assigned to pods. | ## Enable TLS communication -- GitLab