From 409bdd3ded28f34b341640b6b0afd8a0d1fbb06d Mon Sep 17 00:00:00 2001 From: Hossein Pursultani Date: Fri, 25 Mar 2022 13:23:48 +1100 Subject: [PATCH] Pass kubectl options to shared secret Jobs Allows passing kubectl command-line options to shared secret Jobs via `shared-secrets.kubectl.options` Chart value. This value is passed as `KUBECTL_OPTS` to shared secret containers, where the scripts use it as additional command-line options. Note that this only works for generic command line options. See: https://kubernetes.io/docs/reference/kubectl/kubectl/#options By default, the GitLab Chart uses `--cache-dir /tmp/.kube/cache` to ensure when `kubectl` is run as `nobody` user it can create a cache directory. This will help to alleviate problems similar to #3201 where shared secret Jobs slow down. Changelog: performance --- .../shared-secrets/_generate_secrets.sh.tpl | 22 ++++++++++--------- templates/shared-secrets/job.yaml | 3 +++ .../shared-secrets/self-signed-cert-job.yml | 21 ++++++++++-------- values.yaml | 2 ++ 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/templates/shared-secrets/_generate_secrets.sh.tpl b/templates/shared-secrets/_generate_secrets.sh.tpl index 0cc079b6a6..09e857eb19 100644 --- a/templates/shared-secrets/_generate_secrets.sh.tpl +++ b/templates/shared-secrets/_generate_secrets.sh.tpl @@ -4,6 +4,8 @@ namespace={{ .Release.Namespace }} release={{ .Release.Name }} env={{ index .Values "shared-secrets" "env" }} +KUBECTL="kubectl ${KUBECTL_OPTS} --namespace=$namespace" + pushd $(mktemp -d) # Args pattern, length @@ -24,10 +26,10 @@ function label_secret(){ local secret_name=$1 {{ if not .Values.global.application.create -}} # Remove application labels if they exist - kubectl --namespace=$namespace label \ + $KUBECTL label \ secret $secret_name $(echo '{{ include "gitlab.application.labels" . | replace ": " "=" | replace "\r\n" " " | replace "\n" " " }}' | sed -E 's/=[^ ]*/-/g') {{ end }} - kubectl --namespace=$namespace label \ + $KUBECTL label \ --overwrite \ secret $secret_name {{ include "gitlab.standardLabels" . | replace ": " "=" | replace "\r\n" " " | replace "\n" " " }} {{ include "gitlab.commonLabels" . | replace ": " "=" | replace "\r\n" " " | replace "\n" " " }} } @@ -37,8 +39,8 @@ function generate_secret_if_needed(){ local secret_args=( "${@:2}") local secret_name=$1 - if ! $(kubectl --namespace=$namespace get secret $secret_name > /dev/null 2>&1); then - kubectl --namespace=$namespace create secret generic $secret_name ${secret_args[@]} + if ! $($KUBECTL get secret $secret_name > /dev/null 2>&1); then + $KUBECTL create secret generic $secret_name ${secret_args[@]} else echo "secret \"$secret_name\" already exists." @@ -48,16 +50,16 @@ function generate_secret_if_needed(){ if [ -z "${from##*literal*}" ]; then local key=$(echo -n ${arg} | cut -d '=' -f2) local desiredValue=$(echo -n ${arg} | cut -d '=' -f3-) - local flags="--namespace=$namespace --allow-missing-template-keys=false" + local flags="--allow-missing-template-keys=false" - if ! $(kubectl $flags get secret $secret_name -ojsonpath="{.data.${key}}" > /dev/null 2>&1); then + if ! $($KUBECTL $flags get secret $secret_name -ojsonpath="{.data.${key}}" > /dev/null 2>&1); then echo "key \"${key}\" does not exist. patching it in." if [ "${desiredValue}" != "" ]; then desiredValue=$(echo -n "${desiredValue}" | base64 -w 0) fi - kubectl --namespace=$namespace patch secret ${secret_name} -p "{\"data\":{\"$key\":\"${desiredValue}\"}}" + $KUBECTL patch secret ${secret_name} -p "{\"data\":{\"$key\":\"${desiredValue}\"}}" fi fi done @@ -136,8 +138,8 @@ if [ -n "$env" ]; then rails_secret={{ template "gitlab.rails-secrets.secret" . }} # Fetch the values from the existing secret if it exists - if $(kubectl --namespace=$namespace get secret $rails_secret > /dev/null 2>&1); then - kubectl --namespace=$namespace get secret $rails_secret -o jsonpath="{.data.secrets\.yml}" | base64 --decode > secrets.yml + if $($KUBECTL get secret $rails_secret > /dev/null 2>&1); then + $KUBECTL get secret $rails_secret -o jsonpath="{.data.secrets\.yml}" | base64 --decode > secrets.yml secret_key_base=$(fetch_rails_value secrets.yml "${env}.secret_key_base") otp_key_base=$(fetch_rails_value secrets.yml "${env}.otp_key_base") db_key_base=$(fetch_rails_value secrets.yml "${env}.db_key_base") @@ -173,7 +175,7 @@ $(echo "${openid_connect_signing_key}" | awk '{print " " $0}') ci_jwt_signing_key: | $(echo "${ci_jwt_signing_key}" | awk '{print " " $0}') EOF - kubectl --validate=false --namespace=$namespace apply -f rails-secrets.yml + $KUBECTL --validate=false apply -f rails-secrets.yml label_secret $rails_secret fi diff --git a/templates/shared-secrets/job.yaml b/templates/shared-secrets/job.yaml index 2768b0146e..7250019346 100644 --- a/templates/shared-secrets/job.yaml +++ b/templates/shared-secrets/job.yaml @@ -42,6 +42,9 @@ spec: image: {{ include "gitlab.kubectl.image" . | quote }} {{- include "gitlab.image.pullPolicy" $imageCfg | indent 10 }} command: ['/bin/bash', '/scripts/generate-secrets'] + env: + - name: KUBECTL_OPTS + value: "{{ $sharedSecretValues.kubectl.options }}" volumeMounts: - name: scripts mountPath: /scripts diff --git a/templates/shared-secrets/self-signed-cert-job.yml b/templates/shared-secrets/self-signed-cert-job.yml index d23f613de2..d98dbb8bda 100644 --- a/templates/shared-secrets/self-signed-cert-job.yml +++ b/templates/shared-secrets/self-signed-cert-job.yml @@ -76,33 +76,36 @@ spec: - | certname={{ template "gitlab.wildcard-self-signed-cert-name" . }} # create wildcard certificate secret - kubectl create secret tls $certname \ + kubectl ${KUBECTL_OPTS} create secret tls $certname \ --cert=/output/wildcard.pem --key=/output/wildcard-key.pem || true {{- if not .Values.global.application.create }} - kubectl --namespace=$namespace label \ + kubectl ${KUBECTL_OPTS} --namespace=$namespace label \ secret $certname $(echo '{{ include "gitlab.application.labels" . | replace ": " "=" | replace "\r\n" " " | replace "\n" " " }}' | sed -E 's/=[^ ]*/-/g') {{- end }} - kubectl --namespace=$namespace label --overwrite \ + kubectl ${KUBECTL_OPTS} --namespace=$namespace label --overwrite \ secret $certname {{ include "gitlab.standardLabels" . | replace ": " "=" | replace "\r\n" " " | replace "\n" " " }} {{ include "gitlab.commonLabels" . | replace ": " "=" | replace "\r\n" " " | replace "\n" " " }} # create CA certificate secret - kubectl create secret generic ${certname}-ca \ + kubectl ${KUBECTL_OPTS} create secret generic ${certname}-ca \ --from-file=cfssl_ca=/output/ca.pem || true {{- if not .Values.global.application.create }} - kubectl --namespace=$namespace label \ + kubectl ${KUBECTL_OPTS} --namespace=$namespace label \ secret ${certname}-ca $(echo '{{ include "gitlab.application.labels" . | replace ": " "=" | replace "\r\n" " " | replace "\n" " " }}' | sed -E 's/=[^ ]*/-/g') {{- end }} - kubectl --namespace=$namespace label --overwrite \ + kubectl ${KUBECTL_OPTS} --namespace=$namespace label --overwrite \ secret ${certname}-ca {{ include "gitlab.standardLabels" . | replace ": " "=" | replace "\r\n" " " | replace "\n" " " }} {{ include "gitlab.commonLabels" . | replace ": " "=" | replace "\r\n" " " | replace "\n" " " }} # create certificate chain for GitLab Runner cat /output/ca.pem /output/wildcard.pem > /tmp/{{ template "gitlab.gitlab.hostname" $ }}.crt - kubectl create secret generic ${certname}-chain \ + kubectl ${KUBECTL_OPTS} create secret generic ${certname}-chain \ --from-file=/tmp/{{ template "gitlab.gitlab.hostname" $ }}.crt || true {{- if not .Values.global.application.create }} - kubectl --namespace=$namespace label \ + kubectl ${KUBECTL_OPTS} --namespace=$namespace label \ secret ${certname}-chain $(echo '{{ include "gitlab.application.labels" . | replace ": " "=" | replace "\r\n" " " | replace "\n" " " }}' | sed -E 's/=[^ ]*/-/g') {{- end }} - kubectl --namespace=$namespace label --overwrite \ + kubectl ${KUBECTL_OPTS} --namespace=$namespace label --overwrite \ secret ${certname}-chain {{ include "gitlab.standardLabels" . | replace ": " "=" | replace "\r\n" " " | replace "\n" " " }} {{ include "gitlab.commonLabels" . | replace ": " "=" | replace "\r\n" " " | replace "\n" " " }} + env: + - name: KUBECTL_OPTS + value: "{{ $sharedSecretValues.kubectl.options }}" volumeMounts: - name: certs-path mountPath: /output diff --git a/values.yaml b/values.yaml index ad09ae4567..1e6668f06f 100644 --- a/values.yaml +++ b/values.yaml @@ -1018,6 +1018,8 @@ shared-secrets: tolerations: [] podLabels: {} annotations: {} + kubectl: + options: '--cache-dir /tmp/.kube/cache' ## Installation & configuration of gitlab/gitlab-runner ## See requirements.yaml for current version -- GitLab