From 60d72f05f94aa26e5528fe57e598ea0afc5a370f Mon Sep 17 00:00:00 2001 From: Kurt Bomya Date: Sat, 11 Jun 2022 05:59:57 +0000 Subject: [PATCH 1/4] Add documentation for restoration automation --- doc/backup-restore/restore.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/backup-restore/restore.md b/doc/backup-restore/restore.md index df57966225..cbcf564892 100644 --- a/doc/backup-restore/restore.md +++ b/doc/backup-restore/restore.md @@ -160,6 +160,13 @@ The restoration process does not update the `gitlab-initial-root-password` secre ```shell /srv/gitlab/bin/rails runner "user = User.first; user.password='#{password}'; user.password_confirmation='#{password}'; user.save!" ``` + +## (Optional) Configure automated backups to non-production environments on a schedule + +You may automate periodic testing of backups to a non-production environment. The restoration process can be configured to execute on a cron schedule if the `backups.bucket` is also routinely updated with the latest backup files. + +1. Configure s3 bucket replication from your production `backups.bucket` to your non-production `backups.bucket`. +1. Set `restores.cron.enabled` to `true` ## Additional Information -- GitLab From 554c110850fede3ba3870f237096fe261a12789d Mon Sep 17 00:00:00 2001 From: Kurt Bomya Date: Sat, 11 Jun 2022 06:02:19 +0000 Subject: [PATCH 2/4] Add CronJob k8s object gitlab-toolbox restore automation --- .../charts/toolbox/templates/restore-job.yaml | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 charts/gitlab/charts/toolbox/templates/restore-job.yaml diff --git a/charts/gitlab/charts/toolbox/templates/restore-job.yaml b/charts/gitlab/charts/toolbox/templates/restore-job.yaml new file mode 100644 index 0000000000..043980716d --- /dev/null +++ b/charts/gitlab/charts/toolbox/templates/restore-job.yaml @@ -0,0 +1,217 @@ +{{- if and .Values.enabled .Values.restores.cron.enabled }} +{{- $imageCfg := dict "global" .Values.global.image "local" .Values.image -}} +{{- $initImageCfg := dict "global" .Values.global.busybox.image "local" .Values.init.image -}} +{{- include "database.datamodel.prepare" . -}} +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: {{ template "fullname" . }}-restore + namespace: {{ $.Release.Namespace }} + labels: + {{- include "gitlab.standardLabels" . | nindent 4 }} + {{- include "gitlab.commonLabels" . | nindent 4 }} +spec: + concurrencyPolicy: {{ .Values.restores.cron.concurrencyPolicy }} + failedJobsHistoryLimit: {{ .Values.restores.cron.failedJobsHistoryLimit }} + schedule: {{ .Values.restores.cron.schedule | quote }} + startingDeadlineSeconds: {{ .Values.restores.cron.startingDeadlineSeconds }} + successfulJobsHistoryLimit: {{ .Values.restores.cron.successfulJobsHistoryLimit }} + suspend: {{ .Values.restores.cron.suspend }} + jobTemplate: + spec: + backoffLimit: {{ .Values.restores.cron.backoffLimit }} + {{- if .Values.restores.cron.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ .Values.restores.cron.activeDeadlineSeconds }} + {{- end }} + template: + metadata: + labels: + {{- include "gitlab.standardLabels" . | nindent 12 }} + {{- include "gitlab.commonLabels" . | nindent 12 }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + cluster-autoscaler.kubernetes.io/safe-to-evict: "false" + {{- range $key, $value := .Values.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + spec: + restartPolicy: {{ .Values.restores.cron.restartPolicy }} + {{- if .Values.tolerations }} + tolerations: + {{- toYaml .Values.tolerations | nindent 12 }} + {{- end }} + securityContext: + runAsUser: {{ .Values.securityContext.runAsUser }} + fsGroup: {{ .Values.securityContext.fsGroup }} + {{- if or .Values.serviceAccount.enabled .Values.global.serviceAccount.enabled }} + serviceAccountName: {{ include "gitlab.serviceAccount.name" . }} + {{- end }} + initContainers: + {{- include "gitlab.extraInitContainers" . | nindent 12 }} + {{- include "gitlab.certificates.initContainer" . | nindent 12 }} + - name: configure + command: ['sh', '/config/configure'] + image: {{ include "gitlab.busybox.image" (dict "local" .Values.init "global" $.Values.global.busybox) | quote }} + {{- include "gitlab.image.pullPolicy" $initImageCfg | indent 14 }} + env: + {{- include "gitlab.extraEnv" $ | nindent 16 }} + {{- include "gitlab.extraEnvFrom" (dict "root" $ "local" (dict)) | nindent 16 }} + volumeMounts: + {{- include "gitlab.extraVolumeMounts" . | nindent 16 }} + {{- include "gitlab.psql.ssl.volumeMount" . | nindent 16 }} + - name: toolbox-config + mountPath: /config + readOnly: true + - name: init-toolbox-secrets + mountPath: /init-config + readOnly: true + - name: toolbox-secrets + mountPath: /init-secrets + readOnly: false + resources: + {{- toYaml .Values.init.resources | nindent 16 }} + {{- include "gitlab.image.pullSecrets" $imageCfg | indent 10 }} + containers: + {{- include "gitlab.extraContainers" . | nindent 12 }} + - name: {{ .Chart.Name }}-restore + args: + - /bin/bash + - -c + {{- if eq .Values.restores.objectStorage.backend "s3" }} + - export LATEST_RESTORE_FILE=$(aws s3 ls s3://{{ .Values.global.appConfig.backups.bucket }} {{ .Values.restores.cron.extraArgs }} | sort | tail -1 | awk '{ print $4 }' | sed 's/-ee.*$/-ee/g') + - backup-utility --restore -f $LATEST_RESTORE_FILE + {{- end }} + image: "{{ coalesce .Values.image.repository (include "image.repository" .) }}:{{ coalesce .Values.image.tag (include "gitlab.versionTag" . ) }}" + {{- include "gitlab.image.pullPolicy" $imageCfg | indent 14 }} + env: + {{- include "gitlab.extraEnv" $ | nindent 16 }} + {{- include "gitlab.extraEnvFrom" (dict "root" $ "local" (dict)) | nindent 16 }} + - name: ARTIFACTS_BUCKET_NAME + value: {{ .Values.global.appConfig.artifacts.bucket }} + - name: REGISTRY_BUCKET_NAME + value: {{ .Values.global.registry.bucket }} + - name: LFS_BUCKET_NAME + value: {{ .Values.global.appConfig.lfs.bucket }} + - name: UPLOADS_BUCKET_NAME + value: {{ .Values.global.appConfig.uploads.bucket }} + - name: PACKAGES_BUCKET_NAME + value: {{ .Values.global.appConfig.packages.bucket }} + - name: EXTERNAL_DIFFS_BUCKET_NAME + value: {{ .Values.global.appConfig.externalDiffs.bucket }} + - name: TERRAFORM_STATE_BUCKET_NAME + value: {{ .Values.global.appConfig.terraformState.bucket }} + - name: CI_SECURE_FILES_BUCKET_NAME + value: {{ .Values.global.appConfig.ciSecureFiles.bucket }} + - name: BACKUP_BUCKET_NAME + value: {{ .Values.global.appConfig.backups.bucket }} + - name: BACKUP_BACKEND + value: {{ .Values.restores.objectStorage.backend }} + - name: TMP_BUCKET_NAME + value: {{ .Values.global.appConfig.backups.tmpBucket }} + - name: PAGES_BUCKET_NAME + value: {{ .Values.global.pages.objectStore.bucket }} + - name: GITALY_FEATURE_DEFAULT_ON + value: "1" + - name: CONFIG_TEMPLATE_DIRECTORY + value: '/var/opt/gitlab/templates' + - name: CONFIG_DIRECTORY + value: '/srv/gitlab/config' + {{- if eq .Values.restores.objectStorage.backend "gcs" }} + - name: GOOGLE_APPLICATION_CREDENTIALS + value: '/etc/gitlab/objectstorage/{{ default "config" .Values.restores.objectStorage.config.key }}' + {{- end }} + volumeMounts: + {{- include "gitlab.extraVolumeMounts" . | nindent 16 }} + - name: toolbox-config + mountPath: '/var/opt/gitlab/templates' + - name: toolbox-secrets + mountPath: '/etc/gitlab' + readOnly: true + - name: toolbox-secrets + mountPath: /srv/gitlab/config/secrets.yml + subPath: rails-secrets/secrets.yml + - name: toolbox-tmp + mountPath: '/srv/gitlab/tmp' + {{- if and .Values.restores.cron.persistence.enabled .Values.restores.cron.persistence.subPath }} + subPath: "{{ .Values.restores.cron.persistence.subPath }}" + {{- end }} + readOnly: false + {{- include "gitlab.certificates.volumeMount" . | nindent 16 }} + resources: + {{- toYaml .Values.restores.cron.resources | nindent 16 }} + volumes: + {{- include "gitlab.extraVolumes" . | nindent 12 }} + {{- include "gitlab.psql.ssl.volume" . | nindent 12 }} + - name: toolbox-config + projected: + sources: + - configMap: + name: {{ template "fullname" . }} + - name: toolbox-tmp + {{- if .Values.restores.cron.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ template "fullname" . }}-restore-tmp + {{- else }} + emptyDir: {} + {{- end }} + - name: init-toolbox-secrets + projected: + defaultMode: 0400 + sources: + - secret: + name: {{ template "gitlab.rails-secrets.secret" . }} + items: + - key: secrets.yml + path: rails-secrets/secrets.yml + - secret: + name: {{ template "gitlab.gitlab-shell.authToken.secret" . }} + items: + - key: {{ template "gitlab.gitlab-shell.authToken.key" . }} + path: shell/.gitlab_shell_secret + {{- include "gitlab.gitaly.clientSecrets" . | nindent 16 }} + {{- include "gitlab.redis.secrets" . | nindent 16 }} + {{- range $.Values.local.psql }} + {{- include "gitlab.psql.secret" . | nindent 16 }} + {{- end }} + - secret: + name: {{ template "gitlab.registry.certificate.secret" . }} + items: + - key: registry-auth.key + path: registry/gitlab-registry.key + {{- include "gitlab.registry.notificationSecret.mount" $ | nindent 16 -}} + {{- if or .Values.restores.objectStorage.config (not .Values.global.minio.enabled) }} + - secret: + name: {{ required "A valid backups.objectStorage.config.secret is needed!" .Values.restores.objectStorage.config.secret }} + items: + - key: {{ default "config" .Values.restores.objectStorage.config.key }} + path: objectstorage/.s3cfg + {{- end }} + {{- if eq .Values.restores.objectStorage.backend "gcs" }} + - secret: + name: {{ required "A valid backups.objectStorage.config.secret is needed!" .Values.restores.objectStorage.config.secret }} + items: + - key: {{ default "config" .Values.restores.objectStorage.config.key }} + path: objectstorage/{{ default "config" .Values.restores.objectStorage.config.key }} + {{- end }} + {{- include "gitlab.kas.mountSecrets" $ | nindent 16 }} + {{- include "gitlab.pages.mountSecrets" $ | nindent 16 }} + {{- include "gitlab.minio.mountSecrets" $ | nindent 16 }} + {{- include "gitlab.appConfig.objectStorage.mountSecrets" (dict "name" "object_store" "config" $.Values.global.appConfig.object_store) | nindent 16 }} + {{- include "gitlab.appConfig.objectStorage.mountSecrets" (dict "name" "artifacts" "config" $.Values.global.appConfig.artifacts) | nindent 16 }} + {{- include "gitlab.appConfig.objectStorage.mountSecrets" (dict "name" "lfs" "config" $.Values.global.appConfig.lfs) | nindent 16 }} + {{- include "gitlab.appConfig.objectStorage.mountSecrets" (dict "name" "uploads" "config" $.Values.global.appConfig.uploads) | nindent 16 }} + {{- include "gitlab.appConfig.objectStorage.mountSecrets" (dict "name" "packages" "config" $.Values.global.appConfig.packages) | nindent 16 }} + {{- include "gitlab.appConfig.objectStorage.mountSecrets" (dict "name" "external_diffs" "config" $.Values.global.appConfig.externalDiffs) | nindent 16 }} + {{- include "gitlab.appConfig.objectStorage.mountSecrets" (dict "name" "terraform_state" "config" $.Values.global.appConfig.terraformState) | nindent 16 }} + {{- include "gitlab.appConfig.objectStorage.mountSecrets" (dict "name" "ci_secure_files" "config" $.Values.global.appConfig.ciSecureFiles) | nindent 16 }} + {{- include "gitlab.appConfig.objectStorage.mountSecrets" (dict "name" "dependency_proxy" "config" $.Values.global.appConfig.dependencyProxy) | nindent 16 }} + {{- include "gitlab.appConfig.objectStorage.mountSecrets" (dict "name" "pages" "config" $.Values.global.pages.objectStore) | nindent 16 }} + {{- include "gitlab.appConfig.ldap.servers.mountSecrets" $ | nindent 16 }} + {{- include "gitlab.appConfig.omniauth.mountSecrets" $ | nindent 16 }} + - name: toolbox-secrets + emptyDir: + medium: "Memory" + {{- include "gitlab.certificates.volumes" . | nindent 12 }} + {{- include "gitlab.nodeSelector" . | nindent 10 }} +{{- end }} + -- GitLab From 997e2c8876d295ad0748207d7fee7e99b8cb6914 Mon Sep 17 00:00:00 2001 From: Kurt Bomya Date: Sat, 11 Jun 2022 06:03:32 +0000 Subject: [PATCH 3/4] Update backup/restore values.yaml parent with skeleton config --- charts/gitlab/charts/toolbox/values.yaml | 43 ++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/charts/gitlab/charts/toolbox/values.yaml b/charts/gitlab/charts/toolbox/values.yaml index dfb4c7766a..dc667aa629 100644 --- a/charts/gitlab/charts/toolbox/values.yaml +++ b/charts/gitlab/charts/toolbox/values.yaml @@ -173,6 +173,49 @@ backups: # secret: my-backup-secret # key: config # gcpProject: my-gcp-project-id +restores: + cron: + enabled: false + concurrencyPolicy: Replace + failedJobsHistoryLimit: 1 + schedule: "0 1 * * *" + startingDeadlineSeconds: null + successfulJobsHistoryLimit: 3 + suspend: false + backoffLimit: 6 + # activeDeadlineSeconds: + restartPolicy: "OnFailure" + extraArgs: "" + resources: + # limits: + # cpu: 1 + # memory: 2G + requests: + cpu: 50m + memory: 350M + persistence: + enabled: false + ## toolbox temporarily Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + accessMode: ReadWriteOnce + size: 10Gi + subPath: "" + ## if volumeName is set, use this existing PersistentVolume + # volumeName: + matchLabels: {} + matchExpressions: [] + objectStorage: + backend: s3 + config: {} + # secret: my-backup-secret + # key: config + # gcpProject: my-gcp-project-id extra: {} rack_attack: -- GitLab From e7fe915f0cd755926d354ef8938c100cd295e95e Mon Sep 17 00:00:00 2001 From: Kurt Bomya Date: Wed, 15 Jun 2022 18:49:56 +0000 Subject: [PATCH 4/4] Stringing together restore in single command --- charts/gitlab/charts/toolbox/templates/restore-job.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/charts/gitlab/charts/toolbox/templates/restore-job.yaml b/charts/gitlab/charts/toolbox/templates/restore-job.yaml index 043980716d..1950c8d03f 100644 --- a/charts/gitlab/charts/toolbox/templates/restore-job.yaml +++ b/charts/gitlab/charts/toolbox/templates/restore-job.yaml @@ -78,8 +78,7 @@ spec: - /bin/bash - -c {{- if eq .Values.restores.objectStorage.backend "s3" }} - - export LATEST_RESTORE_FILE=$(aws s3 ls s3://{{ .Values.global.appConfig.backups.bucket }} {{ .Values.restores.cron.extraArgs }} | sort | tail -1 | awk '{ print $4 }' | sed 's/-ee.*$/-ee/g') - - backup-utility --restore -f $LATEST_RESTORE_FILE + - aws s3 ls s3://{{ .Values.global.appConfig.backups.bucket }} {{ .Values.restores.cron.extraArgs }} | sort | tail -1 | awk '{ print $4 }' | sed 's/-ee.*$/-ee/g' | xargs backup-utility --restore -f {{- end }} image: "{{ coalesce .Values.image.repository (include "image.repository" .) }}:{{ coalesce .Values.image.tag (include "gitlab.versionTag" . ) }}" {{- include "gitlab.image.pullPolicy" $imageCfg | indent 14 }} -- GitLab