From f742f4ccb78317336893d50aab9e7fe77a309aea Mon Sep 17 00:00:00 2001
From: Gerard Hickey <ghickey@gitlab.com>
Date: Wed, 21 Sep 2022 18:17:02 -0700
Subject: [PATCH 01/42] Added azcopy support to backup_utility script

Signed-off-by: Gerard Hickey <ghickey@gitlab.com>
---
 gitlab-toolbox/scripts/bin/backup-utility | 25 +++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index 759078a7f..cd4d5d9f2 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -78,6 +78,7 @@ function fetch_remote_backup(){
         esac
         ;;
       gcs) gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null ;;
+      azure) azcopy copy "${BACKUP_BUCKET_NAME}/${file_name}?$(get_azure_token)" ${output_path} > /dev/null ;;
       *) echo "Unknown backend: ${BACKUP_BACKEND}" ;;
     esac
   fi
@@ -108,6 +109,15 @@ function get_version(){
   cat $rails_dir/VERSION
 }
 
+function get_azure_token(){
+  if [ ! -e '/etc/gitlab/objectstorage/azure_token' ]; then
+    echo "Azure SAS token not found in /etc/gitlab/objectstorage/azure_token"
+    exit 1
+  fi
+
+  cat /etc/gitlab/objectstorage/azure_token
+}
+
 function get_backup_name(){
   if [ -n "$BACKUP_TIMESTAMP" ]; then
     echo ${BACKUP_TIMESTAMP}_gitlab_backup
@@ -141,6 +151,9 @@ function get_existing_backups(){
       # https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames#other-wildcard-characters
       existing_backups=($(gsutil ls gs://$BACKUP_BUCKET_NAME/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_[0-9][0-9][0-9][0-9]_[0-9][0-9]_[0-9][0-9]_\*_gitlab_backup.tar | LC_ALL=C sort))
       ;;
+    azure)
+      existing_backups=($(azcopy list "${BACKUP_BUCKET_NAME}?$(get_azure_token)" | awk '/^INFO: / {print $2}' | sed 's/;$//' | grep -E '^[0-9]{10}_[0-9]{4}_[0-9]{2}_[0-9]{2}_.+_gitlab_backup.tar$' | LC_ALL=C sort))
+      ;;
     *)
       echo "Unknown backend for backup: ${BACKUP_BACKEND}"
       exit 1
@@ -162,6 +175,7 @@ function remove_backup(){
       esac
       ;;
     gcs) gsutil rm ${backup_to_remove} > /dev/null ;;
+    azure) azcopy remove "${BACKUP_BUCKET_NAME}/${backup_to_remove}?$(get_azure_token)" > /dev/null ;;
     *)
       echo "Unknown backend for backup: ${BACKUP_BACKEND}"
       exit 1
@@ -269,6 +283,17 @@ function backup(){
       fi
       echo "[DONE] Backup can be found at gs://$BACKUP_BUCKET_NAME/${backup_name}.tar"
       ;;
+    azure)
+      if [ -z "${STORAGE_CLASS}" ]; then
+        azcopy copy --block-blob-tier "${STORAGE_CLASS}" \
+          "${backup_tars_path}/${backup_name}.tar" \
+          "${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
+      else
+        azcopy copy "${backup_tars_path}/${backup_name}.tar" \
+          "${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
+      fi
+      echo "[DONE] Backup can be found at ${BACKUP_BUCKET_NAME}/${backup_name}.tar"
+      ;;
     *) echo "Unknown backend for backup: ${BACKUP_BACKEND}" ;;
   esac
 
-- 
GitLab


From 74faf77977f2fe77d6a34bbe347dfbcddd102a8c Mon Sep 17 00:00:00 2001
From: Gerard Hickey <ghickey@gitlab.com>
Date: Mon, 31 Oct 2022 15:29:49 -0700
Subject: [PATCH 02/42] Refactored token path to a variable

Signed-off-by: Gerard Hickey <ghickey@gitlab.com>
---
 gitlab-toolbox/scripts/bin/backup-utility | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index cd4d5d9f2..02dea92f6 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -110,12 +110,13 @@ function get_version(){
 }
 
 function get_azure_token(){
-  if [ ! -e '/etc/gitlab/objectstorage/azure_token' ]; then
-    echo "Azure SAS token not found in /etc/gitlab/objectstorage/azure_token"
+  token_file='/etc/gitlab/objectstorage/azure_token'
+  if [ ! -e ${token_file} ]; then
+    echo "Azure SAS token not found in ${token_file}"
     exit 1
   fi
 
-  cat /etc/gitlab/objectstorage/azure_token
+  cat ${token_file}
 }
 
 function get_backup_name(){
-- 
GitLab


From 55ad296db300782a3a62308b81be36ca8d2167ab Mon Sep 17 00:00:00 2001
From: Gerard Hickey <ghickey@gitlab.com>
Date: Mon, 31 Oct 2022 17:02:05 -0700
Subject: [PATCH 03/42] Updated filtering of azcopy output to find backups

Signed-off-by: Gerard Hickey <ghickey@gitlab.com>
---
 gitlab-toolbox/scripts/bin/backup-utility | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index 02dea92f6..a4d5a87b2 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -153,7 +153,7 @@ function get_existing_backups(){
       existing_backups=($(gsutil ls gs://$BACKUP_BUCKET_NAME/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_[0-9][0-9][0-9][0-9]_[0-9][0-9]_[0-9][0-9]_\*_gitlab_backup.tar | LC_ALL=C sort))
       ;;
     azure)
-      existing_backups=($(azcopy list "${BACKUP_BUCKET_NAME}?$(get_azure_token)" | awk '/^INFO: / {print $2}' | sed 's/;$//' | grep -E '^[0-9]{10}_[0-9]{4}_[0-9]{2}_[0-9]{2}_.+_gitlab_backup.tar$' | LC_ALL=C sort))
+      existing_backups=($(azcopy list "${BACKUP_BUCKET_NAME}?$(get_azure_token)" | awk '/^INFO: [[:alnum:]\/\.]+;/ {print gensub(";$", "", 1, $2)}' | grep -E '^[0-9]{10}_[0-9]{4}_[0-9]{2}_[0-9]{2}_.+_gitlab_backup.tar$' | LC_ALL=C sort))
       ;;
     *)
       echo "Unknown backend for backup: ${BACKUP_BACKEND}"
-- 
GitLab


From 375ad884305f3d505b8989bf1bf6e85bc4c0ad52 Mon Sep 17 00:00:00 2001
From: Gerard Hickey <ghickey@gitlab.com>
Date: Fri, 4 Nov 2022 17:13:46 -0700
Subject: [PATCH 04/42] Added Azure support to object_storage_backup.rb

Signed-off-by: Gerard Hickey <ghickey@gitlab.com>
---
 gitlab-toolbox/scripts/lib/object_storage_backup.rb | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index 4a3243975..6e3c04177 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -39,6 +39,10 @@ class ObjectStorageBackup
     elsif @backend == "gcs"
       check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name})
       cmd = %W(gsutil -m rsync -d -x #{REGEXP_EXCLUDE} -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name})
+    elsif @backend == "azure"
+      azure_token = open('/etc/gitlab/objectstore/azure_token', 'r').read()
+      check_bucket_cmd = %W(azcopy list "#{@remote_bucket_name}?#{azure_token}")
+      cmd = %W(azcopy copy "#{@remote_bucket_name}?#{azure_token}" /srv/gitlab/tmp/#{@name})
     end
 
     # Check if the bucket exists
@@ -96,6 +100,9 @@ class ObjectStorageBackup
       end
     elsif @backend == "gcs"
       cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}/#{dir_name})
+    elsif @backend == "azure"
+      azure_token = open('/etc/gitlab/objectstore/azure_token', 'r').read()
+      cmd = %W(azcopy copy #{source_path} "#{@remote_bucket_name}?#{azure_token}")
     end
 
     output, status = run_cmd(cmd)
@@ -114,6 +121,9 @@ class ObjectStorageBackup
       end
     elsif @backend == "gcs"
       cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/)
+    elsif @backend == "azure"
+      azure_token = open('/etc/gitlab/objectstore/azure_token', 'r').read()
+      cmd = %W(azcopy copy "#{@remote_bucket_name}?#{azure_token}" "#{@tmp_bucket_name}/#{backup_file_name}?#{azure_token}")
     end
 
     output, status = run_cmd(cmd)
@@ -143,6 +153,9 @@ class ObjectStorageBackup
     end
     output, status = run_cmd(cmd)
     failure_abort('bucket cleanup', output) unless status.zero?
+  elsif @backend == 'azure'
+    azure_token = open('/etc/gitlab/objectstore/azure_token', 'r').read()
+    cmd = %W(azcopy remove --recursive=true "#{@remote_bucket_name}?#{azure_token}")
   end
 
   def restore_from_backup
-- 
GitLab


From 34c35205757c8c0193251b331ef8bca3dc3d34e9 Mon Sep 17 00:00:00 2001
From: Gerard Hickey <ghickey@gitlab.com>
Date: Tue, 10 Jan 2023 13:27:59 -0500
Subject: [PATCH 05/42] Fixed if statement in cleanup() routine

Signed-off-by: Gerard Hickey <ghickey@gitlab.com>
---
 gitlab-toolbox/scripts/lib/object_storage_backup.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index 6e3c04177..c724ffb06 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -150,12 +150,12 @@ class ObjectStorageBackup
       end
 
       cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*)
+    elsif @backend == 'azure'
+      azure_token = open('/etc/gitlab/objectstore/azure_token', 'r').read()
+      cmd = %W(azcopy remove --recursive=true "#{@remote_bucket_name}?#{azure_token}")
     end
     output, status = run_cmd(cmd)
     failure_abort('bucket cleanup', output) unless status.zero?
-  elsif @backend == 'azure'
-    azure_token = open('/etc/gitlab/objectstore/azure_token', 'r').read()
-    cmd = %W(azcopy remove --recursive=true "#{@remote_bucket_name}?#{azure_token}")
   end
 
   def restore_from_backup
-- 
GitLab


From 550aa2fefe630c54d3a2fe03d6b06e317a259f43 Mon Sep 17 00:00:00 2001
From: Gerard Hickey <ghickey@gitlab.com>
Date: Tue, 10 Jan 2023 15:17:37 -0500
Subject: [PATCH 06/42] Fixed path to Azure SAS token

Signed-off-by: Gerard Hickey <ghickey@gitlab.com>
---
 gitlab-toolbox/scripts/lib/object_storage_backup.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index c724ffb06..884da7bc0 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -151,7 +151,7 @@ class ObjectStorageBackup
 
       cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*)
     elsif @backend == 'azure'
-      azure_token = open('/etc/gitlab/objectstore/azure_token', 'r').read()
+      azure_token = open('/etc/gitlab/objectstorage/azure_token', 'r').read()
       cmd = %W(azcopy remove --recursive=true "#{@remote_bucket_name}?#{azure_token}")
     end
     output, status = run_cmd(cmd)
-- 
GitLab


From e3a5dc971e5f2508eab05fedc8668302831c3bdf Mon Sep 17 00:00:00 2001
From: Gerard Hickey <ghickey@gitlab.com>
Date: Wed, 11 Jan 2023 14:32:15 -0500
Subject: [PATCH 07/42] Refactored reading Azure token

Signed-off-by: Gerard Hickey <ghickey@gitlab.com>
---
 gitlab-toolbox/scripts/lib/object_storage_backup.rb | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index 884da7bc0..592db73df 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -40,7 +40,7 @@ class ObjectStorageBackup
       check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name})
       cmd = %W(gsutil -m rsync -d -x #{REGEXP_EXCLUDE} -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name})
     elsif @backend == "azure"
-      azure_token = open('/etc/gitlab/objectstore/azure_token', 'r').read()
+      azure_token = read_azure_token
       check_bucket_cmd = %W(azcopy list "#{@remote_bucket_name}?#{azure_token}")
       cmd = %W(azcopy copy "#{@remote_bucket_name}?#{azure_token}" /srv/gitlab/tmp/#{@name})
     end
@@ -101,7 +101,7 @@ class ObjectStorageBackup
     elsif @backend == "gcs"
       cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}/#{dir_name})
     elsif @backend == "azure"
-      azure_token = open('/etc/gitlab/objectstore/azure_token', 'r').read()
+      azure_token = open('/etc/gitlab/objectstoragage/azure_token', 'r').read()
       cmd = %W(azcopy copy #{source_path} "#{@remote_bucket_name}?#{azure_token}")
     end
 
@@ -122,7 +122,7 @@ class ObjectStorageBackup
     elsif @backend == "gcs"
       cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/)
     elsif @backend == "azure"
-      azure_token = open('/etc/gitlab/objectstore/azure_token', 'r').read()
+      azure_token = read_azure_token
       cmd = %W(azcopy copy "#{@remote_bucket_name}?#{azure_token}" "#{@tmp_bucket_name}/#{backup_file_name}?#{azure_token}")
     end
 
@@ -151,7 +151,7 @@ class ObjectStorageBackup
 
       cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*)
     elsif @backend == 'azure'
-      azure_token = open('/etc/gitlab/objectstorage/azure_token', 'r').read()
+      azure_token = read_azure_token
       cmd = %W(azcopy remove --recursive=true "#{@remote_bucket_name}?#{azure_token}")
     end
     output, status = run_cmd(cmd)
@@ -180,4 +180,7 @@ class ObjectStorageBackup
     return stdout.read, wait_thr.value.exitstatus
   end
 
+  def read_azure_token
+    return open('/etc/gitlab/objectstorage/azure_token', 'r').read()
+  end
 end
-- 
GitLab


From 752b2384931d977bb71ba92ac0087765e49fa8fe Mon Sep 17 00:00:00 2001
From: Gerard Hickey <ghickey@gitlab.com>
Date: Wed, 11 Jan 2023 18:16:37 -0500
Subject: [PATCH 08/42] Added diag output and sanitized check_bucket_cmd

Signed-off-by: Gerard Hickey <ghickey@gitlab.com>
---
 gitlab-toolbox/scripts/lib/object_storage_backup.rb | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index 592db73df..cd9950b51 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -36,12 +36,15 @@ class ObjectStorageBackup
         check_bucket_cmd = %W(aws s3api head-bucket --bucket #{@remote_bucket_name}) + @aws_s3_settings.split(" ")
         cmd = %W(aws s3 sync --delete --exclude #{GLOB_EXCLUDE} s3://#{@remote_bucket_name}/ /srv/gitlab/tmp/#{@name}/) + @aws_s3_settings.split(" ") + @aws_kms_settings.split(" ")
       end
+      sanitized_check_bucket_cmd = check_bucket_cmd
     elsif @backend == "gcs"
       check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name})
+      sanitized_check_bucket_cmd = check_bucket_cmd
       cmd = %W(gsutil -m rsync -d -x #{REGEXP_EXCLUDE} -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name})
     elsif @backend == "azure"
       azure_token = read_azure_token
       check_bucket_cmd = %W(azcopy list "#{@remote_bucket_name}?#{azure_token}")
+      sanitized_check_bucket_cmd = %W(azcopy list "#{@remote_bucket_name}?TOKEN")
       cmd = %W(azcopy copy "#{@remote_bucket_name}?#{azure_token}" /srv/gitlab/tmp/#{@name})
     end
 
@@ -49,6 +52,8 @@ class ObjectStorageBackup
     output, status = run_cmd(check_bucket_cmd)
     unless status.zero?
       puts "Bucket not found: #{@remote_bucket_name}. Skipping backup of #{@name} ...".blue
+      puts "Output from #{sanitized_check_bucket_cmd}:".blue
+      puts output.red
       return
     end
 
-- 
GitLab


From db95f5bd32991f1f231c82f7685b9273e23bd4ce Mon Sep 17 00:00:00 2001
From: Gerard Hickey <ghickey@gitlab.com>
Date: Thu, 12 Jan 2023 13:31:23 -0500
Subject: [PATCH 09/42] Updated object-storage-backup to include Azure
 container path

Signed-off-by: Gerard Hickey <ghickey@gitlab.com>
---
 gitlab-toolbox/scripts/bin/object-storage-backup | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/gitlab-toolbox/scripts/bin/object-storage-backup b/gitlab-toolbox/scripts/bin/object-storage-backup
index 0ff9d638a..78bb1a6bd 100755
--- a/gitlab-toolbox/scripts/bin/object-storage-backup
+++ b/gitlab-toolbox/scripts/bin/object-storage-backup
@@ -5,6 +5,9 @@ require 'object_storage_backup'
 abort ("backup_item and output_tar_path arguments needs to be passed to the script") unless ARGV.length == 2
 
 bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}"
+if ENV["BACKUP_BACKEND"] == "azure"
+  bucket_name = "#{ENV['BACKUP_BUCKET_NAME']}/#{bucket_name}"
+end
 tmp_bucket  = ENV['TMP_BUCKET_NAME'] || 'tmp'
 backend_type = ENV['BACKUP_BACKEND'] || 's3'
 s3_tool = ENV['S3_TOOL'] || 's3cmd'
-- 
GitLab


From 3d564d4666a0dc1783d701535b4d328ac353f408 Mon Sep 17 00:00:00 2001
From: Gerard Hickey <ghickey@gitlab.com>
Date: Thu, 12 Jan 2023 14:49:40 -0500
Subject: [PATCH 10/42] Removed quotes around remote bucket and token for Azure

Signed-off-by: Gerard Hickey <ghickey@gitlab.com>
---
 gitlab-toolbox/scripts/lib/object_storage_backup.rb | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index cd9950b51..989027ecc 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -43,8 +43,8 @@ class ObjectStorageBackup
       cmd = %W(gsutil -m rsync -d -x #{REGEXP_EXCLUDE} -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name})
     elsif @backend == "azure"
       azure_token = read_azure_token
-      check_bucket_cmd = %W(azcopy list "#{@remote_bucket_name}?#{azure_token}")
-      sanitized_check_bucket_cmd = %W(azcopy list "#{@remote_bucket_name}?TOKEN")
+      check_bucket_cmd = %W(azcopy list #{@remote_bucket_name}?#{azure_token})
+      sanitized_check_bucket_cmd = %W(azcopy list #{@remote_bucket_name}?TOKEN)
       cmd = %W(azcopy copy "#{@remote_bucket_name}?#{azure_token}" /srv/gitlab/tmp/#{@name})
     end
 
@@ -107,7 +107,7 @@ class ObjectStorageBackup
       cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}/#{dir_name})
     elsif @backend == "azure"
       azure_token = open('/etc/gitlab/objectstoragage/azure_token', 'r').read()
-      cmd = %W(azcopy copy #{source_path} "#{@remote_bucket_name}?#{azure_token}")
+      cmd = %W(azcopy copy #{source_path} #{@remote_bucket_name}?#{azure_token})
     end
 
     output, status = run_cmd(cmd)
@@ -128,7 +128,7 @@ class ObjectStorageBackup
       cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/)
     elsif @backend == "azure"
       azure_token = read_azure_token
-      cmd = %W(azcopy copy "#{@remote_bucket_name}?#{azure_token}" "#{@tmp_bucket_name}/#{backup_file_name}?#{azure_token}")
+      cmd = %W(azcopy copy #{@remote_bucket_name}?#{azure_token} #{@tmp_bucket_name}/#{backup_file_name}?#{azure_token})
     end
 
     output, status = run_cmd(cmd)
@@ -157,7 +157,7 @@ class ObjectStorageBackup
       cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*)
     elsif @backend == 'azure'
       azure_token = read_azure_token
-      cmd = %W(azcopy remove --recursive=true "#{@remote_bucket_name}?#{azure_token}")
+      cmd = %W(azcopy remove --recursive=true #{@remote_bucket_name}?#{azure_token})
     end
     output, status = run_cmd(cmd)
     failure_abort('bucket cleanup', output) unless status.zero?
-- 
GitLab


From 76566d9832a1f1d43a60be0313879976ee0c1341 Mon Sep 17 00:00:00 2001
From: Gerard Hickey <ghickey@gitlab.com>
Date: Thu, 12 Jan 2023 17:33:02 -0500
Subject: [PATCH 11/42] debug: checking copy of backup file

---
 gitlab-toolbox/scripts/bin/backup-utility | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index a4d5a87b2..f2f91b4ce 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -47,7 +47,7 @@ function usage()
                                             Requires s3config or AWS credentials to be able to list and delete objects.
      --cleanup                              Run the backup cleanup without creating a new backup. Can be used with the
                                             'maximum-backups' option to clean old remote backups.
-     --aws-kms-key-id                       Add KMS key id when S3 bucket is encrypted with a customer key. 
+     --aws-kms-key-id                       Add KMS key id when S3 bucket is encrypted with a customer key.
      --aws-s3-endpoint-url                  Specify an AWS S3 endpoint URL
      --aws-region                           Add AWS region (required for AWS STS regionalized endpoint)
 HEREDOC
@@ -290,6 +290,7 @@ function backup(){
           "${backup_tars_path}/${backup_name}.tar" \
           "${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
       else
+        echo "azcopy copy '${backup_tars_path}/${backup_name}.tar' '${BACKUP_BUCKET_NAME}/${backup_name}.tar?TOKEN'"
         azcopy copy "${backup_tars_path}/${backup_name}.tar" \
           "${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
       fi
@@ -442,12 +443,12 @@ do
       AWS_S3_SETTINGS_LIST+=(--endpoint-url $2)
       shift
       shift
-      ;; 
+      ;;
     --aws-region)
       AWS_REGION=$2
       shift
       shift
-      ;; 
+      ;;
     *)
       usage
       echo "Unexpected parameter: $key"
-- 
GitLab


From ea00f380a32fb4bb6cc5b69d5568bc5b8b56ae69 Mon Sep 17 00:00:00 2001
From: Gerard Hickey <ghickey@gitlab.com>
Date: Thu, 12 Jan 2023 17:33:02 -0500
Subject: [PATCH 12/42] debug: checking copy of backup file

---
 gitlab-toolbox/scripts/bin/backup-utility | 1 +
 1 file changed, 1 insertion(+)

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index f2f91b4ce..c5d94d266 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -286,6 +286,7 @@ function backup(){
       ;;
     azure)
       if [ -z "${STORAGE_CLASS}" ]; then
+        echo "azcopy copy --block-blob-tier '${STORAGE_CLASS}'' '${backup_tars_path}/${backup_name}.tar' '${BACKUP_BUCKET_NAME}/${backup_name}.tar?TOKEN'"
         azcopy copy --block-blob-tier "${STORAGE_CLASS}" \
           "${backup_tars_path}/${backup_name}.tar" \
           "${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
-- 
GitLab


From 79e63c8c02d3f78bbb53f5728053cab6a6ffe749 Mon Sep 17 00:00:00 2001
From: Gerard Hickey <ghickey@gitlab.com>
Date: Thu, 12 Jan 2023 23:03:48 -0500
Subject: [PATCH 13/42] Reversed order of Azure parameters

---
 gitlab-toolbox/scripts/lib/object_storage_backup.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index 989027ecc..d3f73288e 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -45,7 +45,7 @@ class ObjectStorageBackup
       azure_token = read_azure_token
       check_bucket_cmd = %W(azcopy list #{@remote_bucket_name}?#{azure_token})
       sanitized_check_bucket_cmd = %W(azcopy list #{@remote_bucket_name}?TOKEN)
-      cmd = %W(azcopy copy "#{@remote_bucket_name}?#{azure_token}" /srv/gitlab/tmp/#{@name})
+      cmd = %W(azcopy copy /srv/gitlab/tmp/#{@name} "#{@remote_bucket_name}?#{azure_token}")
     end
 
     # Check if the bucket exists
-- 
GitLab


From 6adc635083debbfaa42ceb56503970e45da907a6 Mon Sep 17 00:00:00 2001
From: Gerard Hickey <ghickey@gitlab.com>
Date: Tue, 17 Jan 2023 18:00:14 -0500
Subject: [PATCH 14/42] debug: statements around check_backup_cmd

---
 gitlab-toolbox/scripts/lib/object_storage_backup.rb | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index d3f73288e..728ca818f 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -49,7 +49,12 @@ class ObjectStorageBackup
     end
 
     # Check if the bucket exists
+    ## DEBUG
+    puts "running command: #{sanitized_check_bucket_cmd}"
     output, status = run_cmd(check_bucket_cmd)
+    puts "status = #{status}"
+    puts "output = #{output}"
+
     unless status.zero?
       puts "Bucket not found: #{@remote_bucket_name}. Skipping backup of #{@name} ...".blue
       puts "Output from #{sanitized_check_bucket_cmd}:".blue
-- 
GitLab


From adc010326bbb09c0ec884a4558688e002f75643e Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Tue, 14 Feb 2023 18:36:11 +0100
Subject: [PATCH 15/42] Add support for Azure endpoint base URL

---
 gitlab-toolbox/scripts/bin/backup-utility | 28 +++++++++++++----------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index c5d94d266..29a5b4cfb 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -36,7 +36,7 @@ function usage()
                                             May be defined multiple times. Valid values for COMPONENT are
                                             db, repositories, and any of the object storages (e.g. 'lfs').
      --backend BACKEND                      Object storage backend to use for backups.
-                                            Can be either 's3' or 'gcs'.
+                                            Can be either 's3', 'gcs' or 'azure'.
      --s3config CONFIG                      S3 backend configuration to use for backups storage.
                                             Special config file for s3cmd (see: https://s3tools.org/usage)
                                             Not required when using the awscli tool.
@@ -50,6 +50,7 @@ function usage()
      --aws-kms-key-id                       Add KMS key id when S3 bucket is encrypted with a customer key.
      --aws-s3-endpoint-url                  Specify an AWS S3 endpoint URL
      --aws-region                           Add AWS region (required for AWS STS regionalized endpoint)
+     --azure-endpoint-url                   Specify the Azure endpoint, e.g. https://YOUR_STORAGE_ACCOUNT.blob.core.windows.net
 HEREDOC
 }
 
@@ -78,7 +79,7 @@ function fetch_remote_backup(){
         esac
         ;;
       gcs) gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null ;;
-      azure) azcopy copy "${BACKUP_BUCKET_NAME}/${file_name}?$(get_azure_token)" ${output_path} > /dev/null ;;
+      azure) azcopy copy "${AZURE_ENDPOINT_URL}/${BACKUP_BUCKET_NAME}/${file_name}?$(get_azure_token)" ${output_path} > /dev/null ;;
       *) echo "Unknown backend: ${BACKUP_BACKEND}" ;;
     esac
   fi
@@ -153,7 +154,7 @@ function get_existing_backups(){
       existing_backups=($(gsutil ls gs://$BACKUP_BUCKET_NAME/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_[0-9][0-9][0-9][0-9]_[0-9][0-9]_[0-9][0-9]_\*_gitlab_backup.tar | LC_ALL=C sort))
       ;;
     azure)
-      existing_backups=($(azcopy list "${BACKUP_BUCKET_NAME}?$(get_azure_token)" | awk '/^INFO: [[:alnum:]\/\.]+;/ {print gensub(";$", "", 1, $2)}' | grep -E '^[0-9]{10}_[0-9]{4}_[0-9]{2}_[0-9]{2}_.+_gitlab_backup.tar$' | LC_ALL=C sort))
+      existing_backups=($(azcopy list "${AZURE_ENDPOINT_URL}/${BACKUP_BUCKET_NAME}?$(get_azure_token)" | awk '/^INFO: [[:alnum:]\/\.]+;/ {print gensub(";$", "", 1, $2)}' | grep -E '^[0-9]{10}_[0-9]{4}_[0-9]{2}_[0-9]{2}_.+_gitlab_backup.tar$' | LC_ALL=C sort))
       ;;
     *)
       echo "Unknown backend for backup: ${BACKUP_BACKEND}"
@@ -176,7 +177,7 @@ function remove_backup(){
       esac
       ;;
     gcs) gsutil rm ${backup_to_remove} > /dev/null ;;
-    azure) azcopy remove "${BACKUP_BUCKET_NAME}/${backup_to_remove}?$(get_azure_token)" > /dev/null ;;
+    azure) azcopy remove "${AZURE_ENDPOINT_URL}/${BACKUP_BUCKET_NAME}/${backup_to_remove}?$(get_azure_token)" > /dev/null ;;
     *)
       echo "Unknown backend for backup: ${BACKUP_BACKEND}"
       exit 1
@@ -286,16 +287,14 @@ function backup(){
       ;;
     azure)
       if [ -z "${STORAGE_CLASS}" ]; then
-        echo "azcopy copy --block-blob-tier '${STORAGE_CLASS}'' '${backup_tars_path}/${backup_name}.tar' '${BACKUP_BUCKET_NAME}/${backup_name}.tar?TOKEN'"
-        azcopy copy --block-blob-tier "${STORAGE_CLASS}" \
-          "${backup_tars_path}/${backup_name}.tar" \
-          "${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
+        echo "azcopy copy '${backup_tars_path}/${backup_name}.tar' '${AZURE_ENDPOINT_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?TOKEN'"
+        azcopy copy "${backup_tars_path}/${backup_name}.tar" "${AZURE_ENDPOINT_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
       else
-        echo "azcopy copy '${backup_tars_path}/${backup_name}.tar' '${BACKUP_BUCKET_NAME}/${backup_name}.tar?TOKEN'"
-        azcopy copy "${backup_tars_path}/${backup_name}.tar" \
-          "${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
+        echo "azcopy copy --block-blob-tier '${STORAGE_CLASS}' '${backup_tars_path}/${backup_name}.tar' '${AZURE_ENDPOINT_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar??TOKEN'" 
+        azcopy copy --block-blob-tier "${STORAGE_CLASS}" \
+          "${backup_tars_path}/${backup_name}.tar" "${AZURE_ENDPOINT_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
       fi
-      echo "[DONE] Backup can be found at ${BACKUP_BUCKET_NAME}/${backup_name}.tar"
+      echo "[DONE] Backup can be found at ${AZURE_ENDPOINT_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar"
       ;;
     *) echo "Unknown backend for backup: ${BACKUP_BACKEND}" ;;
   esac
@@ -450,6 +449,11 @@ do
       shift
       shift
       ;;
+    --azure-endpoint-url)
+      AZURE_ENDPOINT_URL=$2
+      shift
+      shift
+      ;;
     *)
       usage
       echo "Unexpected parameter: $key"
-- 
GitLab


From a790a2fff237bc3c40dcddb809e0ba6db0f07b8a Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Tue, 14 Feb 2023 23:58:24 +0100
Subject: [PATCH 16/42] Pass Azure Base URL to ObjectStorageBackup

---
 gitlab-toolbox/scripts/bin/backup-utility     | 22 +++++++++----------
 .../scripts/bin/object-storage-backup         |  6 ++---
 .../scripts/bin/object-storage-restore        |  3 ++-
 .../scripts/lib/object_storage_backup.rb      | 19 ++++++++--------
 4 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index 29a5b4cfb..06ce67977 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -50,7 +50,7 @@ function usage()
      --aws-kms-key-id                       Add KMS key id when S3 bucket is encrypted with a customer key.
      --aws-s3-endpoint-url                  Specify an AWS S3 endpoint URL
      --aws-region                           Add AWS region (required for AWS STS regionalized endpoint)
-     --azure-endpoint-url                   Specify the Azure endpoint, e.g. https://YOUR_STORAGE_ACCOUNT.blob.core.windows.net
+     --azure-base-url                       Specify the Azure endpoint, e.g. https://YOUR_STORAGE_ACCOUNT.blob.core.windows.net
 HEREDOC
 }
 
@@ -79,7 +79,7 @@ function fetch_remote_backup(){
         esac
         ;;
       gcs) gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null ;;
-      azure) azcopy copy "${AZURE_ENDPOINT_URL}/${BACKUP_BUCKET_NAME}/${file_name}?$(get_azure_token)" ${output_path} > /dev/null ;;
+      azure) azcopy copy "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${file_name}?$(get_azure_token)" ${output_path} > /dev/null ;;
       *) echo "Unknown backend: ${BACKUP_BACKEND}" ;;
     esac
   fi
@@ -154,7 +154,7 @@ function get_existing_backups(){
       existing_backups=($(gsutil ls gs://$BACKUP_BUCKET_NAME/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_[0-9][0-9][0-9][0-9]_[0-9][0-9]_[0-9][0-9]_\*_gitlab_backup.tar | LC_ALL=C sort))
       ;;
     azure)
-      existing_backups=($(azcopy list "${AZURE_ENDPOINT_URL}/${BACKUP_BUCKET_NAME}?$(get_azure_token)" | awk '/^INFO: [[:alnum:]\/\.]+;/ {print gensub(";$", "", 1, $2)}' | grep -E '^[0-9]{10}_[0-9]{4}_[0-9]{2}_[0-9]{2}_.+_gitlab_backup.tar$' | LC_ALL=C sort))
+      existing_backups=($(azcopy list "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}?$(get_azure_token)" | awk '/^INFO: [[:alnum:]\/\.]+;/ {print gensub(";$", "", 1, $2)}' | grep -E '^[0-9]{10}_[0-9]{4}_[0-9]{2}_[0-9]{2}_.+_gitlab_backup.tar$' | LC_ALL=C sort))
       ;;
     *)
       echo "Unknown backend for backup: ${BACKUP_BACKEND}"
@@ -177,7 +177,7 @@ function remove_backup(){
       esac
       ;;
     gcs) gsutil rm ${backup_to_remove} > /dev/null ;;
-    azure) azcopy remove "${AZURE_ENDPOINT_URL}/${BACKUP_BUCKET_NAME}/${backup_to_remove}?$(get_azure_token)" > /dev/null ;;
+    azure) azcopy remove "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_to_remove}?$(get_azure_token)" > /dev/null ;;
     *)
       echo "Unknown backend for backup: ${BACKUP_BACKEND}"
       exit 1
@@ -287,14 +287,14 @@ function backup(){
       ;;
     azure)
       if [ -z "${STORAGE_CLASS}" ]; then
-        echo "azcopy copy '${backup_tars_path}/${backup_name}.tar' '${AZURE_ENDPOINT_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?TOKEN'"
-        azcopy copy "${backup_tars_path}/${backup_name}.tar" "${AZURE_ENDPOINT_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
+        echo "azcopy copy '${backup_tars_path}/${backup_name}.tar' '${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?TOKEN'"
+        azcopy copy "${backup_tars_path}/${backup_name}.tar" "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
       else
-        echo "azcopy copy --block-blob-tier '${STORAGE_CLASS}' '${backup_tars_path}/${backup_name}.tar' '${AZURE_ENDPOINT_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar??TOKEN'" 
+        echo "azcopy copy --block-blob-tier '${STORAGE_CLASS}' '${backup_tars_path}/${backup_name}.tar' '${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar??TOKEN'" 
         azcopy copy --block-blob-tier "${STORAGE_CLASS}" \
-          "${backup_tars_path}/${backup_name}.tar" "${AZURE_ENDPOINT_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
+          "${backup_tars_path}/${backup_name}.tar" "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
       fi
-      echo "[DONE] Backup can be found at ${AZURE_ENDPOINT_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar"
+      echo "[DONE] Backup can be found at ${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar"
       ;;
     *) echo "Unknown backend for backup: ${BACKUP_BACKEND}" ;;
   esac
@@ -449,8 +449,8 @@ do
       shift
       shift
       ;;
-    --azure-endpoint-url)
-      AZURE_ENDPOINT_URL=$2
+    --azure-base-url)
+      AZURE_BASE_URL=$2
       shift
       shift
       ;;
diff --git a/gitlab-toolbox/scripts/bin/object-storage-backup b/gitlab-toolbox/scripts/bin/object-storage-backup
index 78bb1a6bd..61fe8db89 100755
--- a/gitlab-toolbox/scripts/bin/object-storage-backup
+++ b/gitlab-toolbox/scripts/bin/object-storage-backup
@@ -5,12 +5,10 @@ require 'object_storage_backup'
 abort ("backup_item and output_tar_path arguments needs to be passed to the script") unless ARGV.length == 2
 
 bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}"
-if ENV["BACKUP_BACKEND"] == "azure"
-  bucket_name = "#{ENV['BACKUP_BUCKET_NAME']}/#{bucket_name}"
-end
 tmp_bucket  = ENV['TMP_BUCKET_NAME'] || 'tmp'
 backend_type = ENV['BACKUP_BACKEND'] || 's3'
 s3_tool = ENV['S3_TOOL'] || 's3cmd'
 aws_s3_settings = ENV['AWS_S3_SETTINGS'] || ''
 aws_kms_settings = ENV['AWS_KMS_SETTINGS'] || ''
-ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings).backup
+azure_base_url = ENV['AZURE_BASE_URL'] || ''
+ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings, azure_base_url).backup
diff --git a/gitlab-toolbox/scripts/bin/object-storage-restore b/gitlab-toolbox/scripts/bin/object-storage-restore
index dc2f63cfa..1a80a0040 100755
--- a/gitlab-toolbox/scripts/bin/object-storage-restore
+++ b/gitlab-toolbox/scripts/bin/object-storage-restore
@@ -10,4 +10,5 @@ backend_type = ENV['BACKUP_BACKEND'] || 's3'
 s3_tool = ENV['S3_TOOL'] || 's3cmd'
 aws_s3_settings = ENV['AWS_S3_SETTINGS'] || ''
 aws_kms_settings = ENV['AWS_KMS_SETTINGS'] || ''
-ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings).restore
+azure_base_url = ENV['AZURE_BASE_URL'] || ''
+ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings, azure_base_url).restore
diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index 728ca818f..de8709c6f 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -16,7 +16,7 @@ class ObjectStorageBackup
   REGEXP_EXCLUDE='tmp/builds/.*$'
   private_constant :REGEXP_EXCLUDE
 
-  def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3', s3_tool = 's3cmd', aws_s3_settings = '', aws_kms_settings = '')
+  def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3', s3_tool = 's3cmd', aws_s3_settings = '', aws_kms_settings = '', azure_base_url = '')
     @name = name
     @local_tar_path = local_tar_path
     @remote_bucket_name = remote_bucket_name
@@ -25,6 +25,7 @@ class ObjectStorageBackup
     @s3_tool = s3_tool
     @aws_s3_settings = aws_s3_settings
     @aws_kms_settings = aws_kms_settings
+    @azure_base_url = azure_base_url
   end
 
   def backup
@@ -43,9 +44,9 @@ class ObjectStorageBackup
       cmd = %W(gsutil -m rsync -d -x #{REGEXP_EXCLUDE} -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name})
     elsif @backend == "azure"
       azure_token = read_azure_token
-      check_bucket_cmd = %W(azcopy list #{@remote_bucket_name}?#{azure_token})
-      sanitized_check_bucket_cmd = %W(azcopy list #{@remote_bucket_name}?TOKEN)
-      cmd = %W(azcopy copy /srv/gitlab/tmp/#{@name} "#{@remote_bucket_name}?#{azure_token}")
+      check_bucket_cmd = %W(azcopy list #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token})
+      sanitized_check_bucket_cmd = %W(azcopy list #{@azure_base_url}/#{@remote_bucket_name}?TOKEN)
+      cmd = %W(azcopy copy /srv/gitlab/tmp/#{@name} #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token} --recursive)
     end
 
     # Check if the bucket exists
@@ -111,8 +112,8 @@ class ObjectStorageBackup
     elsif @backend == "gcs"
       cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}/#{dir_name})
     elsif @backend == "azure"
-      azure_token = open('/etc/gitlab/objectstoragage/azure_token', 'r').read()
-      cmd = %W(azcopy copy #{source_path} #{@remote_bucket_name}?#{azure_token})
+      azure_token = read_azure_token
+      cmd = %W(azcopy copy #{source_path} #{@azure_base_url}/?#{@remote_bucket_name}?#{azure_token})
     end
 
     output, status = run_cmd(cmd)
@@ -133,7 +134,7 @@ class ObjectStorageBackup
       cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/)
     elsif @backend == "azure"
       azure_token = read_azure_token
-      cmd = %W(azcopy copy #{@remote_bucket_name}?#{azure_token} #{@tmp_bucket_name}/#{backup_file_name}?#{azure_token})
+      cmd = %W(azcopy copy #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token} #{@azure_base_url}/#{@tmp_bucket_name}/#{backup_file_name}?#{azure_token})
     end
 
     output, status = run_cmd(cmd)
@@ -162,7 +163,7 @@ class ObjectStorageBackup
       cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*)
     elsif @backend == 'azure'
       azure_token = read_azure_token
-      cmd = %W(azcopy remove --recursive=true #{@remote_bucket_name}?#{azure_token})
+      cmd = %W(azcopy remove --recursive=true #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token})
     end
     output, status = run_cmd(cmd)
     failure_abort('bucket cleanup', output) unless status.zero?
@@ -191,6 +192,6 @@ class ObjectStorageBackup
   end
 
   def read_azure_token
-    return open('/etc/gitlab/objectstorage/azure_token', 'r').read()
+    open('/etc/gitlab/objectstorage/azure_token', 'r').read()
   end
 end
-- 
GitLab


From e10f54ba36d497f3364ce2bf523a687b3ec0352b Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Wed, 15 Feb 2023 00:00:44 +0100
Subject: [PATCH 17/42] Fix Azure get existing backups

---
 gitlab-toolbox/scripts/bin/backup-utility | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index 06ce67977..8ea877a75 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -154,7 +154,7 @@ function get_existing_backups(){
       existing_backups=($(gsutil ls gs://$BACKUP_BUCKET_NAME/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_[0-9][0-9][0-9][0-9]_[0-9][0-9]_[0-9][0-9]_\*_gitlab_backup.tar | LC_ALL=C sort))
       ;;
     azure)
-      existing_backups=($(azcopy list "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}?$(get_azure_token)" | awk '/^INFO: [[:alnum:]\/\.]+;/ {print gensub(";$", "", 1, $2)}' | grep -E '^[0-9]{10}_[0-9]{4}_[0-9]{2}_[0-9]{2}_.+_gitlab_backup.tar$' | LC_ALL=C sort))
+      existing_backups=($(azcopy list "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}?$(get_azure_token)" | awk '/^INFO: / {print $2}' | sed 's/;$//' | grep -E '^[0-9]{10}_[0-9]{4}_[0-9]{2}_[0-9]{2}_.+_gitlab_backup.tar$' | LC_ALL=C sort))
       ;;
     *)
       echo "Unknown backend for backup: ${BACKUP_BACKEND}"
-- 
GitLab


From d5ec2c144fca783f54648af97500389c4c344dab Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Wed, 15 Feb 2023 10:11:35 +0100
Subject: [PATCH 18/42] Remove debugging output

---
 gitlab-toolbox/scripts/lib/object_storage_backup.rb | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index de8709c6f..7b2bec077 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -50,16 +50,9 @@ class ObjectStorageBackup
     end
 
     # Check if the bucket exists
-    ## DEBUG
-    puts "running command: #{sanitized_check_bucket_cmd}"
     output, status = run_cmd(check_bucket_cmd)
-    puts "status = #{status}"
-    puts "output = #{output}"
-
     unless status.zero?
       puts "Bucket not found: #{@remote_bucket_name}. Skipping backup of #{@name} ...".blue
-      puts "Output from #{sanitized_check_bucket_cmd}:".blue
-      puts output.red
       return
     end
 
-- 
GitLab


From fa034a1dda0e5ec409cfcef6990a16f9c075107c Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Wed, 15 Feb 2023 10:17:56 +0100
Subject: [PATCH 19/42] Improve doc for toolbox azure-base-url argument

---
 gitlab-toolbox/scripts/bin/backup-utility | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index 8ea877a75..44ab044d3 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -50,7 +50,8 @@ function usage()
      --aws-kms-key-id                       Add KMS key id when S3 bucket is encrypted with a customer key.
      --aws-s3-endpoint-url                  Specify an AWS S3 endpoint URL
      --aws-region                           Add AWS region (required for AWS STS regionalized endpoint)
-     --azure-base-url                       Specify the Azure endpoint, e.g. https://YOUR_STORAGE_ACCOUNT.blob.core.windows.net
+     --azure-base-url                       Specify the Azure base URL containing your storage account
+                                            (e.g. https://YOUR_STORAGE_ACCOUNT.blob.core.windows.net)
 HEREDOC
 }
 
@@ -287,10 +288,8 @@ function backup(){
       ;;
     azure)
       if [ -z "${STORAGE_CLASS}" ]; then
-        echo "azcopy copy '${backup_tars_path}/${backup_name}.tar' '${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?TOKEN'"
         azcopy copy "${backup_tars_path}/${backup_name}.tar" "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
       else
-        echo "azcopy copy --block-blob-tier '${STORAGE_CLASS}' '${backup_tars_path}/${backup_name}.tar' '${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar??TOKEN'" 
         azcopy copy --block-blob-tier "${STORAGE_CLASS}" \
           "${backup_tars_path}/${backup_name}.tar" "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
       fi
-- 
GitLab


From 8d78d4ccd2db22c00383b385cbdd6f69ed788892 Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Wed, 15 Feb 2023 13:13:38 +0100
Subject: [PATCH 20/42] Use quiet mode for azcopy

See: https://github.com/Azure/azure-storage-azcopy/issues/534
---
 gitlab-toolbox/scripts/bin/backup-utility | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index 44ab044d3..b43c8c2c1 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -288,10 +288,11 @@ function backup(){
       ;;
     azure)
       if [ -z "${STORAGE_CLASS}" ]; then
-        azcopy copy "${backup_tars_path}/${backup_name}.tar" "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
+        azcopy copy "${backup_tars_path}/${backup_name}.tar" --output-level quiet \
+           "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)"
       else
-        azcopy copy --block-blob-tier "${STORAGE_CLASS}" \
-          "${backup_tars_path}/${backup_name}.tar" "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" > /dev/null
+        azcopy copy --block-blob-tier "${STORAGE_CLASS}" --output-level quiet \
+          "${backup_tars_path}/${backup_name}.tar" "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)"
       fi
       echo "[DONE] Backup can be found at ${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar"
       ;;
-- 
GitLab


From 34dc6ea041d3b7bfca178aef047a4313c73735b7 Mon Sep 17 00:00:00 2001
From: Gerard Hickey <ghickey@gitlab.com>
Date: Thu, 9 Feb 2023 17:47:13 +0100
Subject: [PATCH 21/42] Fixed listing of backups in Azure blobs

Signed-off-by: Gerard Hickey <ghickey@gitlab.com>
---
 gitlab-toolbox/scripts/bin/backup-utility | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index b43c8c2c1..06e9d178f 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -155,7 +155,7 @@ function get_existing_backups(){
       existing_backups=($(gsutil ls gs://$BACKUP_BUCKET_NAME/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_[0-9][0-9][0-9][0-9]_[0-9][0-9]_[0-9][0-9]_\*_gitlab_backup.tar | LC_ALL=C sort))
       ;;
     azure)
-      existing_backups=($(azcopy list "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}?$(get_azure_token)" | awk '/^INFO: / {print $2}' | sed 's/;$//' | grep -E '^[0-9]{10}_[0-9]{4}_[0-9]{2}_[0-9]{2}_.+_gitlab_backup.tar$' | LC_ALL=C sort))
+      existing_backups=($(azcopy list "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}?$(get_azure_token)" | awk '/^INFO: [[:alnum:]_\.\-]+;/ {gsub(/;$/, "", $2); print $2}' | grep -E '^[0-9]{10}_[0-9]{4}_[0-9]{2}_[0-9]{2}_.+_gitlab_backup.tar$' | LC_ALL=C sort))
       ;;
     *)
       echo "Unknown backend for backup: ${BACKUP_BACKEND}"
-- 
GitLab


From a2ff2d6d6eaae2a515680d1ca30ca8d1ae70164a Mon Sep 17 00:00:00 2001
From: Gerard Hickey <ghickey@gitlab.com>
Date: Thu, 9 Feb 2023 17:56:54 +0100
Subject: [PATCH 22/42] Fix typo in toolbox label of ubi image

---
 gitlab-toolbox/Dockerfile.ubi8 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gitlab-toolbox/Dockerfile.ubi8 b/gitlab-toolbox/Dockerfile.ubi8
index 3ad76db25..c3e94d0ef 100644
--- a/gitlab-toolbox/Dockerfile.ubi8
+++ b/gitlab-toolbox/Dockerfile.ubi8
@@ -17,7 +17,7 @@ LABEL source="https://gitlab.com/gitlab-org/build/CNG/-/tree/master/gitlab-toolb
       version=${GITLAB_VERSION} \
       release=${GITLAB_VERSION} \
       summary="Toolbox is an entry point for interaction with other containers in the cluster." \
-      description="Toolbox is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage."
+      description="Toolbox is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to interact with object storage."
 
 ADD gitlab-toolbox-ee.tar.gz /
 ADD gitlab-python.tar.gz /
-- 
GitLab


From 8bb45af8c1902d07bd07a0e759f6ac4362df9764 Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Mon, 20 Feb 2023 23:04:05 +0100
Subject: [PATCH 23/42] Create SAS tokens from Azure access key

Co-authored-by: Gerard Hickey <ghickey@gitlab.com>
---
 gitlab-toolbox/scripts/bin/backup-utility     | 48 ++++++++++---------
 .../scripts/bin/object-storage-azure-token    |  6 +++
 .../scripts/bin/object-storage-backup         |  5 +-
 .../scripts/bin/object-storage-restore        |  5 +-
 .../scripts/lib/object_storage_azure.rb       | 37 ++++++++++++++
 .../scripts/lib/object_storage_backup.rb      | 18 +++----
 6 files changed, 84 insertions(+), 35 deletions(-)
 create mode 100755 gitlab-toolbox/scripts/bin/object-storage-azure-token
 create mode 100644 gitlab-toolbox/scripts/lib/object_storage_azure.rb

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index 06e9d178f..5c099fe07 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -7,6 +7,7 @@ export BACKUP_BACKEND=${BACKUP_BACKEND-s3}
 export S3_TOOL=${S3_TOOL-s3cmd}
 export AWS_KMS_SETTINGS=""
 export AWS_S3_SETTINGS=""
+export AZURE_BLOB_HOST="blob.core.windows.net"
 AWS_KMS_SETTINGS_LIST=()
 AWS_S3_SETTINGS_LIST=()
 S3_CMD_BACKUP_OPTION=""
@@ -41,17 +42,17 @@ function usage()
                                             Special config file for s3cmd (see: https://s3tools.org/usage)
                                             Not required when using the awscli tool.
      --s3tool TOOL                          S3 CLI tool to use. Can be either 's3cmd' or 'awscli'.
-     --storage-class CLASSNAME              Pass this storage class to the gcs, s3cmd or aws cli for more cost-efficient
-                                            storage of backups.
+     --storage-class CLASSNAME              Pass this storage class to the gcs, s3cmd, aws or azcopy cli for more
+                                            cost-efficient storage of backups.
      --maximum-backups N                    Only keep the most recent N number of backups, deleting others after success.
                                             Requires s3config or AWS credentials to be able to list and delete objects.
      --cleanup                              Run the backup cleanup without creating a new backup. Can be used with the
                                             'maximum-backups' option to clean old remote backups.
      --aws-kms-key-id                       Add KMS key id when S3 bucket is encrypted with a customer key.
-     --aws-s3-endpoint-url                  Specify an AWS S3 endpoint URL
-     --aws-region                           Add AWS region (required for AWS STS regionalized endpoint)
-     --azure-base-url                       Specify the Azure base URL containing your storage account
-                                            (e.g. https://YOUR_STORAGE_ACCOUNT.blob.core.windows.net)
+     --aws-s3-endpoint-url                  Specify an AWS S3 endpoint URL.
+     --aws-region                           Add AWS region (required for AWS STS regionalized endpoint).
+     --azure-storage-account                Name of the Azure storage account to push backups to.
+     --azure-blob-host                      Host of the Azure blob service.
 HEREDOC
 }
 
@@ -80,7 +81,7 @@ function fetch_remote_backup(){
         esac
         ;;
       gcs) gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null ;;
-      azure) azcopy copy "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${file_name}?$(get_azure_token)" ${output_path} > /dev/null ;;
+      azure) azcopy copy "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${file_name}?$(get_azure_token)" ${output_path} > /dev/null ;;
       *) echo "Unknown backend: ${BACKUP_BACKEND}" ;;
     esac
   fi
@@ -111,14 +112,12 @@ function get_version(){
   cat $rails_dir/VERSION
 }
 
-function get_azure_token(){
-  token_file='/etc/gitlab/objectstorage/azure_token'
-  if [ ! -e ${token_file} ]; then
-    echo "Azure SAS token not found in ${token_file}"
-    exit 1
-  fi
+function get_azure_url(){
+  echo "https://${AZURE_STORAGE_ACCOUNT}.${AZURE_BLOB_HOST}"
+}
 
-  cat ${token_file}
+function get_azure_token(){
+  echo -n $(object-storage-azure-token)
 }
 
 function get_backup_name(){
@@ -155,7 +154,7 @@ function get_existing_backups(){
       existing_backups=($(gsutil ls gs://$BACKUP_BUCKET_NAME/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_[0-9][0-9][0-9][0-9]_[0-9][0-9]_[0-9][0-9]_\*_gitlab_backup.tar | LC_ALL=C sort))
       ;;
     azure)
-      existing_backups=($(azcopy list "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}?$(get_azure_token)" | awk '/^INFO: [[:alnum:]_\.\-]+;/ {gsub(/;$/, "", $2); print $2}' | grep -E '^[0-9]{10}_[0-9]{4}_[0-9]{2}_[0-9]{2}_.+_gitlab_backup.tar$' | LC_ALL=C sort))
+      existing_backups=($(azcopy list "$(get_azure_url)/${BACKUP_BUCKET_NAME}?$(get_azure_token)" | awk '/^INFO: [[:alnum:]_\.\-]+;/ {gsub(/;$/, "", $2); print $2}' | grep -E '^[0-9]{10}_[0-9]{4}_[0-9]{2}_[0-9]{2}_.+_gitlab_backup.tar$' | LC_ALL=C sort))
       ;;
     *)
       echo "Unknown backend for backup: ${BACKUP_BACKEND}"
@@ -178,7 +177,7 @@ function remove_backup(){
       esac
       ;;
     gcs) gsutil rm ${backup_to_remove} > /dev/null ;;
-    azure) azcopy remove "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_to_remove}?$(get_azure_token)" > /dev/null ;;
+    azure) azcopy remove "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${backup_to_remove}?$(get_azure_token)" > /dev/null ;;
     *)
       echo "Unknown backend for backup: ${BACKUP_BACKEND}"
       exit 1
@@ -288,11 +287,11 @@ function backup(){
       ;;
     azure)
       if [ -z "${STORAGE_CLASS}" ]; then
-        azcopy copy "${backup_tars_path}/${backup_name}.tar" --output-level quiet \
-           "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)"
+        azcopy copy "${backup_tars_path}/${backup_name}.tar" \
+           "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)"
       else
-        azcopy copy --block-blob-tier "${STORAGE_CLASS}" --output-level quiet \
-          "${backup_tars_path}/${backup_name}.tar" "${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)"
+        azcopy copy --block-blob-tier "${STORAGE_CLASS}" \
+          "${backup_tars_path}/${backup_name}.tar" "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)"
       fi
       echo "[DONE] Backup can be found at ${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar"
       ;;
@@ -449,8 +448,13 @@ do
       shift
       shift
       ;;
-    --azure-base-url)
-      AZURE_BASE_URL=$2
+    --azure-storage-account)
+      export AZURE_STORAGE_ACCOUNT=$2
+      shift
+      shift
+      ;;
+    --azure-blob-host)
+      export AZURE_BLOB_HOST=$2
       shift
       shift
       ;;
diff --git a/gitlab-toolbox/scripts/bin/object-storage-azure-token b/gitlab-toolbox/scripts/bin/object-storage-azure-token
new file mode 100755
index 000000000..292a0cb1b
--- /dev/null
+++ b/gitlab-toolbox/scripts/bin/object-storage-azure-token
@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'object_storage_azure'
+
+azure_storage_account = ENV["AZURE_STORAGE_ACCOUNT"] || ''
+puts AzureSASTokenProvider.new(azure_storage_account).sas_token
\ No newline at end of file
diff --git a/gitlab-toolbox/scripts/bin/object-storage-backup b/gitlab-toolbox/scripts/bin/object-storage-backup
index 61fe8db89..5ecfd520a 100755
--- a/gitlab-toolbox/scripts/bin/object-storage-backup
+++ b/gitlab-toolbox/scripts/bin/object-storage-backup
@@ -10,5 +10,6 @@ backend_type = ENV['BACKUP_BACKEND'] || 's3'
 s3_tool = ENV['S3_TOOL'] || 's3cmd'
 aws_s3_settings = ENV['AWS_S3_SETTINGS'] || ''
 aws_kms_settings = ENV['AWS_KMS_SETTINGS'] || ''
-azure_base_url = ENV['AZURE_BASE_URL'] || ''
-ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings, azure_base_url).backup
+azure_storage_account = ENV['AZURE_STORAGE_ACCOUNT'] || ''
+azure_blob_url = ENV['AZURE_BLOB_URL'] || ''
+ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings, azure_storage_account, azure_blob_url).backup
diff --git a/gitlab-toolbox/scripts/bin/object-storage-restore b/gitlab-toolbox/scripts/bin/object-storage-restore
index 1a80a0040..a802037d2 100755
--- a/gitlab-toolbox/scripts/bin/object-storage-restore
+++ b/gitlab-toolbox/scripts/bin/object-storage-restore
@@ -10,5 +10,6 @@ backend_type = ENV['BACKUP_BACKEND'] || 's3'
 s3_tool = ENV['S3_TOOL'] || 's3cmd'
 aws_s3_settings = ENV['AWS_S3_SETTINGS'] || ''
 aws_kms_settings = ENV['AWS_KMS_SETTINGS'] || ''
-azure_base_url = ENV['AZURE_BASE_URL'] || ''
-ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings, azure_base_url).restore
+azure_storage_account = ENV['AZURE_STORAGE_ACCOUNT'] || ''
+azure_blob_url = ENV['AZURE_BLOB_URL'] || ''
+ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings, azure_storage_account, azure_blob_url).restore
diff --git a/gitlab-toolbox/scripts/lib/object_storage_azure.rb b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
new file mode 100644
index 000000000..4eaaca59c
--- /dev/null
+++ b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
@@ -0,0 +1,37 @@
+require 'fileutils'
+require 'azure/storage/common'
+require 'openssl'
+require 'uri'
+
+class AzureSASTokenProvider
+
+  def initialize(storage_account,
+                 token_expire_s = 12 * 30 * 60,
+                 token_file = '/tmp/azure_sas_token',
+                 access_key_file = '/etc/gitlab/objectstorage/azure_access_key')
+    @storage_account = storage_account
+    @token_expire_s = token_expire_s
+    @token_file = token_file
+    @access_key_file = access_key_file
+  end
+
+  def sas_token
+    # Read the cached token if it exist and within half its lifetime
+    if File.exist?(@token_file) && File.stat(@token_file).mtime >= (Time.now - (@token_expire_s / 2))
+      return open(@token_file, 'r').read
+    end
+
+    key = open(@access_key_file, 'r').read
+    generator = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@storage_account, key)
+    token = generator.generate_account_sas_token({
+      service: 'b',
+      resource: 'co',
+      permissions: 'racwdl',
+      protocol: 'https',
+      start: (Time.now - 10 * 60).iso8601,
+      expiry: (Time.now + @token_expire_s).iso8601
+    })
+    open(@token_file, 'w').write token
+    token
+  end
+end
diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index 7b2bec077..8147dcfd0 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -1,5 +1,6 @@
 require 'open3'
 require 'fileutils'
+require 'object_storage_azure'
 
 class String
   def red; "\e[31m#{self}\e[0m" end
@@ -16,7 +17,9 @@ class ObjectStorageBackup
   REGEXP_EXCLUDE='tmp/builds/.*$'
   private_constant :REGEXP_EXCLUDE
 
-  def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3', s3_tool = 's3cmd', aws_s3_settings = '', aws_kms_settings = '', azure_base_url = '')
+  def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3',
+                 s3_tool = 's3cmd', aws_s3_settings = '', aws_kms_settings = '',
+                 azure_storage_account = '', azure_blob_host = 'blob.core.windows.net')
     @name = name
     @local_tar_path = local_tar_path
     @remote_bucket_name = remote_bucket_name
@@ -25,7 +28,8 @@ class ObjectStorageBackup
     @s3_tool = s3_tool
     @aws_s3_settings = aws_s3_settings
     @aws_kms_settings = aws_kms_settings
-    @azure_base_url = azure_base_url
+    @azure_token_prov = AzureSASTokenProvider.new(azure_storage_account)
+    @azure_base_url = "https://#{azure_storage_account}.#{azure_blob_host}"
   end
 
   def backup
@@ -43,7 +47,7 @@ class ObjectStorageBackup
       sanitized_check_bucket_cmd = check_bucket_cmd
       cmd = %W(gsutil -m rsync -d -x #{REGEXP_EXCLUDE} -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name})
     elsif @backend == "azure"
-      azure_token = read_azure_token
+      azure_token = @azure_token_prov.sas_token
       check_bucket_cmd = %W(azcopy list #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token})
       sanitized_check_bucket_cmd = %W(azcopy list #{@azure_base_url}/#{@remote_bucket_name}?TOKEN)
       cmd = %W(azcopy copy /srv/gitlab/tmp/#{@name} #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token} --recursive)
@@ -126,7 +130,7 @@ class ObjectStorageBackup
     elsif @backend == "gcs"
       cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/)
     elsif @backend == "azure"
-      azure_token = read_azure_token
+      azure_token = @azure_token_prov.sas_token
       cmd = %W(azcopy copy #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token} #{@azure_base_url}/#{@tmp_bucket_name}/#{backup_file_name}?#{azure_token})
     end
 
@@ -155,7 +159,7 @@ class ObjectStorageBackup
 
       cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*)
     elsif @backend == 'azure'
-      azure_token = read_azure_token
+      azure_token = @azure_token_prov.sas_token
       cmd = %W(azcopy remove --recursive=true #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token})
     end
     output, status = run_cmd(cmd)
@@ -183,8 +187,4 @@ class ObjectStorageBackup
     _, stdout, wait_thr = Open3.popen2e(*cmd)
     return stdout.read, wait_thr.value.exitstatus
   end
-
-  def read_azure_token
-    open('/etc/gitlab/objectstorage/azure_token', 'r').read()
-  end
 end
-- 
GitLab


From 4cbdc642b6dab465625025922e25973a34c5bce2 Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Tue, 21 Feb 2023 00:17:44 +0100
Subject: [PATCH 24/42] Azure SAS Token without Azure SDK

SAS token generation with the Azure
Storage SDK with a custom one.

This allows removing the Azure SDK from the
Toolbox container, reducing image size and
preventing dependency conflicts with gems
from upstream images.
---
 .../scripts/lib/object_storage_azure.rb       | 49 ++++++++++++++-----
 1 file changed, 37 insertions(+), 12 deletions(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_azure.rb b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
index 4eaaca59c..d55d599f4 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_azure.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
@@ -1,12 +1,13 @@
+require 'base64'
 require 'fileutils'
-require 'azure/storage/common'
 require 'openssl'
+require 'time'
 require 'uri'
 
 class AzureSASTokenProvider
 
   def initialize(storage_account,
-                 token_expire_s = 12 * 30 * 60,
+                 token_expire_s = 12 * 60 * 60,
                  token_file = '/tmp/azure_sas_token',
                  access_key_file = '/etc/gitlab/objectstorage/azure_access_key')
     @storage_account = storage_account
@@ -21,17 +22,41 @@ class AzureSASTokenProvider
       return open(@token_file, 'r').read
     end
 
-    key = open(@access_key_file, 'r').read
-    generator = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@storage_account, key)
-    token = generator.generate_account_sas_token({
-      service: 'b',
-      resource: 'co',
-      permissions: 'racwdl',
-      protocol: 'https',
-      start: (Time.now - 10 * 60).iso8601,
-      expiry: (Time.now + @token_expire_s).iso8601
-    })
+    access_key = open(@access_key_file, 'r').read
+    start = (Time.now - 10 * 60).utc.iso8601
+    expiry = (Time.now + @token_expire_s).utc.iso8601
+    token = generate(access_key, @storage_account, start, expiry)
     open(@token_file, 'w').write token
     token
   end
+
+  private
+
+  def generate(access_key, account, start_iso, expiry_iso)
+    permissions = 'rawdl'  # read, write, delete, list | REMOVE ME: 'racwdl'
+    protocol = 'https'     # allow https only
+    service = 'b'          # blob storage
+    resources = 'co'       # container + object level
+    version = '2018-11-09' # API Version
+    ip_range = ''          # Allowed IPs
+
+    # item order and empty items/lines matter
+    to_sign = [
+      account,
+      permissions,
+      service,
+      resources,
+      start_iso,
+      expiry_iso,
+      ip_range,
+      protocol,
+      version,
+      ""
+    ].join("\n")
+
+    sig = OpenSSL::HMAC.digest('sha256', Base64.decode64(access_key), to_sign)
+    sig = Base64.strict_encode64(sig)
+    sig = URI.encode_www_form_component(sig)
+    "sv=#{version}&ss=#{service}&srt=#{resources}&sp=#{permissions}&se=#{expiry_iso}&st=#{start_iso}&spr=#{protocol}&sig=#{sig}"
+  end
 end
-- 
GitLab


From f71e85292dda78b2a061d3ac5ab7f1b4e2874d60 Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Tue, 21 Feb 2023 15:09:14 +0100
Subject: [PATCH 25/42] Azcopy set output-level to essential

Use essential output level instead of
redirecting stdout to /dev/null as
azcopy also outputs error information to
stdout.

Essential output level is defined
as everything but progress, info and
prompts.
---
 gitlab-toolbox/scripts/bin/backup-utility           |  8 ++++----
 gitlab-toolbox/scripts/lib/object_storage_backup.rb | 11 ++++-------
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index 5c099fe07..006f464ac 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -81,7 +81,7 @@ function fetch_remote_backup(){
         esac
         ;;
       gcs) gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null ;;
-      azure) azcopy copy "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${file_name}?$(get_azure_token)" ${output_path} > /dev/null ;;
+      azure) azcopy copy "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${file_name}?$(get_azure_token)" ${output_path} --output-level essential ;;
       *) echo "Unknown backend: ${BACKUP_BACKEND}" ;;
     esac
   fi
@@ -177,7 +177,7 @@ function remove_backup(){
       esac
       ;;
     gcs) gsutil rm ${backup_to_remove} > /dev/null ;;
-    azure) azcopy remove "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${backup_to_remove}?$(get_azure_token)" > /dev/null ;;
+    azure) azcopy remove "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${backup_to_remove}?$(get_azure_token)" --output-level essential ;;
     *)
       echo "Unknown backend for backup: ${BACKUP_BACKEND}"
       exit 1
@@ -287,10 +287,10 @@ function backup(){
       ;;
     azure)
       if [ -z "${STORAGE_CLASS}" ]; then
-        azcopy copy "${backup_tars_path}/${backup_name}.tar" \
+        azcopy copy "${backup_tars_path}/${backup_name}.tar" --output-level essential \
            "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)"
       else
-        azcopy copy --block-blob-tier "${STORAGE_CLASS}" \
+        azcopy copy --block-blob-tier "${STORAGE_CLASS}" --output-level essential \
           "${backup_tars_path}/${backup_name}.tar" "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)"
       fi
       echo "[DONE] Backup can be found at ${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar"
diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index 8147dcfd0..207c78829 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -41,16 +41,13 @@ class ObjectStorageBackup
         check_bucket_cmd = %W(aws s3api head-bucket --bucket #{@remote_bucket_name}) + @aws_s3_settings.split(" ")
         cmd = %W(aws s3 sync --delete --exclude #{GLOB_EXCLUDE} s3://#{@remote_bucket_name}/ /srv/gitlab/tmp/#{@name}/) + @aws_s3_settings.split(" ") + @aws_kms_settings.split(" ")
       end
-      sanitized_check_bucket_cmd = check_bucket_cmd
     elsif @backend == "gcs"
       check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name})
-      sanitized_check_bucket_cmd = check_bucket_cmd
       cmd = %W(gsutil -m rsync -d -x #{REGEXP_EXCLUDE} -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name})
     elsif @backend == "azure"
       azure_token = @azure_token_prov.sas_token
       check_bucket_cmd = %W(azcopy list #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token})
-      sanitized_check_bucket_cmd = %W(azcopy list #{@azure_base_url}/#{@remote_bucket_name}?TOKEN)
-      cmd = %W(azcopy copy /srv/gitlab/tmp/#{@name} #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token} --recursive)
+      cmd = %W(azcopy copy /srv/gitlab/tmp/#{@name} #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token} --output-level essential --recursive=true)
     end
 
     # Check if the bucket exists
@@ -110,7 +107,7 @@ class ObjectStorageBackup
       cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}/#{dir_name})
     elsif @backend == "azure"
       azure_token = read_azure_token
-      cmd = %W(azcopy copy #{source_path} #{@azure_base_url}/?#{@remote_bucket_name}?#{azure_token})
+      cmd = %W(azcopy copy #{source_path} #{@azure_base_url}/?#{@remote_bucket_name}?#{azure_token} --output-level essential)
     end
 
     output, status = run_cmd(cmd)
@@ -131,7 +128,7 @@ class ObjectStorageBackup
       cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/)
     elsif @backend == "azure"
       azure_token = @azure_token_prov.sas_token
-      cmd = %W(azcopy copy #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token} #{@azure_base_url}/#{@tmp_bucket_name}/#{backup_file_name}?#{azure_token})
+      cmd = %W(azcopy copy #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token} #{@azure_base_url}/#{@tmp_bucket_name}/#{backup_file_name}?#{azure_token} --output-level essential)
     end
 
     output, status = run_cmd(cmd)
@@ -160,7 +157,7 @@ class ObjectStorageBackup
       cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*)
     elsif @backend == 'azure'
       azure_token = @azure_token_prov.sas_token
-      cmd = %W(azcopy remove --recursive=true #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token})
+      cmd = %W(azcopy remove #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token} --output-level essential --recursive=true)
     end
     output, status = run_cmd(cmd)
     failure_abort('bucket cleanup', output) unless status.zero?
-- 
GitLab


From 7ed66b4d5301a87fefe46bc03d9c1857a9ed5f64 Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Wed, 22 Feb 2023 12:20:00 +0100
Subject: [PATCH 26/42] Fix usage of Azure blob host env var

---
 gitlab-toolbox/scripts/bin/object-storage-backup  | 2 +-
 gitlab-toolbox/scripts/bin/object-storage-restore | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/gitlab-toolbox/scripts/bin/object-storage-backup b/gitlab-toolbox/scripts/bin/object-storage-backup
index 5ecfd520a..079929e46 100755
--- a/gitlab-toolbox/scripts/bin/object-storage-backup
+++ b/gitlab-toolbox/scripts/bin/object-storage-backup
@@ -11,5 +11,5 @@ s3_tool = ENV['S3_TOOL'] || 's3cmd'
 aws_s3_settings = ENV['AWS_S3_SETTINGS'] || ''
 aws_kms_settings = ENV['AWS_KMS_SETTINGS'] || ''
 azure_storage_account = ENV['AZURE_STORAGE_ACCOUNT'] || ''
-azure_blob_url = ENV['AZURE_BLOB_URL'] || ''
+azure_blob_url = ENV['AZURE_BLOB_HOST'] || ''
 ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings, azure_storage_account, azure_blob_url).backup
diff --git a/gitlab-toolbox/scripts/bin/object-storage-restore b/gitlab-toolbox/scripts/bin/object-storage-restore
index a802037d2..ff9e7d602 100755
--- a/gitlab-toolbox/scripts/bin/object-storage-restore
+++ b/gitlab-toolbox/scripts/bin/object-storage-restore
@@ -11,5 +11,5 @@ s3_tool = ENV['S3_TOOL'] || 's3cmd'
 aws_s3_settings = ENV['AWS_S3_SETTINGS'] || ''
 aws_kms_settings = ENV['AWS_KMS_SETTINGS'] || ''
 azure_storage_account = ENV['AZURE_STORAGE_ACCOUNT'] || ''
-azure_blob_url = ENV['AZURE_BLOB_URL'] || ''
+azure_blob_url = ENV['AZURE_BLOB_HOST'] || ''
 ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings, azure_storage_account, azure_blob_url).restore
-- 
GitLab


From 53273b87b6a87899f9b5ad998741d592a4fcdb83 Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Wed, 22 Feb 2023 12:20:25 +0100
Subject: [PATCH 27/42] Fix sync of remote bucket via azcopy

---
 gitlab-toolbox/scripts/lib/object_storage_backup.rb | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index 207c78829..a3ba27dd3 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -47,7 +47,8 @@ class ObjectStorageBackup
     elsif @backend == "azure"
       azure_token = @azure_token_prov.sas_token
       check_bucket_cmd = %W(azcopy list #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token})
-      cmd = %W(azcopy copy /srv/gitlab/tmp/#{@name} #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token} --output-level essential --recursive=true)
+      cmd = %W(azcopy sync #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token} /srv/gitlab/tmp/#{@name}
+               --output-level essential --recursive=true --exclude-regex=#{REGEXP_EXCLUDE} --delete-destination=true)
     end
 
     # Check if the bucket exists
@@ -176,7 +177,7 @@ class ObjectStorageBackup
     failure_abort('un-archive', output) unless status.zero?
 
     Dir.glob("#{extracted_tar_path}/*").each do |file|
-     upload_to_object_storage(file)
+      upload_to_object_storage(file)
     end
   end
 
-- 
GitLab


From 32eb05296108c0304be77b7ec07e2a537ca2b81f Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Wed, 22 Feb 2023 14:53:43 +0100
Subject: [PATCH 28/42] Configure Azure backups via file

File is expected to be yaml format
with the following keys and values:

azure_storage_account_name: <the storage account (required)>
azure_storage_access_key: <base 64 encoded access key (required)>
azure_storage_domain: <host of the azure blob service (optional>
---
 gitlab-toolbox/scripts/bin/backup-utility     | 18 ++++-------
 .../scripts/bin/object-storage-azure-token    |  4 +--
 .../scripts/bin/object-storage-azure-url      |  6 ++++
 .../scripts/bin/object-storage-backup         |  5 ++--
 .../scripts/bin/object-storage-restore        |  5 ++--
 .../scripts/lib/object_storage_azure.rb       | 30 ++++++++++---------
 .../scripts/lib/object_storage_backup.rb      | 22 ++++++--------
 7 files changed, 43 insertions(+), 47 deletions(-)
 create mode 100755 gitlab-toolbox/scripts/bin/object-storage-azure-url

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index 006f464ac..c4fd75a27 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -7,7 +7,7 @@ export BACKUP_BACKEND=${BACKUP_BACKEND-s3}
 export S3_TOOL=${S3_TOOL-s3cmd}
 export AWS_KMS_SETTINGS=""
 export AWS_S3_SETTINGS=""
-export AZURE_BLOB_HOST="blob.core.windows.net"
+export AZURE_CONFIG_FILE="/etc/gitlab/objectstorage/azure_config"
 AWS_KMS_SETTINGS_LIST=()
 AWS_S3_SETTINGS_LIST=()
 S3_CMD_BACKUP_OPTION=""
@@ -51,8 +51,7 @@ function usage()
      --aws-kms-key-id                       Add KMS key id when S3 bucket is encrypted with a customer key.
      --aws-s3-endpoint-url                  Specify an AWS S3 endpoint URL.
      --aws-region                           Add AWS region (required for AWS STS regionalized endpoint).
-     --azure-storage-account                Name of the Azure storage account to push backups to.
-     --azure-blob-host                      Host of the Azure blob service.
+     --azure-config-file                    Path of the config file to configure Azure Block Storage access. 
 HEREDOC
 }
 
@@ -113,11 +112,11 @@ function get_version(){
 }
 
 function get_azure_url(){
-  echo "https://${AZURE_STORAGE_ACCOUNT}.${AZURE_BLOB_HOST}"
+  $(object-storage-azure-url)
 }
 
 function get_azure_token(){
-  echo -n $(object-storage-azure-token)
+  $(object-storage-azure-token)
 }
 
 function get_backup_name(){
@@ -448,13 +447,8 @@ do
       shift
       shift
       ;;
-    --azure-storage-account)
-      export AZURE_STORAGE_ACCOUNT=$2
-      shift
-      shift
-      ;;
-    --azure-blob-host)
-      export AZURE_BLOB_HOST=$2
+    --azure-config-file)
+      export AZURE_CONFIG_FILE=$2
       shift
       shift
       ;;
diff --git a/gitlab-toolbox/scripts/bin/object-storage-azure-token b/gitlab-toolbox/scripts/bin/object-storage-azure-token
index 292a0cb1b..27c0179a4 100755
--- a/gitlab-toolbox/scripts/bin/object-storage-azure-token
+++ b/gitlab-toolbox/scripts/bin/object-storage-azure-token
@@ -2,5 +2,5 @@
 
 require 'object_storage_azure'
 
-azure_storage_account = ENV["AZURE_STORAGE_ACCOUNT"] || ''
-puts AzureSASTokenProvider.new(azure_storage_account).sas_token
\ No newline at end of file
+config = ENV['AZURE_CONFIG_FILE'] || "/etc/gitlab/objectstorage/azure_config"
+puts AzureBackupUtil.new(config).sas_token
\ No newline at end of file
diff --git a/gitlab-toolbox/scripts/bin/object-storage-azure-url b/gitlab-toolbox/scripts/bin/object-storage-azure-url
new file mode 100755
index 000000000..e9f636dcf
--- /dev/null
+++ b/gitlab-toolbox/scripts/bin/object-storage-azure-url
@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'object_storage_azure'
+
+config = ENV['AZURE_CONFIG_FILE'] || "/etc/gitlab/objectstorage/azure_config"
+puts AzureBackupUtil.new(config).url
\ No newline at end of file
diff --git a/gitlab-toolbox/scripts/bin/object-storage-backup b/gitlab-toolbox/scripts/bin/object-storage-backup
index 079929e46..25c1ce0e7 100755
--- a/gitlab-toolbox/scripts/bin/object-storage-backup
+++ b/gitlab-toolbox/scripts/bin/object-storage-backup
@@ -10,6 +10,5 @@ backend_type = ENV['BACKUP_BACKEND'] || 's3'
 s3_tool = ENV['S3_TOOL'] || 's3cmd'
 aws_s3_settings = ENV['AWS_S3_SETTINGS'] || ''
 aws_kms_settings = ENV['AWS_KMS_SETTINGS'] || ''
-azure_storage_account = ENV['AZURE_STORAGE_ACCOUNT'] || ''
-azure_blob_url = ENV['AZURE_BLOB_HOST'] || ''
-ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings, azure_storage_account, azure_blob_url).backup
+azure_config = ENV['AZURE_CONFIG_FILE'] || "/etc/gitlab/objectstorage/azure_config"
+ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings, azure_config).backup
diff --git a/gitlab-toolbox/scripts/bin/object-storage-restore b/gitlab-toolbox/scripts/bin/object-storage-restore
index ff9e7d602..6ceb7166f 100755
--- a/gitlab-toolbox/scripts/bin/object-storage-restore
+++ b/gitlab-toolbox/scripts/bin/object-storage-restore
@@ -10,6 +10,5 @@ backend_type = ENV['BACKUP_BACKEND'] || 's3'
 s3_tool = ENV['S3_TOOL'] || 's3cmd'
 aws_s3_settings = ENV['AWS_S3_SETTINGS'] || ''
 aws_kms_settings = ENV['AWS_KMS_SETTINGS'] || ''
-azure_storage_account = ENV['AZURE_STORAGE_ACCOUNT'] || ''
-azure_blob_url = ENV['AZURE_BLOB_HOST'] || ''
-ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings, azure_storage_account, azure_blob_url).restore
+azure_config = ENV['AZURE_CONFIG_FILE'] || "/etc/gitlab/objectstorage/azure_config"
+ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings, azure_config).restore
diff --git a/gitlab-toolbox/scripts/lib/object_storage_azure.rb b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
index d55d599f4..83b09050b 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_azure.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
@@ -3,36 +3,38 @@ require 'fileutils'
 require 'openssl'
 require 'time'
 require 'uri'
+require 'yaml'
 
-class AzureSASTokenProvider
-
-  def initialize(storage_account,
+class AzureBackupUtil
+  def initialize(object_store_conf_file = '/etc/gitlab/objectstorage/azure_config',
                  token_expire_s = 12 * 60 * 60,
-                 token_file = '/tmp/azure_sas_token',
-                 access_key_file = '/etc/gitlab/objectstorage/azure_access_key')
-    @storage_account = storage_account
+                 sas_token_file = '/tmp/azure_sas_token')
+    @conf = YAML.load_file object_store_conf_file
+    @conf['azure_storage_domain'] = 'blob.core.windows.net' unless @conf.key?('azure_storage_domain')
+    @sas_token_file = sas_token_file
     @token_expire_s = token_expire_s
-    @token_file = token_file
-    @access_key_file = access_key_file
+  end
+
+  def url
+    "https://#{@conf['azure_storage_account_name']}.#{@conf['azure_storage_domain']}"
   end
 
   def sas_token
     # Read the cached token if it exist and within half its lifetime
-    if File.exist?(@token_file) && File.stat(@token_file).mtime >= (Time.now - (@token_expire_s / 2))
-      return open(@token_file, 'r').read
+    if File.exist?(@sas_token_file) && File.stat(@sas_token_file).mtime >= (Time.now - (@token_expire_s / 2))
+      return open(@sas_token_file, 'r').read
     end
 
-    access_key = open(@access_key_file, 'r').read
     start = (Time.now - 10 * 60).utc.iso8601
     expiry = (Time.now + @token_expire_s).utc.iso8601
-    token = generate(access_key, @storage_account, start, expiry)
-    open(@token_file, 'w').write token
+    token = generate_sas(@conf["azure_storage_access_key"], @conf["azure_storage_account_name"], start, expiry)
+    open(@sas_token_file, 'w').write token
     token
   end
 
   private
 
-  def generate(access_key, account, start_iso, expiry_iso)
+  def generate_sas(access_key, account, start_iso, expiry_iso)
     permissions = 'rawdl'  # read, write, delete, list | REMOVE ME: 'racwdl'
     protocol = 'https'     # allow https only
     service = 'b'          # blob storage
diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index a3ba27dd3..b033632d4 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -18,8 +18,7 @@ class ObjectStorageBackup
   private_constant :REGEXP_EXCLUDE
 
   def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3',
-                 s3_tool = 's3cmd', aws_s3_settings = '', aws_kms_settings = '',
-                 azure_storage_account = '', azure_blob_host = 'blob.core.windows.net')
+                 s3_tool = 's3cmd', aws_s3_settings = '', aws_kms_settings = '', azure_config_file = '')
     @name = name
     @local_tar_path = local_tar_path
     @remote_bucket_name = remote_bucket_name
@@ -28,8 +27,7 @@ class ObjectStorageBackup
     @s3_tool = s3_tool
     @aws_s3_settings = aws_s3_settings
     @aws_kms_settings = aws_kms_settings
-    @azure_token_prov = AzureSASTokenProvider.new(azure_storage_account)
-    @azure_base_url = "https://#{azure_storage_account}.#{azure_blob_host}"
+    @azure_util = AzureBackupUtil.new(azure_config_file)
   end
 
   def backup
@@ -45,9 +43,8 @@ class ObjectStorageBackup
       check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name})
       cmd = %W(gsutil -m rsync -d -x #{REGEXP_EXCLUDE} -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name})
     elsif @backend == "azure"
-      azure_token = @azure_token_prov.sas_token
-      check_bucket_cmd = %W(azcopy list #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token})
-      cmd = %W(azcopy sync #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token} /srv/gitlab/tmp/#{@name}
+      check_bucket_cmd = %W(azcopy list #{@azure_util.url}/#{@remote_bucket_name}?#{@azure_util.sas_token})
+      cmd = %W(azcopy sync #{@azure_util.url}/#{@remote_bucket_name}?#{@azure_util.sas_token} /srv/gitlab/tmp/#{@name}
                --output-level essential --recursive=true --exclude-regex=#{REGEXP_EXCLUDE} --delete-destination=true)
     end
 
@@ -107,8 +104,7 @@ class ObjectStorageBackup
     elsif @backend == "gcs"
       cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}/#{dir_name})
     elsif @backend == "azure"
-      azure_token = read_azure_token
-      cmd = %W(azcopy copy #{source_path} #{@azure_base_url}/?#{@remote_bucket_name}?#{azure_token} --output-level essential)
+      cmd = %W(azcopy copy #{source_path} #{@azure_util.url}/?#{@remote_bucket_name}?#{@azure_util.sas_token} --output-level essential)
     end
 
     output, status = run_cmd(cmd)
@@ -128,8 +124,8 @@ class ObjectStorageBackup
     elsif @backend == "gcs"
       cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/)
     elsif @backend == "azure"
-      azure_token = @azure_token_prov.sas_token
-      cmd = %W(azcopy copy #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token} #{@azure_base_url}/#{@tmp_bucket_name}/#{backup_file_name}?#{azure_token} --output-level essential)
+      cmd = %W(azcopy copy #{@azure_util.url}/#{@remote_bucket_name}?#{@azure_util.sas_token} #{@azure_base_url}/#{@tmp_bucket_name}/#{backup_file_name}?#{@azure_util.sas_token}
+               --output-level essential --recursive==true)
     end
 
     output, status = run_cmd(cmd)
@@ -157,8 +153,8 @@ class ObjectStorageBackup
 
       cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*)
     elsif @backend == 'azure'
-      azure_token = @azure_token_prov.sas_token
-      cmd = %W(azcopy remove #{@azure_base_url}/#{@remote_bucket_name}?#{azure_token} --output-level essential --recursive=true)
+      cmd = %W(azcopy remove #{@azure_util.url}/#{@remote_bucket_name}?#{@azure_util.sas_token}
+               --output-level essential --recursive=true)
     end
     output, status = run_cmd(cmd)
     failure_abort('bucket cleanup', output) unless status.zero?
-- 
GitLab


From 85d9bc113e49f6606b8f9bfa7db4d2cab6c7c573 Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Wed, 22 Feb 2023 17:31:22 +0100
Subject: [PATCH 29/42] Quiet output when fetching remote backup

---
 gitlab-toolbox/scripts/bin/backup-utility | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index c4fd75a27..7f27fc0a2 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -80,7 +80,8 @@ function fetch_remote_backup(){
         esac
         ;;
       gcs) gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null ;;
-      azure) azcopy copy "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${file_name}?$(get_azure_token)" ${output_path} --output-level essential ;;
+      azure) 
+        azcopy copy "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${file_name}?$(get_azure_token)" ${output_path} --output-level quiet ;;
       *) echo "Unknown backend: ${BACKUP_BACKEND}" ;;
     esac
   fi
@@ -112,11 +113,11 @@ function get_version(){
 }
 
 function get_azure_url(){
-  $(object-storage-azure-url)
+  echo -n $(object-storage-azure-url)
 }
 
 function get_azure_token(){
-  $(object-storage-azure-token)
+  echo -n $(object-storage-azure-token)
 }
 
 function get_backup_name(){
-- 
GitLab


From 04cc925fb0fea264c4c0f60941ee97f906963bcc Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Thu, 23 Feb 2023 10:20:26 +0100
Subject: [PATCH 30/42] Make use of azcopy sync

---
 gitlab-toolbox/scripts/lib/object_storage_backup.rb | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index b033632d4..f3bfc64d9 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -45,7 +45,7 @@ class ObjectStorageBackup
     elsif @backend == "azure"
       check_bucket_cmd = %W(azcopy list #{@azure_util.url}/#{@remote_bucket_name}?#{@azure_util.sas_token})
       cmd = %W(azcopy sync #{@azure_util.url}/#{@remote_bucket_name}?#{@azure_util.sas_token} /srv/gitlab/tmp/#{@name}
-               --output-level essential --recursive=true --exclude-regex=#{REGEXP_EXCLUDE} --delete-destination=true)
+               --output-level essential --exclude-regex=#{REGEXP_EXCLUDE} --delete-destination=true)
     end
 
     # Check if the bucket exists
@@ -104,7 +104,7 @@ class ObjectStorageBackup
     elsif @backend == "gcs"
       cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}/#{dir_name})
     elsif @backend == "azure"
-      cmd = %W(azcopy copy #{source_path} #{@azure_util.url}/?#{@remote_bucket_name}?#{@azure_util.sas_token} --output-level essential)
+      cmd = %W(azcopy sync #{source_path} #{@azure_util.url}/#{@remote_bucket_name}?#{@azure_util.sas_token} --output-level essential)
     end
 
     output, status = run_cmd(cmd)
@@ -124,8 +124,10 @@ class ObjectStorageBackup
     elsif @backend == "gcs"
       cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/)
     elsif @backend == "azure"
-      cmd = %W(azcopy copy #{@azure_util.url}/#{@remote_bucket_name}?#{@azure_util.sas_token} #{@azure_base_url}/#{@tmp_bucket_name}/#{backup_file_name}?#{@azure_util.sas_token}
-               --output-level essential --recursive==true)
+      cmd = %W(azcopy sync
+              #{@azure_util.url}/#{@remote_bucket_name}/?#{@azure_util.sas_token}
+              #{@azure_util.url}/#{@tmp_bucket_name}/#{backup_file_name}/?#{@azure_util.sas_token}
+               --output-level essential)
     end
 
     output, status = run_cmd(cmd)
-- 
GitLab


From 581b655b5e6731a6f6f7b17a6c0ace3060641f0a Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Tue, 7 Mar 2023 10:03:03 +0100
Subject: [PATCH 31/42] Refactor toolbox backend case distinction

---
 .../scripts/lib/object_storage_backup.rb      | 39 +++++++++++--------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index f3bfc64d9..c16ee86ce 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -31,7 +31,8 @@ class ObjectStorageBackup
   end
 
   def backup
-    if @backend == "s3"
+    case @backend
+    when 's3'
       if @s3_tool == "s3cmd"
         check_bucket_cmd = %W(s3cmd --limit=1 ls s3://#{@remote_bucket_name})
         cmd = %W(s3cmd --stop-on-error --delete-removed --exclude #{GLOB_EXCLUDE} sync s3://#{@remote_bucket_name}/ /srv/gitlab/tmp/#{@name}/)
@@ -39,15 +40,15 @@ class ObjectStorageBackup
         check_bucket_cmd = %W(aws s3api head-bucket --bucket #{@remote_bucket_name}) + @aws_s3_settings.split(" ")
         cmd = %W(aws s3 sync --delete --exclude #{GLOB_EXCLUDE} s3://#{@remote_bucket_name}/ /srv/gitlab/tmp/#{@name}/) + @aws_s3_settings.split(" ") + @aws_kms_settings.split(" ")
       end
-    elsif @backend == "gcs"
+    when 'gcs'
       check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name})
       cmd = %W(gsutil -m rsync -d -x #{REGEXP_EXCLUDE} -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name})
-    elsif @backend == "azure"
+    when 'azure'
       check_bucket_cmd = %W(azcopy list #{@azure_util.url}/#{@remote_bucket_name}?#{@azure_util.sas_token})
       cmd = %W(azcopy sync #{@azure_util.url}/#{@remote_bucket_name}?#{@azure_util.sas_token} /srv/gitlab/tmp/#{@name}
-               --output-level essential --exclude-regex=#{REGEXP_EXCLUDE} --delete-destination=true)
+              --output-level essential --exclude-regex=#{REGEXP_EXCLUDE} --delete-destination=true)
     end
-
+ 
     # Check if the bucket exists
     output, status = run_cmd(check_bucket_cmd)
     unless status.zero?
@@ -95,15 +96,17 @@ class ObjectStorageBackup
 
   def upload_to_object_storage(source_path)
     dir_name = File.basename(source_path)
-    if @backend == "s3"
+
+    case @backend
+    when 's3'
       if @s3_tool == "s3cmd"
         cmd = %W(s3cmd --stop-on-error sync #{source_path}/ s3://#{@remote_bucket_name}/#{dir_name}/)
       elsif @s3_tool == "awscli"
         cmd = %W(aws s3 sync #{source_path}/ s3://#{@remote_bucket_name}/#{dir_name}/) + @aws_s3_settings.split(" ") + @aws_kms_settings.split(" ")
       end
-    elsif @backend == "gcs"
+    when 'gcs'
       cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}/#{dir_name})
-    elsif @backend == "azure"
+    when 'azure'
       cmd = %W(azcopy sync #{source_path} #{@azure_util.url}/#{@remote_bucket_name}?#{@azure_util.sas_token} --output-level essential)
     end
 
@@ -115,34 +118,35 @@ class ObjectStorageBackup
   def backup_existing
     backup_file_name = "#{@name}.#{Time.now.to_i}"
 
-    if @backend == "s3"
+    case @backend
+    when 's3'
       if @s3_tool == "s3cmd"
         cmd = %W(s3cmd sync s3://#{@remote_bucket_name} s3://#{@tmp_bucket_name}/#{backup_file_name}/)
       elsif @s3_tool == "awscli"
         cmd = %W(aws s3 sync s3://#{@remote_bucket_name} s3://#{@tmp_bucket_name}/#{backup_file_name}/) + @aws_s3_settings.split(" ") + @aws_kms_settings.split(" ")
       end
-    elsif @backend == "gcs"
+    when 'gcs'
       cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/)
-    elsif @backend == "azure"
+    when 'azure'
       cmd = %W(azcopy sync
-              #{@azure_util.url}/#{@remote_bucket_name}/?#{@azure_util.sas_token}
-              #{@azure_util.url}/#{@tmp_bucket_name}/#{backup_file_name}/?#{@azure_util.sas_token}
+               #{@azure_util.url}/#{@remote_bucket_name}/?#{@azure_util.sas_token}
+               #{@azure_util.url}/#{@tmp_bucket_name}/#{backup_file_name}/?#{@azure_util.sas_token}
                --output-level essential)
     end
 
     output, status = run_cmd(cmd)
-
     failure_abort('sync existing', output) unless status.zero?
   end
 
   def cleanup
-    if @backend == "s3"
+    case @backend
+    when 's3'
       if @s3_tool == "s3cmd"
         cmd = %W(s3cmd --stop-on-error del --force --recursive s3://#{@remote_bucket_name})
       elsif @s3_tool == "awscli"
         cmd = %W(aws s3 rm --recursive s3://#{@remote_bucket_name}) + @aws_s3_settings.split(" ")
       end
-    elsif @backend == "gcs"
+    when 'gcs'
       # Check if the bucket has any objects
       list_objects_cmd = %W(gsutil ls gs://#{@remote_bucket_name}/)
       output, status = run_cmd(list_objects_cmd)
@@ -154,10 +158,11 @@ class ObjectStorageBackup
       end
 
       cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*)
-    elsif @backend == 'azure'
+    when 'azure'
       cmd = %W(azcopy remove #{@azure_util.url}/#{@remote_bucket_name}?#{@azure_util.sas_token}
                --output-level essential --recursive=true)
     end
+
     output, status = run_cmd(cmd)
     failure_abort('bucket cleanup', output) unless status.zero?
   end
-- 
GitLab


From d77925efd5959fade93a03563fc14b6a5153e909 Mon Sep 17 00:00:00 2001
From: Stan Hu <stanhu@gmail.com>
Date: Tue, 7 Mar 2023 09:15:02 +0000
Subject: [PATCH 32/42] Add explicit permissions to Azure token file

---
 gitlab-toolbox/scripts/lib/object_storage_azure.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_azure.rb b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
index 83b09050b..53e543304 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_azure.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
@@ -28,7 +28,7 @@ class AzureBackupUtil
     start = (Time.now - 10 * 60).utc.iso8601
     expiry = (Time.now + @token_expire_s).utc.iso8601
     token = generate_sas(@conf["azure_storage_access_key"], @conf["azure_storage_account_name"], start, expiry)
-    open(@sas_token_file, 'w').write token
+    open(@sas_token_file, 'w', 0600).write token
     token
   end
 
-- 
GitLab


From f649dbbad51fe4a22b930d5873eeed15ee3a50c9 Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Tue, 7 Mar 2023 12:05:04 +0100
Subject: [PATCH 33/42] Add verification to azure backup config

---
 .../scripts/lib/object_storage_azure.rb       | 28 +++++++++++++++----
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_azure.rb b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
index 53e543304..f4ad3e75e 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_azure.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
@@ -6,17 +6,33 @@ require 'uri'
 require 'yaml'
 
 class AzureBackupUtil
-  def initialize(object_store_conf_file = '/etc/gitlab/objectstorage/azure_config',
+  class AzureConfigInvalidError < StandardError
+    def initialize(config_file, key)
+      super "Azure config file (#{config_file}) needs a valid #{key}"
+    end
+  end
+
+  def initialize(azure_config_file = 'etc/gitlab/objectstorage/azure_config',
                  token_expire_s = 12 * 60 * 60,
                  sas_token_file = '/tmp/azure_sas_token')
-    @conf = YAML.load_file object_store_conf_file
-    @conf['azure_storage_domain'] = 'blob.core.windows.net' unless @conf.key?('azure_storage_domain')
+    @config = load_config azure_config_file
     @sas_token_file = sas_token_file
     @token_expire_s = token_expire_s
   end
 
+  def load_config(config_file)
+    conf = YAML.load_file config_file
+    mandatory_keys = %w(azure_storage_account_name azure_storage_access_key)
+    mandatory_keys.each do |key|
+      raise AzureConfigInvalidError.new(config_file, key) if conf[key].nil?
+    end
+
+    conf['azure_storage_domain'] = 'blob.core.windows.net' if conf['azure_storage_domain'].nil?
+    conf
+  end
+
   def url
-    "https://#{@conf['azure_storage_account_name']}.#{@conf['azure_storage_domain']}"
+    "https://#{@config['azure_storage_account_name'].to_s.strip}.#{@config['azure_storage_domain'].to_s.strip}"
   end
 
   def sas_token
@@ -27,7 +43,7 @@ class AzureBackupUtil
 
     start = (Time.now - 10 * 60).utc.iso8601
     expiry = (Time.now + @token_expire_s).utc.iso8601
-    token = generate_sas(@conf["azure_storage_access_key"], @conf["azure_storage_account_name"], start, expiry)
+    token = generate_sas(@config['azure_storage_access_key'], @config['azure_storage_account_name'], start, expiry)
     open(@sas_token_file, 'w', 0600).write token
     token
   end
@@ -35,7 +51,7 @@ class AzureBackupUtil
   private
 
   def generate_sas(access_key, account, start_iso, expiry_iso)
-    permissions = 'rawdl'  # read, write, delete, list | REMOVE ME: 'racwdl'
+    permissions = 'rawdl'  # read, access, write, delete, list
     protocol = 'https'     # allow https only
     service = 'b'          # blob storage
     resources = 'co'       # container + object level
-- 
GitLab


From 41366683bc29df4b2bc2f720092455480ceefdc0 Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Tue, 7 Mar 2023 23:58:26 +0100
Subject: [PATCH 34/42] Add newlines and oxford comma

---
 gitlab-toolbox/scripts/bin/backup-utility             | 4 ++--
 gitlab-toolbox/scripts/bin/object-storage-azure-token | 2 +-
 gitlab-toolbox/scripts/bin/object-storage-azure-url   | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index 7f27fc0a2..76c47b98b 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -37,12 +37,12 @@ function usage()
                                             May be defined multiple times. Valid values for COMPONENT are
                                             db, repositories, and any of the object storages (e.g. 'lfs').
      --backend BACKEND                      Object storage backend to use for backups.
-                                            Can be either 's3', 'gcs' or 'azure'.
+                                            Can be either 's3', 'gcs', or 'azure'.
      --s3config CONFIG                      S3 backend configuration to use for backups storage.
                                             Special config file for s3cmd (see: https://s3tools.org/usage)
                                             Not required when using the awscli tool.
      --s3tool TOOL                          S3 CLI tool to use. Can be either 's3cmd' or 'awscli'.
-     --storage-class CLASSNAME              Pass this storage class to the gcs, s3cmd, aws or azcopy cli for more
+     --storage-class CLASSNAME              Pass this storage class to the gcs, s3cmd, aws, or azcopy cli for more
                                             cost-efficient storage of backups.
      --maximum-backups N                    Only keep the most recent N number of backups, deleting others after success.
                                             Requires s3config or AWS credentials to be able to list and delete objects.
diff --git a/gitlab-toolbox/scripts/bin/object-storage-azure-token b/gitlab-toolbox/scripts/bin/object-storage-azure-token
index 27c0179a4..7fb9909f9 100755
--- a/gitlab-toolbox/scripts/bin/object-storage-azure-token
+++ b/gitlab-toolbox/scripts/bin/object-storage-azure-token
@@ -3,4 +3,4 @@
 require 'object_storage_azure'
 
 config = ENV['AZURE_CONFIG_FILE'] || "/etc/gitlab/objectstorage/azure_config"
-puts AzureBackupUtil.new(config).sas_token
\ No newline at end of file
+puts AzureBackupUtil.new(config).sas_token
diff --git a/gitlab-toolbox/scripts/bin/object-storage-azure-url b/gitlab-toolbox/scripts/bin/object-storage-azure-url
index e9f636dcf..c1f03b253 100755
--- a/gitlab-toolbox/scripts/bin/object-storage-azure-url
+++ b/gitlab-toolbox/scripts/bin/object-storage-azure-url
@@ -3,4 +3,4 @@
 require 'object_storage_azure'
 
 config = ENV['AZURE_CONFIG_FILE'] || "/etc/gitlab/objectstorage/azure_config"
-puts AzureBackupUtil.new(config).url
\ No newline at end of file
+puts AzureBackupUtil.new(config).url
-- 
GitLab


From 1b9b4a9e9d3c6c70fee8f542df7b16e5779061a4 Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Tue, 7 Mar 2023 23:59:15 +0100
Subject: [PATCH 35/42] Fix default Azure config path

---
 gitlab-toolbox/scripts/lib/object_storage_azure.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_azure.rb b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
index f4ad3e75e..ee2a79791 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_azure.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
@@ -12,7 +12,7 @@ class AzureBackupUtil
     end
   end
 
-  def initialize(azure_config_file = 'etc/gitlab/objectstorage/azure_config',
+  def initialize(azure_config_file = '/etc/gitlab/objectstorage/azure_config',
                  token_expire_s = 12 * 60 * 60,
                  sas_token_file = '/tmp/azure_sas_token')
     @config = load_config azure_config_file
-- 
GitLab


From 245a01ecfa1acdbe6d7ffab57fde966a07adf39c Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Wed, 8 Mar 2023 01:27:14 +0100
Subject: [PATCH 36/42] Refactor Azure SAS token caching

Store/cache the Azure SAS token in a
Ruby variable instead of a file to
prevent access/modification by other
processes.
---
 .../scripts/lib/object_storage_azure.rb       | 29 ++++++++++---------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_azure.rb b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
index ee2a79791..f721d2ced 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_azure.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
@@ -18,6 +18,7 @@ class AzureBackupUtil
     @config = load_config azure_config_file
     @sas_token_file = sas_token_file
     @token_expire_s = token_expire_s
+    @current_token = {}
   end
 
   def load_config(config_file)
@@ -36,21 +37,21 @@ class AzureBackupUtil
   end
 
   def sas_token
-    # Read the cached token if it exist and within half its lifetime
-    if File.exist?(@sas_token_file) && File.stat(@sas_token_file).mtime >= (Time.now - (@token_expire_s / 2))
-      return open(@sas_token_file, 'r').read
+    if @current_token['token'].nil? ||
+       @current_token['expiry'].nil? ||
+       @current_token['expiry'] - (@token_expire_s / 2) < Time.now
+      # refresh token
+      start = Time.now - 10 * 60
+      expiry = Time.now + @token_expire_s
+      @current_token = generate_sas(@config['azure_storage_access_key'], @config['azure_storage_account_name'],
+                                    start, expiry)
     end
-
-    start = (Time.now - 10 * 60).utc.iso8601
-    expiry = (Time.now + @token_expire_s).utc.iso8601
-    token = generate_sas(@config['azure_storage_access_key'], @config['azure_storage_account_name'], start, expiry)
-    open(@sas_token_file, 'w', 0600).write token
-    token
+    @current_token['token']
   end
 
   private
 
-  def generate_sas(access_key, account, start_iso, expiry_iso)
+  def generate_sas(access_key, account, start, expiry)
     permissions = 'rawdl'  # read, access, write, delete, list
     protocol = 'https'     # allow https only
     service = 'b'          # blob storage
@@ -64,8 +65,8 @@ class AzureBackupUtil
       permissions,
       service,
       resources,
-      start_iso,
-      expiry_iso,
+      start.utc.iso8601,
+      expiry.utc.iso8601,
       ip_range,
       protocol,
       version,
@@ -75,6 +76,8 @@ class AzureBackupUtil
     sig = OpenSSL::HMAC.digest('sha256', Base64.decode64(access_key), to_sign)
     sig = Base64.strict_encode64(sig)
     sig = URI.encode_www_form_component(sig)
-    "sv=#{version}&ss=#{service}&srt=#{resources}&sp=#{permissions}&se=#{expiry_iso}&st=#{start_iso}&spr=#{protocol}&sig=#{sig}"
+    token = %W(v=#{version}&ss=#{service}&srt=#{resources}&sp=#{permissions}
+               &se=#{expiry.utc.iso8601}&st=#{start.utc.iso8601}&spr=#{protocol}&sig=#{sig})
+    { 'token' => token, 'start' => start, 'expiry' => expiry }
   end
 end
-- 
GitLab


From c9a3c0d9fe641ccf26becf4b1b8980894401f941 Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Thu, 9 Mar 2023 13:18:20 +0100
Subject: [PATCH 37/42] Fix Azure token assembly

---
 gitlab-toolbox/scripts/lib/object_storage_azure.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_azure.rb b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
index f721d2ced..d7e809027 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_azure.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
@@ -76,8 +76,8 @@ class AzureBackupUtil
     sig = OpenSSL::HMAC.digest('sha256', Base64.decode64(access_key), to_sign)
     sig = Base64.strict_encode64(sig)
     sig = URI.encode_www_form_component(sig)
-    token = %W(v=#{version}&ss=#{service}&srt=#{resources}&sp=#{permissions}
-               &se=#{expiry.utc.iso8601}&st=#{start.utc.iso8601}&spr=#{protocol}&sig=#{sig})
+    token = "sv=#{version}&ss=#{service}&srt=#{resources}&sp=#{permissions}"\
+      "&se=#{expiry.utc.iso8601}&st=#{start.utc.iso8601}&spr=#{protocol}&sig=#{sig}"
     { 'token' => token, 'start' => start, 'expiry' => expiry }
   end
 end
-- 
GitLab


From 0112aa9d9fccf7f5d9cb9abdf39b10b7e5b945a6 Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Fri, 10 Mar 2023 10:07:43 +0100
Subject: [PATCH 38/42] Load azure config safely

---
 gitlab-toolbox/scripts/lib/object_storage_azure.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_azure.rb b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
index d7e809027..c0273285f 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_azure.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
@@ -22,7 +22,7 @@ class AzureBackupUtil
   end
 
   def load_config(config_file)
-    conf = YAML.load_file config_file
+    conf = YAML.safe_load_file config_file
     mandatory_keys = %w(azure_storage_account_name azure_storage_access_key)
     mandatory_keys.each do |key|
       raise AzureConfigInvalidError.new(config_file, key) if conf[key].nil?
-- 
GitLab


From d4099ab6a1bb37f41816975526dd1bbf6583a780 Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Fri, 10 Mar 2023 10:08:18 +0100
Subject: [PATCH 39/42] Refactor Azure util to be created on demand

---
 .../scripts/lib/object_storage_backup.rb      | 20 ++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index c16ee86ce..750e18c68 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -27,7 +27,7 @@ class ObjectStorageBackup
     @s3_tool = s3_tool
     @aws_s3_settings = aws_s3_settings
     @aws_kms_settings = aws_kms_settings
-    @azure_util = AzureBackupUtil.new(azure_config_file)
+    @azure_config_file = azure_config_file
   end
 
   def backup
@@ -44,8 +44,8 @@ class ObjectStorageBackup
       check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name})
       cmd = %W(gsutil -m rsync -d -x #{REGEXP_EXCLUDE} -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name})
     when 'azure'
-      check_bucket_cmd = %W(azcopy list #{@azure_util.url}/#{@remote_bucket_name}?#{@azure_util.sas_token})
-      cmd = %W(azcopy sync #{@azure_util.url}/#{@remote_bucket_name}?#{@azure_util.sas_token} /srv/gitlab/tmp/#{@name}
+      check_bucket_cmd = %W(azcopy list #{azure_util.url}/#{@remote_bucket_name}?#{azure_util.sas_token})
+      cmd = %W(azcopy sync #{azure_util.url}/#{@remote_bucket_name}?#{azure_util.sas_token} /srv/gitlab/tmp/#{@name}
               --output-level essential --exclude-regex=#{REGEXP_EXCLUDE} --delete-destination=true)
     end
  
@@ -107,7 +107,7 @@ class ObjectStorageBackup
     when 'gcs'
       cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}/#{dir_name})
     when 'azure'
-      cmd = %W(azcopy sync #{source_path} #{@azure_util.url}/#{@remote_bucket_name}?#{@azure_util.sas_token} --output-level essential)
+      cmd = %W(azcopy sync #{source_path} #{azure_util.url}/#{@remote_bucket_name}?#{azure_util.sas_token} --output-level essential)
     end
 
     output, status = run_cmd(cmd)
@@ -129,8 +129,8 @@ class ObjectStorageBackup
       cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/)
     when 'azure'
       cmd = %W(azcopy sync
-               #{@azure_util.url}/#{@remote_bucket_name}/?#{@azure_util.sas_token}
-               #{@azure_util.url}/#{@tmp_bucket_name}/#{backup_file_name}/?#{@azure_util.sas_token}
+               #{azure_util.url}/#{@remote_bucket_name}/?#{azure_util.sas_token}
+               #{azure_util.url}/#{@tmp_bucket_name}/#{backup_file_name}/?#{azure_util.sas_token}
                --output-level essential)
     end
 
@@ -159,7 +159,7 @@ class ObjectStorageBackup
 
       cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*)
     when 'azure'
-      cmd = %W(azcopy remove #{@azure_util.url}/#{@remote_bucket_name}?#{@azure_util.sas_token}
+      cmd = %W(azcopy remove #{azure_util.url}/#{@remote_bucket_name}?#{azure_util.sas_token}
                --output-level essential --recursive=true)
     end
 
@@ -188,4 +188,10 @@ class ObjectStorageBackup
     _, stdout, wait_thr = Open3.popen2e(*cmd)
     return stdout.read, wait_thr.value.exitstatus
   end
+
+  private
+
+  def azure_util
+    @azure_util ||= AzureBackupUtil.new @azure_config_file
+  end
 end
-- 
GitLab


From bd370e90f251ffacf66c84549927e2707425512b Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Mon, 13 Mar 2023 15:19:06 +0100
Subject: [PATCH 40/42] Make internal methods private

---
 .../scripts/lib/object_storage_azure.rb       | 22 +++++++++----------
 .../scripts/lib/object_storage_backup.rb      |  4 ++--
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/gitlab-toolbox/scripts/lib/object_storage_azure.rb b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
index c0273285f..b244fb723 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_azure.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_azure.rb
@@ -21,17 +21,6 @@ class AzureBackupUtil
     @current_token = {}
   end
 
-  def load_config(config_file)
-    conf = YAML.safe_load_file config_file
-    mandatory_keys = %w(azure_storage_account_name azure_storage_access_key)
-    mandatory_keys.each do |key|
-      raise AzureConfigInvalidError.new(config_file, key) if conf[key].nil?
-    end
-
-    conf['azure_storage_domain'] = 'blob.core.windows.net' if conf['azure_storage_domain'].nil?
-    conf
-  end
-
   def url
     "https://#{@config['azure_storage_account_name'].to_s.strip}.#{@config['azure_storage_domain'].to_s.strip}"
   end
@@ -51,6 +40,17 @@ class AzureBackupUtil
 
   private
 
+  def load_config(config_file)
+    conf = YAML.safe_load_file config_file
+    mandatory_keys = %w(azure_storage_account_name azure_storage_access_key)
+    mandatory_keys.each do |key|
+      raise AzureConfigInvalidError.new(config_file, key) if conf[key].nil?
+    end
+
+    conf['azure_storage_domain'] = 'blob.core.windows.net' if conf['azure_storage_domain'].nil?
+    conf
+  end
+
   def generate_sas(access_key, account, start, expiry)
     permissions = 'rawdl'  # read, access, write, delete, list
     protocol = 'https'     # allow https only
diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
index 750e18c68..f3e0c9f16 100644
--- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb
+++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb
@@ -89,6 +89,8 @@ class ObjectStorageBackup
     puts "done".green
   end
 
+  private
+
   def failure_abort(action, error_message)
     puts "[Error] #{error_message}".red
     abort "#{action} of #{@name} failed"
@@ -189,8 +191,6 @@ class ObjectStorageBackup
     return stdout.read, wait_thr.value.exitstatus
   end
 
-  private
-
   def azure_util
     @azure_util ||= AzureBackupUtil.new @azure_config_file
   end
-- 
GitLab


From 80e8709717cc5439618c13dc81d086c3fc9b9673 Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Thu, 16 Mar 2023 10:54:29 +0100
Subject: [PATCH 41/42] Show logs of remote backup retrieval

The 'azopy' CLI prints all logs to stdout
which results in poor traceability of errors.

Stdout is now logged for all CLI on
backup fetches to provide more detailed insight.
---
 gitlab-toolbox/scripts/bin/backup-utility | 28 +++++++++++++++--------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index 76c47b98b..5680d2dda 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -63,29 +63,29 @@ function is_url() {
 }
 
 function fetch_remote_backup(){
-  mkdir -p $backups_path
-  output_path=$backups_path/0_gitlab_backup.tar
+  backup_source=$1
+  output_path=$2
 
-  if is_url $1; then
+  if is_url $backup_source; then
     >&2 echo "Downloading from $1";
     curl -f --retry 6 --progress-bar -o $output_path $1
   else # It's a timestamp
-    file_name="$1_gitlab_backup.tar"
+    file_name="${backup_source}_gitlab_backup.tar"
+    echo "Fetching ${BACKUP_BUCKET_NAME}/${file_name} from object storage"
     case "$BACKUP_BACKEND" in
       s3)
         case "$S3_TOOL" in
-          s3cmd) s3cmd ${S3_CMD_BACKUP_OPTION} get "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null ;;
-          awscli) aws s3 cp "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path ${AWS_S3_SETTINGS_LIST[@]} ${AWS_KMS_SETTINGS_LIST[@]} > /dev/null ;;
+          s3cmd) s3cmd ${S3_CMD_BACKUP_OPTION} get "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path ;;
+          awscli) aws s3 cp "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path ${AWS_S3_SETTINGS_LIST[@]} ${AWS_KMS_SETTINGS_LIST[@]} ;;
           *) echo "Unknown S3 tool: ${S3_TOOL}" ;;
         esac
         ;;
-      gcs) gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null ;;
+      gcs) gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path ;;
       azure) 
-        azcopy copy "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${file_name}?$(get_azure_token)" ${output_path} --output-level quiet ;;
+        azcopy copy "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${file_name}?$(get_azure_token)" ${output_path} --output-level essential ;;
       *) echo "Unknown backend: ${BACKUP_BACKEND}" ;;
     esac
   fi
-  echo $output_path
 }
 
 function unpack_backup(){
@@ -316,7 +316,15 @@ function restore(){
     BACKUP=$BACKUP_TIMESTAMP
   fi
 
-  file=$(fetch_remote_backup $BACKUP)
+  mkdir -p $backups_path
+  file=$backups_path/0_gitlab_backup.tar
+  
+  fetch_remote_backup $BACKUP $file
+
+  if [[ ! -f $file ]]; then
+    echo "Failed to download remote file"
+    exit 1
+  fi
 
   dir_name=$(dirname $file)
   file_name=$(basename $file)
-- 
GitLab


From 6cd7b6099ee35d74a26e339a9a1aa2b5cfa63bce Mon Sep 17 00:00:00 2001
From: Clemens Beck <cbeck@gitlab.com>
Date: Thu, 16 Mar 2023 22:15:30 +0100
Subject: [PATCH 42/42] Revert "Show logs of remote backup retrieval"

This reverts commit 80e8709717cc5439618c13dc81d086c3fc9b9673.
---
 gitlab-toolbox/scripts/bin/backup-utility | 28 ++++++++---------------
 1 file changed, 10 insertions(+), 18 deletions(-)

diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility
index 5680d2dda..76c47b98b 100755
--- a/gitlab-toolbox/scripts/bin/backup-utility
+++ b/gitlab-toolbox/scripts/bin/backup-utility
@@ -63,29 +63,29 @@ function is_url() {
 }
 
 function fetch_remote_backup(){
-  backup_source=$1
-  output_path=$2
+  mkdir -p $backups_path
+  output_path=$backups_path/0_gitlab_backup.tar
 
-  if is_url $backup_source; then
+  if is_url $1; then
     >&2 echo "Downloading from $1";
     curl -f --retry 6 --progress-bar -o $output_path $1
   else # It's a timestamp
-    file_name="${backup_source}_gitlab_backup.tar"
-    echo "Fetching ${BACKUP_BUCKET_NAME}/${file_name} from object storage"
+    file_name="$1_gitlab_backup.tar"
     case "$BACKUP_BACKEND" in
       s3)
         case "$S3_TOOL" in
-          s3cmd) s3cmd ${S3_CMD_BACKUP_OPTION} get "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path ;;
-          awscli) aws s3 cp "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path ${AWS_S3_SETTINGS_LIST[@]} ${AWS_KMS_SETTINGS_LIST[@]} ;;
+          s3cmd) s3cmd ${S3_CMD_BACKUP_OPTION} get "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null ;;
+          awscli) aws s3 cp "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path ${AWS_S3_SETTINGS_LIST[@]} ${AWS_KMS_SETTINGS_LIST[@]} > /dev/null ;;
           *) echo "Unknown S3 tool: ${S3_TOOL}" ;;
         esac
         ;;
-      gcs) gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path ;;
+      gcs) gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null ;;
       azure) 
-        azcopy copy "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${file_name}?$(get_azure_token)" ${output_path} --output-level essential ;;
+        azcopy copy "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${file_name}?$(get_azure_token)" ${output_path} --output-level quiet ;;
       *) echo "Unknown backend: ${BACKUP_BACKEND}" ;;
     esac
   fi
+  echo $output_path
 }
 
 function unpack_backup(){
@@ -316,15 +316,7 @@ function restore(){
     BACKUP=$BACKUP_TIMESTAMP
   fi
 
-  mkdir -p $backups_path
-  file=$backups_path/0_gitlab_backup.tar
-  
-  fetch_remote_backup $BACKUP $file
-
-  if [[ ! -f $file ]]; then
-    echo "Failed to download remote file"
-    exit 1
-  fi
+  file=$(fetch_remote_backup $BACKUP)
 
   dir_name=$(dirname $file)
   file_name=$(basename $file)
-- 
GitLab