diff --git a/gitlab-toolbox/Dockerfile.ubi8 b/gitlab-toolbox/Dockerfile.ubi8 index 3ad76db25601d161fa0e91b59268f622b7ce2720..c3e94d0ef9f21a8e4ec2a8e1c7e9c7f020688d95 100644 --- a/gitlab-toolbox/Dockerfile.ubi8 +++ b/gitlab-toolbox/Dockerfile.ubi8 @@ -17,7 +17,7 @@ LABEL source="https://gitlab.com/gitlab-org/build/CNG/-/tree/master/gitlab-toolb version=${GITLAB_VERSION} \ release=${GITLAB_VERSION} \ summary="Toolbox is an entry point for interaction with other containers in the cluster." \ - description="Toolbox is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage." + description="Toolbox is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to interact with object storage." ADD gitlab-toolbox-ee.tar.gz / ADD gitlab-python.tar.gz / diff --git a/gitlab-toolbox/scripts/bin/backup-utility b/gitlab-toolbox/scripts/bin/backup-utility index 759078a7f9d4e7b0f42a867e2c9ab1c4ba007d51..76c47b98b17efba92a96f2d375952fa2a30013a9 100755 --- a/gitlab-toolbox/scripts/bin/backup-utility +++ b/gitlab-toolbox/scripts/bin/backup-utility @@ -7,6 +7,7 @@ export BACKUP_BACKEND=${BACKUP_BACKEND-s3} export S3_TOOL=${S3_TOOL-s3cmd} export AWS_KMS_SETTINGS="" export AWS_S3_SETTINGS="" +export AZURE_CONFIG_FILE="/etc/gitlab/objectstorage/azure_config" AWS_KMS_SETTINGS_LIST=() AWS_S3_SETTINGS_LIST=() S3_CMD_BACKUP_OPTION="" @@ -36,20 +37,21 @@ function usage() May be defined multiple times. Valid values for COMPONENT are db, repositories, and any of the object storages (e.g. 'lfs'). --backend BACKEND Object storage backend to use for backups. - Can be either 's3' or 'gcs'. + Can be either 's3', 'gcs', or 'azure'. --s3config CONFIG S3 backend configuration to use for backups storage. Special config file for s3cmd (see: https://s3tools.org/usage) Not required when using the awscli tool. --s3tool TOOL S3 CLI tool to use. Can be either 's3cmd' or 'awscli'. - --storage-class CLASSNAME Pass this storage class to the gcs, s3cmd or aws cli for more cost-efficient - storage of backups. + --storage-class CLASSNAME Pass this storage class to the gcs, s3cmd, aws, or azcopy cli for more + cost-efficient storage of backups. --maximum-backups N Only keep the most recent N number of backups, deleting others after success. Requires s3config or AWS credentials to be able to list and delete objects. --cleanup Run the backup cleanup without creating a new backup. Can be used with the 'maximum-backups' option to clean old remote backups. - --aws-kms-key-id Add KMS key id when S3 bucket is encrypted with a customer key. - --aws-s3-endpoint-url Specify an AWS S3 endpoint URL - --aws-region Add AWS region (required for AWS STS regionalized endpoint) + --aws-kms-key-id Add KMS key id when S3 bucket is encrypted with a customer key. + --aws-s3-endpoint-url Specify an AWS S3 endpoint URL. + --aws-region Add AWS region (required for AWS STS regionalized endpoint). + --azure-config-file Path of the config file to configure Azure Block Storage access. HEREDOC } @@ -78,6 +80,8 @@ function fetch_remote_backup(){ esac ;; gcs) gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null ;; + azure) + azcopy copy "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${file_name}?$(get_azure_token)" ${output_path} --output-level quiet ;; *) echo "Unknown backend: ${BACKUP_BACKEND}" ;; esac fi @@ -108,6 +112,14 @@ function get_version(){ cat $rails_dir/VERSION } +function get_azure_url(){ + echo -n $(object-storage-azure-url) +} + +function get_azure_token(){ + echo -n $(object-storage-azure-token) +} + function get_backup_name(){ if [ -n "$BACKUP_TIMESTAMP" ]; then echo ${BACKUP_TIMESTAMP}_gitlab_backup @@ -141,6 +153,9 @@ function get_existing_backups(){ # https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames#other-wildcard-characters existing_backups=($(gsutil ls gs://$BACKUP_BUCKET_NAME/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_[0-9][0-9][0-9][0-9]_[0-9][0-9]_[0-9][0-9]_\*_gitlab_backup.tar | LC_ALL=C sort)) ;; + azure) + existing_backups=($(azcopy list "$(get_azure_url)/${BACKUP_BUCKET_NAME}?$(get_azure_token)" | awk '/^INFO: [[:alnum:]_\.\-]+;/ {gsub(/;$/, "", $2); print $2}' | grep -E '^[0-9]{10}_[0-9]{4}_[0-9]{2}_[0-9]{2}_.+_gitlab_backup.tar$' | LC_ALL=C sort)) + ;; *) echo "Unknown backend for backup: ${BACKUP_BACKEND}" exit 1 @@ -162,6 +177,7 @@ function remove_backup(){ esac ;; gcs) gsutil rm ${backup_to_remove} > /dev/null ;; + azure) azcopy remove "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${backup_to_remove}?$(get_azure_token)" --output-level essential ;; *) echo "Unknown backend for backup: ${BACKUP_BACKEND}" exit 1 @@ -269,6 +285,16 @@ function backup(){ fi echo "[DONE] Backup can be found at gs://$BACKUP_BUCKET_NAME/${backup_name}.tar" ;; + azure) + if [ -z "${STORAGE_CLASS}" ]; then + azcopy copy "${backup_tars_path}/${backup_name}.tar" --output-level essential \ + "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" + else + azcopy copy --block-blob-tier "${STORAGE_CLASS}" --output-level essential \ + "${backup_tars_path}/${backup_name}.tar" "$(get_azure_url)/${BACKUP_BUCKET_NAME}/${backup_name}.tar?$(get_azure_token)" + fi + echo "[DONE] Backup can be found at ${AZURE_BASE_URL}/${BACKUP_BUCKET_NAME}/${backup_name}.tar" + ;; *) echo "Unknown backend for backup: ${BACKUP_BACKEND}" ;; esac @@ -416,12 +442,17 @@ do AWS_S3_SETTINGS_LIST+=(--endpoint-url $2) shift shift - ;; + ;; --aws-region) AWS_REGION=$2 shift shift - ;; + ;; + --azure-config-file) + export AZURE_CONFIG_FILE=$2 + shift + shift + ;; *) usage echo "Unexpected parameter: $key" diff --git a/gitlab-toolbox/scripts/bin/object-storage-azure-token b/gitlab-toolbox/scripts/bin/object-storage-azure-token new file mode 100755 index 0000000000000000000000000000000000000000..7fb9909f9f7b4eeea0091d5d896c473b6b241b37 --- /dev/null +++ b/gitlab-toolbox/scripts/bin/object-storage-azure-token @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +require 'object_storage_azure' + +config = ENV['AZURE_CONFIG_FILE'] || "/etc/gitlab/objectstorage/azure_config" +puts AzureBackupUtil.new(config).sas_token diff --git a/gitlab-toolbox/scripts/bin/object-storage-azure-url b/gitlab-toolbox/scripts/bin/object-storage-azure-url new file mode 100755 index 0000000000000000000000000000000000000000..c1f03b253d1b421f376a18ec00d298d37164c4d4 --- /dev/null +++ b/gitlab-toolbox/scripts/bin/object-storage-azure-url @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +require 'object_storage_azure' + +config = ENV['AZURE_CONFIG_FILE'] || "/etc/gitlab/objectstorage/azure_config" +puts AzureBackupUtil.new(config).url diff --git a/gitlab-toolbox/scripts/bin/object-storage-backup b/gitlab-toolbox/scripts/bin/object-storage-backup index 0ff9d638afe548940703fbc9d0af244e5fbc9e6f..25c1ce0e7cfbbc5e79b46baff92bd1cc379955d9 100755 --- a/gitlab-toolbox/scripts/bin/object-storage-backup +++ b/gitlab-toolbox/scripts/bin/object-storage-backup @@ -10,4 +10,5 @@ backend_type = ENV['BACKUP_BACKEND'] || 's3' s3_tool = ENV['S3_TOOL'] || 's3cmd' aws_s3_settings = ENV['AWS_S3_SETTINGS'] || '' aws_kms_settings = ENV['AWS_KMS_SETTINGS'] || '' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings).backup +azure_config = ENV['AZURE_CONFIG_FILE'] || "/etc/gitlab/objectstorage/azure_config" +ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings, azure_config).backup diff --git a/gitlab-toolbox/scripts/bin/object-storage-restore b/gitlab-toolbox/scripts/bin/object-storage-restore index dc2f63cfa296b697cbfcef526736b659b14ee779..6ceb7166fdd5fd122e5bab7a9d76cb6de2463e0b 100755 --- a/gitlab-toolbox/scripts/bin/object-storage-restore +++ b/gitlab-toolbox/scripts/bin/object-storage-restore @@ -10,4 +10,5 @@ backend_type = ENV['BACKUP_BACKEND'] || 's3' s3_tool = ENV['S3_TOOL'] || 's3cmd' aws_s3_settings = ENV['AWS_S3_SETTINGS'] || '' aws_kms_settings = ENV['AWS_KMS_SETTINGS'] || '' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings).restore +azure_config = ENV['AZURE_CONFIG_FILE'] || "/etc/gitlab/objectstorage/azure_config" +ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type, s3_tool, aws_s3_settings, aws_kms_settings, azure_config).restore diff --git a/gitlab-toolbox/scripts/lib/object_storage_azure.rb b/gitlab-toolbox/scripts/lib/object_storage_azure.rb new file mode 100644 index 0000000000000000000000000000000000000000..b244fb7234d49f1c597aef8b3aec75308bdf4b4e --- /dev/null +++ b/gitlab-toolbox/scripts/lib/object_storage_azure.rb @@ -0,0 +1,83 @@ +require 'base64' +require 'fileutils' +require 'openssl' +require 'time' +require 'uri' +require 'yaml' + +class AzureBackupUtil + class AzureConfigInvalidError < StandardError + def initialize(config_file, key) + super "Azure config file (#{config_file}) needs a valid #{key}" + end + end + + def initialize(azure_config_file = '/etc/gitlab/objectstorage/azure_config', + token_expire_s = 12 * 60 * 60, + sas_token_file = '/tmp/azure_sas_token') + @config = load_config azure_config_file + @sas_token_file = sas_token_file + @token_expire_s = token_expire_s + @current_token = {} + end + + def url + "https://#{@config['azure_storage_account_name'].to_s.strip}.#{@config['azure_storage_domain'].to_s.strip}" + end + + def sas_token + if @current_token['token'].nil? || + @current_token['expiry'].nil? || + @current_token['expiry'] - (@token_expire_s / 2) < Time.now + # refresh token + start = Time.now - 10 * 60 + expiry = Time.now + @token_expire_s + @current_token = generate_sas(@config['azure_storage_access_key'], @config['azure_storage_account_name'], + start, expiry) + end + @current_token['token'] + end + + private + + def load_config(config_file) + conf = YAML.safe_load_file config_file + mandatory_keys = %w(azure_storage_account_name azure_storage_access_key) + mandatory_keys.each do |key| + raise AzureConfigInvalidError.new(config_file, key) if conf[key].nil? + end + + conf['azure_storage_domain'] = 'blob.core.windows.net' if conf['azure_storage_domain'].nil? + conf + end + + def generate_sas(access_key, account, start, expiry) + permissions = 'rawdl' # read, access, write, delete, list + protocol = 'https' # allow https only + service = 'b' # blob storage + resources = 'co' # container + object level + version = '2018-11-09' # API Version + ip_range = '' # Allowed IPs + + # item order and empty items/lines matter + to_sign = [ + account, + permissions, + service, + resources, + start.utc.iso8601, + expiry.utc.iso8601, + ip_range, + protocol, + version, + "" + ].join("\n") + + sig = OpenSSL::HMAC.digest('sha256', Base64.decode64(access_key), to_sign) + sig = Base64.strict_encode64(sig) + sig = URI.encode_www_form_component(sig) + token = "sv=#{version}&ss=#{service}&srt=#{resources}&sp=#{permissions}"\ + "&se=#{expiry.utc.iso8601}&st=#{start.utc.iso8601}&spr=#{protocol}&sig=#{sig}" + { 'token' => token, 'start' => start, 'expiry' => expiry } + end +end diff --git a/gitlab-toolbox/scripts/lib/object_storage_backup.rb b/gitlab-toolbox/scripts/lib/object_storage_backup.rb index 4a3243975d0237e6bb350b3486d922977e46dfe5..f3e0c9f16b3581c215345d649985b5522aab693b 100644 --- a/gitlab-toolbox/scripts/lib/object_storage_backup.rb +++ b/gitlab-toolbox/scripts/lib/object_storage_backup.rb @@ -1,5 +1,6 @@ require 'open3' require 'fileutils' +require 'object_storage_azure' class String def red; "\e[31m#{self}\e[0m" end @@ -16,7 +17,8 @@ class ObjectStorageBackup REGEXP_EXCLUDE='tmp/builds/.*$' private_constant :REGEXP_EXCLUDE - def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3', s3_tool = 's3cmd', aws_s3_settings = '', aws_kms_settings = '') + def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3', + s3_tool = 's3cmd', aws_s3_settings = '', aws_kms_settings = '', azure_config_file = '') @name = name @local_tar_path = local_tar_path @remote_bucket_name = remote_bucket_name @@ -25,10 +27,12 @@ class ObjectStorageBackup @s3_tool = s3_tool @aws_s3_settings = aws_s3_settings @aws_kms_settings = aws_kms_settings + @azure_config_file = azure_config_file end def backup - if @backend == "s3" + case @backend + when 's3' if @s3_tool == "s3cmd" check_bucket_cmd = %W(s3cmd --limit=1 ls s3://#{@remote_bucket_name}) cmd = %W(s3cmd --stop-on-error --delete-removed --exclude #{GLOB_EXCLUDE} sync s3://#{@remote_bucket_name}/ /srv/gitlab/tmp/#{@name}/) @@ -36,11 +40,15 @@ class ObjectStorageBackup check_bucket_cmd = %W(aws s3api head-bucket --bucket #{@remote_bucket_name}) + @aws_s3_settings.split(" ") cmd = %W(aws s3 sync --delete --exclude #{GLOB_EXCLUDE} s3://#{@remote_bucket_name}/ /srv/gitlab/tmp/#{@name}/) + @aws_s3_settings.split(" ") + @aws_kms_settings.split(" ") end - elsif @backend == "gcs" + when 'gcs' check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name}) cmd = %W(gsutil -m rsync -d -x #{REGEXP_EXCLUDE} -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name}) + when 'azure' + check_bucket_cmd = %W(azcopy list #{azure_util.url}/#{@remote_bucket_name}?#{azure_util.sas_token}) + cmd = %W(azcopy sync #{azure_util.url}/#{@remote_bucket_name}?#{azure_util.sas_token} /srv/gitlab/tmp/#{@name} + --output-level essential --exclude-regex=#{REGEXP_EXCLUDE} --delete-destination=true) end - + # Check if the bucket exists output, status = run_cmd(check_bucket_cmd) unless status.zero? @@ -81,6 +89,8 @@ class ObjectStorageBackup puts "done".green end + private + def failure_abort(action, error_message) puts "[Error] #{error_message}".red abort "#{action} of #{@name} failed" @@ -88,14 +98,18 @@ class ObjectStorageBackup def upload_to_object_storage(source_path) dir_name = File.basename(source_path) - if @backend == "s3" + + case @backend + when 's3' if @s3_tool == "s3cmd" cmd = %W(s3cmd --stop-on-error sync #{source_path}/ s3://#{@remote_bucket_name}/#{dir_name}/) elsif @s3_tool == "awscli" cmd = %W(aws s3 sync #{source_path}/ s3://#{@remote_bucket_name}/#{dir_name}/) + @aws_s3_settings.split(" ") + @aws_kms_settings.split(" ") end - elsif @backend == "gcs" + when 'gcs' cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}/#{dir_name}) + when 'azure' + cmd = %W(azcopy sync #{source_path} #{azure_util.url}/#{@remote_bucket_name}?#{azure_util.sas_token} --output-level essential) end output, status = run_cmd(cmd) @@ -106,29 +120,35 @@ class ObjectStorageBackup def backup_existing backup_file_name = "#{@name}.#{Time.now.to_i}" - if @backend == "s3" + case @backend + when 's3' if @s3_tool == "s3cmd" cmd = %W(s3cmd sync s3://#{@remote_bucket_name} s3://#{@tmp_bucket_name}/#{backup_file_name}/) elsif @s3_tool == "awscli" cmd = %W(aws s3 sync s3://#{@remote_bucket_name} s3://#{@tmp_bucket_name}/#{backup_file_name}/) + @aws_s3_settings.split(" ") + @aws_kms_settings.split(" ") end - elsif @backend == "gcs" + when 'gcs' cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/) + when 'azure' + cmd = %W(azcopy sync + #{azure_util.url}/#{@remote_bucket_name}/?#{azure_util.sas_token} + #{azure_util.url}/#{@tmp_bucket_name}/#{backup_file_name}/?#{azure_util.sas_token} + --output-level essential) end output, status = run_cmd(cmd) - failure_abort('sync existing', output) unless status.zero? end def cleanup - if @backend == "s3" + case @backend + when 's3' if @s3_tool == "s3cmd" cmd = %W(s3cmd --stop-on-error del --force --recursive s3://#{@remote_bucket_name}) elsif @s3_tool == "awscli" cmd = %W(aws s3 rm --recursive s3://#{@remote_bucket_name}) + @aws_s3_settings.split(" ") end - elsif @backend == "gcs" + when 'gcs' # Check if the bucket has any objects list_objects_cmd = %W(gsutil ls gs://#{@remote_bucket_name}/) output, status = run_cmd(list_objects_cmd) @@ -140,7 +160,11 @@ class ObjectStorageBackup end cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*) + when 'azure' + cmd = %W(azcopy remove #{azure_util.url}/#{@remote_bucket_name}?#{azure_util.sas_token} + --output-level essential --recursive=true) end + output, status = run_cmd(cmd) failure_abort('bucket cleanup', output) unless status.zero? end @@ -158,7 +182,7 @@ class ObjectStorageBackup failure_abort('un-archive', output) unless status.zero? Dir.glob("#{extracted_tar_path}/*").each do |file| - upload_to_object_storage(file) + upload_to_object_storage(file) end end @@ -167,4 +191,7 @@ class ObjectStorageBackup return stdout.read, wait_thr.value.exitstatus end + def azure_util + @azure_util ||= AzureBackupUtil.new @azure_config_file + end end