From d5ce9eed5fb984400309e6dd1fd480befa78d8f6 Mon Sep 17 00:00:00 2001
From: Mitchell Nielsen <mnielsen@gitlab.com>
Date: Thu, 22 Feb 2024 13:40:05 -0600
Subject: [PATCH 1/7] Remove backup and restore tests

Removes the backup and restore tests.

Related to https://gitlab.com/gitlab-org/charts/gitlab/-/issues/5178

Related to https://gitlab.com/groups/gitlab-org/charts/-/epics/37#note_1694708113

As referenced in the links above, our tests have some overlap with
[upstream tests](https://gitlab.com/gitlab-org/gitlab/-/blob/d518f3437de199db731a7686c97fd1a7d1ef95fe/.gitlab/ci/database.gitlab-ci.yml#L121),
but they are not exactly the same. As a result, our pipelines break if
changes to the backup and restore process change upstream. This can
block releases.

Additionally, our process restores from a static, hand-generated backup
from a previous version of GitLab. This is not officially supported, and
therefore does not follow the process that an end user should be
following.

To summarize [this
comment](https://gitlab.com/groups/gitlab-org/charts/-/epics/37#note_1694708137),
we did indeed catch some regressions in Charts CI thanks to these tests.
However, catching them in the Charts CI is arguably too late in the
development chain and these tests should be "shifted left" closer to
their source code. This is also related to
https://gitlab.com/groups/gitlab-org/-/epics/11635, where we're looking
into consolidating backups into a single source where testing and
validation can be concentrated.
---
 spec/features/backups_spec.rb | 179 ----------------------------------
 1 file changed, 179 deletions(-)
 delete mode 100644 spec/features/backups_spec.rb

diff --git a/spec/features/backups_spec.rb b/spec/features/backups_spec.rb
deleted file mode 100644
index dc439effef..0000000000
--- a/spec/features/backups_spec.rb
+++ /dev/null
@@ -1,179 +0,0 @@
-require 'spec_helper'
-require 'open-uri'
-
-describe "Restoring a backup" do
-  before(:all) do
-    stdout, status = wait_for_dependencies
-    fail stdout unless status.success?
-
-    wait_until_app_ready
-    ensure_backups_on_object_storage
-
-    stdout, status = gitaly_purge_storage
-    fail stdout unless status.success?
-
-    stdout, status = restore_from_backup(skip: 'repositories')
-    fail stdout unless status.success?
-
-    # scale the Rails deployments to 0
-    scale_rails_down
-    # wait for rollout to complete (change in replicas)
-    wait_for_rails_rollout
-
-    # We run migrations once early to get the db into a place where we can set the runner token and restore repos
-    # Ignore errors, we will run the migrations again after the token
-    stdout, status = run_migrations
-    warn "WARNING: Migrations did not succeed:\n#{stdout}" unless status.success?
-
-    stdout, status = restore_from_backup(skip: 'db')
-    fail stdout unless status.success?
-
-    stdout, status = set_runner_token
-    fail stdout unless status.success?
-
-    stdout, status = run_migrations
-    fail stdout unless status.success?
-
-    stdout, status = enforce_root_password(ENV['GITLAB_PASSWORD']) if ENV['GITLAB_PASSWORD']
-    fail stdout unless status.success?
-
-    stdout, status = set_admin_token
-    fail stdout unless status.success?
-
-    # scale the Rails code deployments up
-    scale_rails_up
-    # wait for rollout to complete (change in replicas)
-    wait_for_rails_rollout
-
-    # Wait for the site to come up after the restore/migrations
-    wait_until_app_ready
-
-    # Have the gitlab-runner re-register after the restore
-    restart_gitlab_runner
-    wait_for_runner_rollout
-  end
-
-  describe 'Restored gitlab instance' do
-
-    it 'Project testproject1 should exist' do
-      uri = "search?scope=projects&search=testproject1"
-      response = ApiHelper.invoke_get_request(uri)
-      expect(response.collect { |item| item["name_with_namespace"] }).to have_content 'Administrator / testproject1'
-    end
-
-    it 'Issue under testproject1 should exist' do
-      uri = "search?scope=issues&search=test"
-      response = ApiHelper.invoke_get_request(uri)
-      expect(response.collect { |item| item["title"] }).to have_content 'This is a test issue with attachment'
-    end
-
-    it 'Test project repository should have Dockerfile' do
-      uri = "projects/1/repository/tree"
-      response = ApiHelper.invoke_get_request(uri)
-      expect(response.collect { |item| item["name"] })
-        .to have_content('Dockerfile')
-    end
-
-    it 'Should have at least 1 runner registered' do
-      uri = "runners/all"
-      response = ApiHelper.invoke_get_request(uri)
-      expect(response.collect { |item| item["status"] }).to have_content('online', minimum: 1)
-      expect(response.collect { |item| item["online"] }).to have_content('true', minimum: 1)
-    end
-
-    it 'Issue contains attachment in the description' do
-      uri = "issues"
-      response = ApiHelper.invoke_get_request(uri)
-      expect(response.collect { |item| item["description"] }).to have_content '![Screen_Shot_2018-05-01_at_2.53.34_PM](/uploads/90701344e9ebb53fa9ebac83d43afdcc/Screen_Shot_2018-05-01_at_2.53.34_PM.png)'
-    end
-
-    it 'Could pull image from registry' do
-      stdout, status = Open3.capture2e("docker login #{registry_url} --username root --password #{ENV['GITLAB_PASSWORD']}")
-      expect(status.success?).to be(true), "Login failed: #{stdout}"
-
-      stdout, status = Open3.capture2e("docker pull #{registry_url}/root/testproject1/master:d88102fe7cf105b72643ecb9baf41a03070c9f1b")
-      expect(status.success?).to be(true), "Pulling image failed: #{stdout}"
-    end
-
-  end
-
-  describe 'Backups' do
-    it 'Should be able to backup an identical tar' do
-      stdout, status = backup_instance
-      expect(status.success?).to be(true), "Error backing up instance: #{stdout}"
-
-      object_storage.get_object(
-        response_target: "/tmp/#{original_backup_name}",
-        bucket: 'gitlab-backups',
-        key: original_backup_name
-      )
-
-      cmd = "mkdir -p /tmp/#{original_backup_prefix} && tar -xf /tmp/#{original_backup_name} -C /tmp/#{original_backup_prefix}"
-      stdout, status = Open3.capture2e(cmd)
-      expect(status.success?).to be(true), "Error unarchiving original backup: #{stdout}"
-
-      object_storage.get_object(
-        response_target: "/tmp/#{new_backup_name}",
-        bucket: 'gitlab-backups',
-        key: new_backup_name
-      )
-
-      cmd = "mkdir -p /tmp/#{new_backup_prefix} && tar -xf /tmp/#{new_backup_name} -C /tmp/#{new_backup_prefix}"
-      stdout, status = Open3.capture2e(cmd)
-      expect(status.success?).to be(true), "Error unarchiving generated backup: #{stdout}"
-
-      Dir.glob("/tmp/#{original_backup_prefix}/*") do |file|
-        file_path = "/tmp/#{new_backup_prefix}/#{File.basename(file)}"
-        expect(File.exist?(file_path)).to be_truthy, "#{File.basename(file)} exists in original backup but not in test ( #{file_path} )"
-        # extract every tar file
-        if File.extname(file) == 'tar'
-          cmd = "tar -xf #{file} -C /tmp/#{original_backup_prefix}"
-          stdout, status = Open3.capture2e(cmd)
-          expect(status.success?).to be(true), "Error extracting tar #{file}: #{stdout}"
-
-          f_name = file.gsub(original_backup_prefix, new_backup_prefix)
-          cmd = "tar -xf #{f_name} -C /tmp/#{new_backup_prefix}"
-          stdout, status = Open3.capture2e(cmd)
-          expect(status.success?).to be(true), "Error extracting tar #{f_name}: #{stdout}"
-        end
-      end
-
-      # Remove timestamp information from directory structure
-      ## Find nested repo LATEST files to locate a repo directory
-      ## Find all dirs within that directory, and rename them to increments rather than date
-      Dir.glob(["/tmp/#{original_backup_prefix}/repositories/@hashed/*/*/*/LATEST", "/tmp/#{new_backup_prefix}/repositories/@hashed/*/*/*/LATEST"]) do |latest_file|
-        repo_dir = File.dirname(latest_file)
-        backup_id_directories = Dir.glob(File.join(repo_dir, '*')).select { |f| File.directory?(f) }
-
-        backup_id_directories.sort.each_with_index do |directory, index|
-          parent_dir = File.dirname(directory)
-
-          File.rename(directory, File.join(parent_dir, index.to_s))
-        end
-      end
-
-      Dir.glob("/tmp/#{original_backup_prefix}/**/*") do |file|
-        next if ['tar', '.gz'].include? File.extname(file)
-        next if File.directory?(file)
-        next if ['backup_information.yml', 'LATEST'].include? File.basename(file)
-        next if File.dirname(file).include?('manifest')
-
-        test_counterpart = file.gsub(original_backup_prefix, new_backup_prefix)
-
-        expect(File.exist?(test_counterpart)).to be_truthy, "Expected #{test_counterpart} to exist"
-
-        original_content = File.read(file)
-        test_content = File.read(test_counterpart)
-
-        # Strip the ref list header from bundle as its sort order may not be guaranteed
-        if File.extname(file) == '.bundle'
-          original_content = original_content.slice(original_content.index("PACK\u0000")..-1)
-          test_content = test_content.slice(test_content.index("PACK\u0000")..-1)
-        end
-
-        expect(OpenSSL::Digest::SHA256.hexdigest(original_content)).to eq(OpenSSL::Digest::SHA256.hexdigest(test_content)),
-          "Expected #{file} to equal #{test_counterpart}"
-      end
-    end
-  end
-end
-- 
GitLab


From 32d50b6d3dadb4c4e413667258959fddfe004f0f Mon Sep 17 00:00:00 2001
From: Mitchell Nielsen <mnielsen@gitlab.com>
Date: Thu, 22 Feb 2024 13:44:44 -0600
Subject: [PATCH 2/7] Docs: remove guide for updating the backup

With the backup and restore tests remove, we no longer need the
documentation on updating the backup file.
---
 doc/development/troubleshooting.md | 24 ------------------------
 1 file changed, 24 deletions(-)

diff --git a/doc/development/troubleshooting.md b/doc/development/troubleshooting.md
index 1c7eca7a10..bc541698c5 100644
--- a/doc/development/troubleshooting.md
+++ b/doc/development/troubleshooting.md
@@ -46,30 +46,6 @@ NOTE:
 This will delete all data in the databases and should not be run in
 production.
 
-## Backup used for testing needs to be updated
-
-Certain jobs in CI use a backup of GitLab during testing. Complete the steps below to update this backup when needed:
-
-1. Generate the desired backup by running a CI pipeline for the matching stable branch.
-   1. For example: run a CI pipeline for branch `5-4-stable` if current release is `5-5-stable` to create a backup of 14.4.
-   1. Note that this will require the Maintainer role.
-1. In that pipeline, cancel the QA jobs (but leave the spec tests) so that we don't get extra data in the backup.
-1. Let the spec tests finish. They will have installed the old backup, and migrated the instance to the version we want.
-1. Edit the `gitlab-runner` Deployment replicas to 0, so the Runner turns off.
-1. Log in to the UI and delete the Runner from the admin section. This should help avoid cipher errors later.
-1. [Ensure the background migrations all complete](https://docs.gitlab.com/ee/update/#checking-for-background-migrations-before-upgrading), forcing them to complete if needed.
-1. Delete the `toolbox` Pod to ensure there is no existing `tmp` data, keeping the backup small.
-1. If any manual work is needed to modify the contents of the backup, complete it before moving on to the next step.
-1. [Create a new backup](../backup-restore/backup.md) from the new `toolbox` Pod.
-1. Download the new backup from the CI instance of MinIO in the `gitlab-backups` bucket.
-1. Rename and upload the backup to the proper location in Google Cloud Storage (GCS):
-   1. Project: `cloud-native-182609`, path: `gitlab-charts-ci/test-backups/`
-   1. Name format: `$VERSION_gitlab_backup.tar` (example: `14.4.2_gitlab_backup.tar`)
-   1. Edit access and add `Entity=Public`, `Name=allUsers`, and `Access=Reader`.
-1. Finally, update `.variables.TEST_BACKUP_PREFIX` in `.gitlab-ci.yml` to the new version of the backup.
-
-Future pipelines will now use the new backup artifact during testing.
-
 ## CI clusters are low on available resources
 
 You may notice one or more CI clusters run low on available resources like CPU
-- 
GitLab


From bf7362bd0fa5462acc061dec19fe4505d3a4a08d Mon Sep 17 00:00:00 2001
From: Mitchell Nielsen <mnielsen@gitlab.com>
Date: Thu, 22 Feb 2024 15:11:02 -0600
Subject: [PATCH 3/7] Remove feature 'specs' jobs

Now that backup and restore specs are removed, and because they were the
only feature specs in place, we can remove the jobs that run
"type:feature" tests.
---
 .gitlab-ci.yml | 145 -------------------------------------------------
 1 file changed, 145 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ead053f9ec..9ca73e3834 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -757,70 +757,6 @@ review-docs-cleanup:
     - if: '$PIPELINE_TYPE =~ /FEATURE_BRANCH_PIPELINE$/'
     - if: '$PIPELINE_TYPE == "QA_KNAPSACK_UPDATE_PIPELINE"'
 
-review_specs_gke125:
-  extends: .specs
-  variables:
-    <<: *review_gke125_common_vars
-    VARIABLES_FILE: "variables/review_gke125"
-    RSPEC_TAGS: type:feature
-  environment:
-    name: gke125_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG
-    url: https://gitlab-$CI_ENVIRONENMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
-    on_stop: stop_review_gke125
-  resource_group: "gke125-review-app-${REVIEW_REF_PREFIX}${CI_COMMIT_REF_SLUG}"
-  rules:
-    - !reference [.rule:skip_if_no_cluster]
-    - !reference [.specs, rules]
-  needs: ['review_gke125']
-
-review_specs_gke126:
-  extends: .specs
-  variables:
-    <<: *review_gke126_common_vars
-    VARIABLES_FILE: "variables/review_gke126"
-    RSPEC_TAGS: type:feature
-  environment:
-    name: gke126_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG
-    url: https://gitlab-$CI_ENVIRONENMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
-    on_stop: stop_review_gke126
-  resource_group: "gke126-review-app-${REVIEW_REF_PREFIX}${CI_COMMIT_REF_SLUG}"
-  rules:
-    - !reference [.rule:skip_if_no_cluster]
-    - !reference [.specs, rules]
-  needs: ['review_gke126']
-
-review_specs_gke127:
-  extends: .specs
-  variables:
-    <<: *review_gke127_common_vars
-    VARIABLES_FILE: "variables/review_gke127"
-    RSPEC_TAGS: type:feature
-  environment:
-    name: gke127_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG
-    url: https://gitlab-$CI_ENVIRONENMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
-    on_stop: stop_review_gke127
-  resource_group: "gke127-review-app-${REVIEW_REF_PREFIX}${CI_COMMIT_REF_SLUG}"
-  rules:
-    - !reference [.rule:skip_if_no_cluster]
-    - !reference [.specs, rules]
-  needs: ['review_gke127']
-
-review_specs_eks:
-  extends: .specs
-  variables:
-    <<:  *review_eks_common_vars
-    VARIABLES_FILE: "variables/review_eks"
-    RSPEC_TAGS: type:feature
-  environment:
-    name: eks_review/$REVIEW_REF_PREFIX$CI_COMMIT_REF_SLUG
-    url: https://gitlab-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
-    on_stop: stop_review_eks
-  resource_group: "eks-review-app-${REVIEW_REF_PREFIX}${CI_COMMIT_REF_SLUG}"
-  rules:
-    - !reference [.rule:skip_if_no_cluster]
-    - !reference [.specs, rules]
-  needs: ['review_eks']
-
 specs_without_cluster:
   extends: .specs
   variables:
@@ -832,71 +768,6 @@ specs_without_cluster:
     - if: '$PIPELINE_TYPE == "RELEASE_PIPELINE"'
   needs: ['lint_package']
 
-.production_specs:
-  extends: .specs
-  rules:
-    - !reference [.rule:skip_if_no_cluster]
-    - if: '$PIPELINE_TYPE =~ /DEFAULT_BRANCH_PIPELINE$/'
-
-production_specs_without_cluster:
-  extends: .production_specs
-  variables:
-    RSPEC_TAGS: ~type:feature
-  parallel: 6
-  rules:
-    - !reference [.rule:skip_if_no_cluster]
-    - !reference [.specs, rules]
-  needs: ['lint_package']
-
-production_specs_gke125:
-  extends: .production_specs
-  variables:
-    <<: *review_gke125_common_vars
-    VARIABLES_FILE: "variables/stable_gke125"
-    RSPEC_TAGS: type:feature
-  environment:
-    name: gke125_production
-    url: https://gitlab.$KUBE_INGRESS_BASE_DOMAIN
-  resource_group: "gke125-production"
-  needs: ['stable_gke125']
-
-production_specs_gke126:
-  extends: .production_specs
-  variables:
-    <<: *review_gke126_common_vars
-    VARIABLES_FILE: "variables/stable_gke126"
-    RSPEC_TAGS: type:feature
-  environment:
-    name: gke126_production
-    url: https://gitlab.$KUBE_INGRESS_BASE_DOMAIN
-  resource_group: "gke126-production"
-  needs: ['stable_gke126']
-
-production_specs_gke127:
-  extends: .production_specs
-  variables:
-    <<: *review_gke127_common_vars
-    VARIABLES_FILE: "variables/stable_gke127"
-    RSPEC_TAGS: type:feature
-  environment:
-    name: gke127_production
-    url: https://gitlab.$KUBE_INGRESS_BASE_DOMAIN
-  resource_group: "gke127-production"
-  needs: ['stable_gke127']
-
-production_specs_eks:
-  extends: .production_specs
-  allow_failure: true
-  variables:
-    <<: *review_eks_common_vars
-    VARIABLES_FILE: "variables/stable_eks"
-    RSPEC_TAGS: type:feature
-  environment:
-    name: eks_production
-    url: https://gitlab.$KUBE_INGRESS_BASE_DOMAIN
-  resource_group: "eks-production"
-  needs: ['stable_eks']
-
 .qa:
   image: registry.gitlab.com/gitlab-org/gitlab-omnibus-builder/distribution_ci_tools:${CI_TOOLS_VERSION}
   stage: qa
@@ -1044,8 +915,6 @@ qa_gke125:
     on_stop: stop_review_gke125
   needs:
     - review_gke125
-    - job: review_specs_gke125
-      artifacts: false
 
 qa_gke126:
   extends: .qa_branch
@@ -1057,8 +926,6 @@ qa_gke126:
     on_stop: stop_review_gke126
   needs:
     - review_gke126
-    - job: review_specs_gke126
-      artifacts: false
 
 qa_gke127:
   extends: .qa_branch
@@ -1070,8 +937,6 @@ qa_gke127:
     on_stop: stop_review_gke127
   needs:
     - review_gke127
-    - job: review_specs_gke127
-      artifacts: false
 
 qa_eks:
   extends: .qa_branch
@@ -1083,8 +948,6 @@ qa_eks:
     on_stop: stop_review_eks
   needs:
     - review_eks
-    - job: review_specs_eks
-      artifacts: false
 
 qa_gke125_production:
   extends: .qa_production
@@ -1095,8 +958,6 @@ qa_gke125_production:
     name: gke125_production
   needs:
     - stable_gke125
-    - job: production_specs_gke125
-      artifacts: false
 
 qa_gke126_production:
   extends: .qa_production
@@ -1107,8 +968,6 @@ qa_gke126_production:
     name: gke126_production
   needs:
     - stable_gke126
-    - job: production_specs_gke126
-      artifacts: false
 
 qa_gke127_production:
   extends: .qa_production
@@ -1119,8 +978,6 @@ qa_gke127_production:
     name: gke127_production
   needs:
     - stable_gke127
-    - job: production_specs_gke127
-      artifacts: false
 
 qa_eks_production:
   extends: .qa_production
@@ -1131,8 +988,6 @@ qa_eks_production:
     name: eks_production
   needs:
     - stable_eks
-    - job: production_specs_eks
-      artifacts: false
 
 # QA jobs for Full E2E suites
 
-- 
GitLab


From e0a32d8f751cc09264a007ca37f61bc84fb606eb Mon Sep 17 00:00:00 2001
From: Mitchell Nielsen <mnielsen@gitlab.com>
Date: Thu, 22 Feb 2024 15:14:51 -0600
Subject: [PATCH 4/7] Rubocop: remove references to backups_spec.rb

Removes references to backups_spec.rb because the file no longer exists.
---
 .rubocop_todo.yml | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml
index e601c5b025..63721ef80f 100644
--- a/.rubocop_todo.yml
+++ b/.rubocop_todo.yml
@@ -29,7 +29,6 @@ Cop/LineBreakAroundConditionalBlock:
 # SupportedStyles: with_first_argument, with_fixed_indentation
 Layout/ArgumentAlignment:
   Exclude:
-    - 'spec/features/backups_spec.rb'
     - 'spec/spec_helper.rb'
 
 # Offense count: 6
@@ -38,14 +37,6 @@ Layout/EmptyLineAfterGuardClause:
   Exclude:
     - 'spec/configuration/certificates_spec.rb'
 
-# Offense count: 1
-# This cop supports safe autocorrection (--autocorrect).
-# Configuration parameters: EnforcedStyle.
-# SupportedStyles: empty_lines, no_empty_lines
-Layout/EmptyLinesAroundBlockBody:
-  Exclude:
-    - 'spec/features/backups_spec.rb'
-
 # Offense count: 10
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: AllowForAlignment, AllowBeforeTrailingComments, ForceEqualSignAlignment.
@@ -189,7 +180,6 @@ Performance/RegexpMatch:
 Security/Open:
   Exclude:
     - 'scripts/lib/version_fetcher.rb'
-    - 'spec/features/backups_spec.rb'
     - 'spec/gitlab_test_helper.rb'
 
 # Offense count: 8
@@ -344,7 +334,6 @@ Style/SafeNavigation:
 Style/SignalException:
   Exclude:
     - 'danger/metadata/Dangerfile'
-    - 'spec/features/backups_spec.rb'
     - 'spec/helm_template_helper.rb'
 
 # Offense count: 30
-- 
GitLab


From 4f806eba14dea4abc5802176780b0ef5cc77085a Mon Sep 17 00:00:00 2001
From: Mitchell Nielsen <mnielsen@gitlab.com>
Date: Thu, 22 Feb 2024 15:15:48 -0600
Subject: [PATCH 5/7] Remove spec/gitlab_spec_test_helper.rb

The functions defined here were used only in backups_spec.rb,
which has been deleted.
---
 spec/gitlab_test_helper.rb | 298 -------------------------------------
 1 file changed, 298 deletions(-)
 delete mode 100644 spec/gitlab_test_helper.rb

diff --git a/spec/gitlab_test_helper.rb b/spec/gitlab_test_helper.rb
deleted file mode 100644
index 7a2f54139b..0000000000
--- a/spec/gitlab_test_helper.rb
+++ /dev/null
@@ -1,298 +0,0 @@
-require 'active_support'
-require 'open-uri'
-require 'base64'
-require 'fugit'
-
-module Gitlab
-  def self.included(klass)
-    klass.extend(TestHelper)
-  end
-
-  module TestHelper
-    KUBE_TIMEOUT_DEFAULT = '2m'.freeze
-
-    def kube_timeout_parse(variable)
-      timeout = ENV[variable] || KUBE_TIMEOUT_DEFAULT
-
-      # Check the format, simplify.
-      duration = Fugit::Duration.parse(timeout)
-
-      # If invalid, return the error.
-      raise "kube_timeout_parse: #{variable}: invalid duration '#{timeout}'" if duration.nil?
-
-      duration.deflate.to_plain_s
-    end
-
-    def full_command(cmd, env = {})
-      "kubectl exec -it #{pod_name} -- env #{env_hash_to_str(env)} #{cmd}"
-    end
-
-    def gitaly_full_command(cmd)
-      "kubectl exec -it #{gitaly_pod_name} -- #{cmd}"
-    end
-
-    def env_hash_to_str(env)
-      env.map { |key, value| "#{key}=#{value}" }.join(' ')
-    end
-
-    def wait_until_app_ready(retries:30, interval: 10)
-      begin
-        URI.parse(gitlab_url).read
-      rescue
-        sleep interval
-        retries -= 1
-        retry if retries > 0
-        raise
-      end
-    end
-
-    def wait(max: 60, time: 0.1, reload: true)
-      start = Time.now
-
-      while Time.now - start < max
-        result = yield
-        return result if result
-
-        sleep(time)
-
-        page.refresh if reload
-      end
-
-      false
-    end
-
-    def enforce_root_password(password)
-      cmd = full_command("gitlab-rails runner \"user = User.find(1); user.user_type = :human ; user.password='#{password}'; user.password_confirmation='#{password}'; user.save!\"")
-
-      stdout, status = Open3.capture2e(cmd)
-      return [stdout, status]
-    end
-
-    def set_admin_token
-      cmd = full_command("gitlab-rails runner \"unless PersonalAccessToken.find_by_token('#{ENV['GITLAB_ADMIN_TOKEN']}'); user = User.find_by_username('root'); token = user.personal_access_tokens.create(scopes: ['api'], name: 'Token for running specs', expires_at: 365.days.from_now); \\
-      token.set_token('#{ENV['GITLAB_ADMIN_TOKEN']}'); token.save!  end;\"")
-      stdout, status = Open3.capture2e(cmd)
-      return [stdout, status]
-    end
-
-    def gitlab_url
-      protocol = ENV['PROTOCOL'] || 'https'
-      instance_url = ENV['GITLAB_URL'] || "gitlab.#{ENV['GITLAB_ROOT_DOMAIN']}"
-      "#{protocol}://#{instance_url}"
-    end
-
-    def registry_url
-      ENV['REGISTRY_URL'] || "registry.#{ENV['GITLAB_ROOT_DOMAIN']}"
-    end
-
-    def gitaly_purge_storage
-      cmd = gitaly_full_command("find /home/git/repositories/ -mindepth 1 -maxdepth 1 -exec rm -rf {} \\;")
-      stdout, status = Open3.capture2e(cmd)
-
-      return [stdout, status]
-    end
-
-    def get_hpa_minreplicas(app)
-      filters = "app=#{app}"
-
-      if ENV['RELEASE_NAME']
-        filters="#{filters},release=#{ENV['RELEASE_NAME']}"
-      end
-
-      stdout, status = Open3.capture2e("kubectl get hpa -l #{filters} -ojsonpath='{.items[0].spec.minReplicas}' ")
-      return [stdout, status]
-    end
-
-    def scale_deployment(app, replicas)
-      filters = "app=#{app}"
-
-      if ENV['RELEASE_NAME']
-        filters="#{filters},release=#{ENV['RELEASE_NAME']}"
-      end
-
-      puts "Scaling Deployment ('#{filters}') to #{replicas}."
-
-      stdout, status = Open3.capture2e("kubectl scale deployment -l #{filters} --replicas=#{replicas}  --timeout=#{kube_timeout_parse('KUBE_SCALE_TIMEOUT')}")
-      return [stdout, status]
-    end
-
-    def scale_rails_down
-      %w[webservice sidekiq].each do |app|
-        stdout, status = scale_deployment(app, 0)
-        raise stdout unless status.success?
-      end
-    end
-
-    def scale_rails_up
-      %w[webservice sidekiq].each do |app|
-        replicas, status = get_hpa_minreplicas(app)
-        raise replicas unless status.success?
-
-        stdout, status = scale_deployment(app, replicas)
-        raise stdout unless status.success?
-      end
-    end
-
-    def wait_for_rails_rollout
-      wait_for_rollout(type: "deployment", filters: "app in (webservice, sidekiq)")
-    end
-
-    def wait_for_runner_rollout
-      wait_for_rollout(type: "deployment", filters: "app=#{runner_app_label}")
-    end
-
-    def wait_for_rollout(type: nil, filters: nil)
-      raise ArgumentError, "Must supply both 'type' and 'filters'" if type.nil? || filters.nil?
-
-      if ENV['RELEASE_NAME']
-        filters="#{filters},release=#{ENV['RELEASE_NAME']}"
-      end
-
-      stdout, status = Open3.capture2e("kubectl rollout status #{type} -l'#{filters}' --timeout=#{kube_timeout_parse('KUBE_ROLLOUT_TIMEOUT')}")
-      raise stdout unless status.success?
-    end
-
-    def restore_from_backup(skip: [])
-      skip_flags=''
-
-      [skip].flatten.each do |skipped|
-        skip_flags += " --skip #{skipped}"
-      end
-
-      cmd = full_command("backup-utility --restore -t #{original_backup_prefix} #{skip_flags}", { GITLAB_ASSUME_YES: "1" })
-      stdout, status = Open3.capture2e(cmd)
-
-      return [stdout, status]
-    end
-
-    def backup_instance
-      cmd = full_command("backup-utility -t #{new_backup_prefix}", { GITLAB_ASSUME_YES: "1" })
-      stdout, status = Open3.capture2e(cmd)
-
-      return [stdout, status]
-    end
-
-    def run_migrations
-      cmd = full_command("gitlab-rake db:migrate")
-
-      stdout, status = Open3.capture2e(cmd)
-      return [stdout, status]
-    end
-
-    def restart_gitlab_runner
-      filters = "app=#{runner_app_label}"
-
-      if ENV['RELEASE_NAME']
-        filters="#{filters},release=#{ENV['RELEASE_NAME']}"
-      end
-
-      stdout, status = Open3.capture2e("kubectl delete pods -l #{filters} --wait=true")
-      return [stdout, status]
-    end
-
-    def set_runner_token
-      cmd = full_command(
-        "gitlab-rails runner \"" \
-        "settings = ApplicationSetting.current_without_cache; " \
-        "settings.update_columns(encrypted_customers_dot_jwt_signing_key_iv: nil, encrypted_customers_dot_jwt_signing_key: nil, encrypted_ci_jwt_signing_key_iv: nil, encrypted_ci_jwt_signing_key: nil, error_tracking_access_token_encrypted: nil); " \
-        "settings.set_runners_registration_token('#{runner_registration_token}'); " \
-        "settings.save!; " \
-        "Ci::Runner.delete_all" \
-        "\""
-      )
-
-      stdout, status = Open3.capture2e(cmd)
-      return [stdout, status]
-    end
-
-    def wait_for_dependencies
-      cmd = full_command("/scripts/wait-for-deps")
-
-      stdout, status = Open3.capture2e(cmd)
-      return [stdout, status]
-    end
-
-    def find_pod_name(filters)
-      if ENV['RELEASE_NAME']
-        filters="#{filters},release=#{ENV['RELEASE_NAME']}"
-      end
-
-      `kubectl get pod -l #{filters} --field-selector=status.phase=Running -o jsonpath="{.items[0].metadata.name}"`
-    end
-
-    def pod_name
-      filters = 'app=toolbox'
-
-      @pod ||= find_pod_name(filters)
-    end
-
-    def gitaly_pod_name
-      filters = 'app=gitaly'
-
-      @gitaly_pod ||= find_pod_name(filters)
-    end
-
-    def runner_registration_token
-      @runner_registration_token ||= Base64.decode64(
-        IO.popen(%W[kubectl get secret -o jsonpath="{.data.runner-registration-token}" -- #{ENV['RELEASE_NAME']}-gitlab-runner-secret], &:read)
-      )
-    end
-
-    def object_storage
-      return @object_storage if @object_storage
-
-      if ENV['S3_CONFIG_PATH']
-        s3_access_key = File.read("#{ENV['S3_CONFIG_PATH']}/accesskey")
-        s3_secret_key = File.read("#{ENV['S3_CONFIG_PATH']}/secretkey")
-      end
-
-      s3_access_key ||= ENV['S3_ACCESS_KEY']
-      s3_secret_key ||= ENV['S3_SECRET_KEY']
-
-      conf = {
-        region: ENV['S3_REGION'] || 'us-east-1',
-        access_key_id: s3_access_key,
-        secret_access_key: s3_secret_key,
-        endpoint: ENV['S3_ENDPOINT'],
-        force_path_style: true
-      }
-
-      @object_storage = Aws::S3::Client.new(conf)
-    end
-
-    def ensure_backups_on_object_storage
-      file = URI.open(original_backup_source_url).read
-      object_storage.put_object(
-        bucket: 'gitlab-backups',
-        key: original_backup_name,
-        body: file
-      )
-      puts "Uploaded #{original_backup_name}"
-    end
-
-    def original_backup_prefix
-      ENV['TEST_BACKUP_PREFIX']
-    end
-
-    def original_backup_name
-      "#{original_backup_prefix}_gitlab_backup.tar"
-    end
-
-    def original_backup_source_url
-      "https://storage.googleapis.com/gitlab-charts-ci/test-backups/#{original_backup_name}"
-    end
-
-    def new_backup_prefix
-      'test-backup'
-    end
-
-    def new_backup_name
-      "#{new_backup_prefix}_gitlab_backup.tar"
-    end
-
-    def runner_app_label
-      release = ENV['RELEASE_NAME'] || 'gitlab'
-      "#{release}-gitlab-runner"
-    end
-  end
-end
-- 
GitLab


From b7600e2cf8cd96a4418b8ef23f54c8d42b9e00a1 Mon Sep 17 00:00:00 2001
From: Mitchell Nielsen <mnielsen@gitlab.com>
Date: Thu, 22 Feb 2024 15:16:42 -0600
Subject: [PATCH 6/7] Combine .specs and specs_without_cluster

No other jobs extend .specs, so now we can combine the two.
---
 .gitlab-ci.yml | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9ca73e3834..044963bab5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -723,7 +723,7 @@ review-docs-cleanup:
 
 # ---------------------------------------------------------------------------
 
-.specs: &specs
+specs_without_cluster:
   image: registry.gitlab.com/gitlab-org/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-3.1.patched-golang-1.21-rust-1.73-node-18.17-postgresql-14:rubygems-3.4-git-2.36-lfs-2.9-chrome-120-yarn-1.22-graphicsmagick-1.3.36
   stage: specs
   services:
@@ -737,6 +737,7 @@ review-docs-cleanup:
     RUST_VERSION: "1.65"
     PG_VERSION: "13"
     CHROME_VERSION: "109"
+    RSPEC_TAGS: ~type:feature
   extends: .source_autodevops
   script:
     - ./scripts/ci/install_spec_dependencies
@@ -756,16 +757,9 @@ review-docs-cleanup:
     - if: '$PIPELINE_TYPE =~ /STABLE_BRANCH_PIPELINE$/'
     - if: '$PIPELINE_TYPE =~ /FEATURE_BRANCH_PIPELINE$/'
     - if: '$PIPELINE_TYPE == "QA_KNAPSACK_UPDATE_PIPELINE"'
-
-specs_without_cluster:
-  extends: .specs
-  variables:
-    RSPEC_TAGS: ~type:feature
-  parallel: 6
-  rules:
-    - !reference [.specs, rules]
     - if: '$PIPELINE_TYPE == "AUTO_DEPLOY_PIPELINE"'
     - if: '$PIPELINE_TYPE == "RELEASE_PIPELINE"'
+  parallel: 6
   needs: ['lint_package']
 
 .qa:
-- 
GitLab


From a978b8e09cc2e9a8a31dcfa43649a11455ff2cf4 Mon Sep 17 00:00:00 2001
From: Mitchell Nielsen <mnielsen@gitlab.com>
Date: Thu, 22 Feb 2024 15:18:55 -0600
Subject: [PATCH 7/7] Remove unused variables from specs_without_cluster

Removes unused variables from the specs_without_cluster job.
---
 .gitlab-ci.yml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 044963bab5..17fc00ca29 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -730,13 +730,6 @@ specs_without_cluster:
   - docker:${DOCKER_VERSION}-dind
   variables:
     DOCKER_HOST: tcp://docker:2375
-    GITLAB_PASSWORD: $ROOT_PASSWORD
-    S3_CONFIG_PATH: /etc/gitlab/minio
-    RUBYGEMS_VERSION: "3.4"
-    GO_VERSION: "1.19"
-    RUST_VERSION: "1.65"
-    PG_VERSION: "13"
-    CHROME_VERSION: "109"
     RSPEC_TAGS: ~type:feature
   extends: .source_autodevops
   script:
-- 
GitLab