From e4cefd1e41e084ba5593b824fa812b78c2d93e95 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Wed, 1 Oct 2025 10:27:19 -0500 Subject: [PATCH 01/12] Initial commit adding scheduling of BBM --- .../backfill_oid_on_lfs_objects_projects.yml | 8 ++++++ ...ue_backfill_oid_on_lfs_objects_projects.rb | 25 ++++++++++++++++++ db/schema_migrations/20251001143946 | 1 + ...ckfill_oid_on_lfs_objects_projects_spec.rb | 26 +++++++++++++++++++ 4 files changed, 60 insertions(+) create mode 100644 db/docs/batched_background_migrations/backfill_oid_on_lfs_objects_projects.yml create mode 100644 db/post_migrate/20251001143946_queue_backfill_oid_on_lfs_objects_projects.rb create mode 100644 db/schema_migrations/20251001143946 create mode 100644 spec/migrations/20251001143946_queue_backfill_oid_on_lfs_objects_projects_spec.rb diff --git a/db/docs/batched_background_migrations/backfill_oid_on_lfs_objects_projects.yml b/db/docs/batched_background_migrations/backfill_oid_on_lfs_objects_projects.yml new file mode 100644 index 00000000000000..67a37960e37dd3 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_oid_on_lfs_objects_projects.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillOidOnLfsObjectsProjects +description: Backfill lfs_objects_projects.oid from related lfs_object +feature_category: source_code_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/123123123 +milestone: '18.5' +queued_migration_version: 20251001143946 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20251001143946_queue_backfill_oid_on_lfs_objects_projects.rb b/db/post_migrate/20251001143946_queue_backfill_oid_on_lfs_objects_projects.rb new file mode 100644 index 00000000000000..2977278c6ca15c --- /dev/null +++ b/db/post_migrate/20251001143946_queue_backfill_oid_on_lfs_objects_projects.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class QueueBackfillOidOnLfsObjectsProjects < Gitlab::Database::Migration[2.3] + milestone '18.5' + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + MIGRATION = "BackfillOidOnLfsObjectsProjects" + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :lfs_objects_projects, + :id, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :lfs_objects_projects, :id, []) + end +end diff --git a/db/schema_migrations/20251001143946 b/db/schema_migrations/20251001143946 new file mode 100644 index 00000000000000..80b010f34d32df --- /dev/null +++ b/db/schema_migrations/20251001143946 @@ -0,0 +1 @@ +7e591665aaf165e6dbac1d73cb1ee05451688b2ec3cf340cb16d982e955e9fd3 \ No newline at end of file diff --git a/spec/migrations/20251001143946_queue_backfill_oid_on_lfs_objects_projects_spec.rb b/spec/migrations/20251001143946_queue_backfill_oid_on_lfs_objects_projects_spec.rb new file mode 100644 index 00000000000000..1ae310b4a23f57 --- /dev/null +++ b/spec/migrations/20251001143946_queue_backfill_oid_on_lfs_objects_projects_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillOidOnLfsObjectsProjects, migration: :gitlab_main, feature_category: :source_code_management do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + gitlab_schema: :gitlab_main, + table_name: :lfs_objects_projects, + column_name: :id, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end +end -- GitLab From 96bb5438b58bfb0ccdb5c7879b7dc21c6661e186 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Wed, 1 Oct 2025 11:21:31 -0500 Subject: [PATCH 02/12] Initial commit of BBM --- .../backfill_oid_on_lfs_objects_projects.rb | 18 ++++++++++++++++++ ...ackfill_oid_on_lfs_objects_projects_spec.rb | 9 +++++++++ 2 files changed, 27 insertions(+) create mode 100644 lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb create mode 100644 spec/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects_spec.rb diff --git a/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb new file mode 100644 index 00000000000000..b2070ebbc60d56 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillOidOnLfsObjectsProjects < BatchedMigrationJob + operation_name :backfill_lfs_objects_projects_oid + feature_category :source_code_management + + def perform + each_sub_batch do |sub_batch| + sub_batch + .where(oid: nil) + .update_all('oid = lfs_objects.oid FROM lfs_objects') + end + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects_spec.rb b/spec/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects_spec.rb new file mode 100644 index 00000000000000..44f45387d2a780 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillOidOnLfsObjectsProjects, feature_category: :source_code_management do + it "does a thing" do + expect(true).to be_truthy + end +end -- GitLab From 5bcaa9fa72e6b6b1541f84b5c7d7b52dc12c6dfa Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 2 Oct 2025 12:56:06 -0500 Subject: [PATCH 03/12] Add spec testing that migration backfills as expected --- ...ckfill_oid_on_lfs_objects_projects_spec.rb | 50 ++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/spec/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects_spec.rb b/spec/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects_spec.rb index 44f45387d2a780..1eede9308f8fc6 100644 --- a/spec/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects_spec.rb @@ -3,7 +3,53 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::BackfillOidOnLfsObjectsProjects, feature_category: :source_code_management do - it "does a thing" do - expect(true).to be_truthy + let(:connection) { ApplicationRecord.connection } + + let(:projects_table) { table(:projects) } + let(:lfs_objects_table) { table(:lfs_objects) } + let(:lfs_objects_projects_table) { table(:lfs_objects_projects) } + let(:lfs_object_size) { 20 } + + let(:organization) { table(:organizations).create!(name: 'organization', path: 'organization') } + + let(:namespace) { table(:namespaces).create!(name: 'ns1', path: 'ns1', organization_id: organization.id) } + + let(:project) do + projects_table.create!( + namespace_id: namespace.id, + project_namespace_id: namespace.id, + organization_id: organization.id + ) + end + + let(:lfs_object) do + lfs_objects_table.create!( + oid: 'f2b0a1e7550e9b718dafc9b525a04879a766de62e4fbdfc46593d47f7ab74636', + size: lfs_object_size + ) + end + + let!(:lfs_objects_project) do + lfs_objects_projects_table.create!(project_id: project.id, lfs_object_id: lfs_object.id, repository_type: 0) + end + + let(:job_params) do + { + batch_table: :lfs_objects_projects, + batch_column: :id, + pause_ms: 0, + sub_batch_size: QueueBackfillOidOnLfsObjectsProjects::SUB_BATCH_SIZE, + connection: connection + } + end + + let(:migration) { described_class.new(**job_params) } + + it "backfills oid from the lfs_objects" do + expect(lfs_objects_project.oid).to be_nil + + migration.perform + + expect(lfs_objects_project.reload.oid).to eq(lfs_object.oid) end end -- GitLab From 07765289e9d1ff1248ab09f1de41467f9707f804 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 2 Oct 2025 12:15:25 -0700 Subject: [PATCH 04/12] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: GitLab Duo --- .../backfill_oid_on_lfs_objects_projects.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb index b2070ebbc60d56..cad79f79efe8ff 100644 --- a/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb +++ b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb @@ -10,7 +10,7 @@ def perform each_sub_batch do |sub_batch| sub_batch .where(oid: nil) - .update_all('oid = lfs_objects.oid FROM lfs_objects') + .update_all('oid = lfs_objects.oid FROM lfs_objects WHERE lfs_objects_projects.lfs_object_id = lfs_objects.id') end end end -- GitLab From e655ca63c6e135ad064bc2ebcacb426b199edff3 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Wed, 8 Oct 2025 08:26:51 -0600 Subject: [PATCH 05/12] Update with actual MR number (now that we HAVE ONE) --- .../backfill_oid_on_lfs_objects_projects.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/docs/batched_background_migrations/backfill_oid_on_lfs_objects_projects.yml b/db/docs/batched_background_migrations/backfill_oid_on_lfs_objects_projects.yml index 67a37960e37dd3..9a3ccc7086bbe6 100644 --- a/db/docs/batched_background_migrations/backfill_oid_on_lfs_objects_projects.yml +++ b/db/docs/batched_background_migrations/backfill_oid_on_lfs_objects_projects.yml @@ -2,7 +2,7 @@ migration_job_name: BackfillOidOnLfsObjectsProjects description: Backfill lfs_objects_projects.oid from related lfs_object feature_category: source_code_management -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/123123123 +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/207128 milestone: '18.5' queued_migration_version: 20251001143946 finalized_by: # version of the migration that finalized this BBM -- GitLab From 06390901e3b8bfa56fbcd66d7fe43618845470c2 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Wed, 8 Oct 2025 08:32:08 -0600 Subject: [PATCH 06/12] Quick fix of line length --- .../backfill_oid_on_lfs_objects_projects.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb index cad79f79efe8ff..5ca41c65c081e1 100644 --- a/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb +++ b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb @@ -10,7 +10,9 @@ def perform each_sub_batch do |sub_batch| sub_batch .where(oid: nil) - .update_all('oid = lfs_objects.oid FROM lfs_objects WHERE lfs_objects_projects.lfs_object_id = lfs_objects.id') + .update_all( + 'oid = lfs_objects.oid FROM lfs_objects WHERE lfs_objects_projects.lfs_object_id = lfs_objects.id' + ) end end end -- GitLab From c59ec737307f439df63645d538744f8e558a9ddf Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Wed, 8 Oct 2025 08:44:37 -0600 Subject: [PATCH 07/12] Add joins to pull in oid --- .../backfill_oid_on_lfs_objects_projects.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb index 5ca41c65c081e1..cd67ed3c5f42c2 100644 --- a/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb +++ b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb @@ -10,9 +10,8 @@ def perform each_sub_batch do |sub_batch| sub_batch .where(oid: nil) - .update_all( - 'oid = lfs_objects.oid FROM lfs_objects WHERE lfs_objects_projects.lfs_object_id = lfs_objects.id' - ) + .joins("INNER JOIN lfs_objects ON lfs_objects_projects.lfs_object_id = lfs_objects.id") + .update_all("lfs_objects_projects.oid = lfs_objects.oid") end end end -- GitLab From fb9f8f1f26aae00618419821537a2251e18077e2 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Thu, 9 Oct 2025 15:09:28 -0700 Subject: [PATCH 08/12] Update set clause --- .../backfill_oid_on_lfs_objects_projects.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb index cd67ed3c5f42c2..fe62d5b5843bf8 100644 --- a/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb +++ b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb @@ -11,7 +11,7 @@ def perform sub_batch .where(oid: nil) .joins("INNER JOIN lfs_objects ON lfs_objects_projects.lfs_object_id = lfs_objects.id") - .update_all("lfs_objects_projects.oid = lfs_objects.oid") + .update_all("oid = lfs_objects.oid FROM lfs_objects") end end end -- GitLab From 7991633d1afe2f6ca96da47f081274f1eb251bd2 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Fri, 10 Oct 2025 14:19:23 -0700 Subject: [PATCH 09/12] Simplified SQL query --- .../backfill_oid_on_lfs_objects_projects.rb | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb index fe62d5b5843bf8..bdc813d2b91e88 100644 --- a/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb +++ b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb @@ -8,10 +8,19 @@ class BackfillOidOnLfsObjectsProjects < BatchedMigrationJob def perform each_sub_batch do |sub_batch| - sub_batch - .where(oid: nil) - .joins("INNER JOIN lfs_objects ON lfs_objects_projects.lfs_object_id = lfs_objects.id") - .update_all("oid = lfs_objects.oid FROM lfs_objects") + connection.execute( + <<~SQL + UPDATE "lfs_objects_projects" + SET + "oid" = "lfs_objects"."oid" + FROM + "lfs_objects" + WHERE + "lfs_objects_projects"."oid" IS NULL + AND "lfs_objects_projects"."lfs_object_id" = "lfs_objects"."id" + AND "lfs_objects_projects"."id" IN (#{sub_batch.select(:id).limit(sub_batch_size).to_sql}) + SQL + ) end end end -- GitLab From fac1f110c20ca7509a15b061511e63752856e181 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Mon, 20 Oct 2025 22:01:33 -0700 Subject: [PATCH 10/12] Update milestones --- .../backfill_oid_on_lfs_objects_projects.yml | 2 +- ...20251001143946_queue_backfill_oid_on_lfs_objects_projects.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db/docs/batched_background_migrations/backfill_oid_on_lfs_objects_projects.yml b/db/docs/batched_background_migrations/backfill_oid_on_lfs_objects_projects.yml index 9a3ccc7086bbe6..7023f878c3814c 100644 --- a/db/docs/batched_background_migrations/backfill_oid_on_lfs_objects_projects.yml +++ b/db/docs/batched_background_migrations/backfill_oid_on_lfs_objects_projects.yml @@ -3,6 +3,6 @@ migration_job_name: BackfillOidOnLfsObjectsProjects description: Backfill lfs_objects_projects.oid from related lfs_object feature_category: source_code_management introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/207128 -milestone: '18.5' +milestone: '18.6' queued_migration_version: 20251001143946 finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20251001143946_queue_backfill_oid_on_lfs_objects_projects.rb b/db/post_migrate/20251001143946_queue_backfill_oid_on_lfs_objects_projects.rb index 2977278c6ca15c..9ecae47ce624a1 100644 --- a/db/post_migrate/20251001143946_queue_backfill_oid_on_lfs_objects_projects.rb +++ b/db/post_migrate/20251001143946_queue_backfill_oid_on_lfs_objects_projects.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class QueueBackfillOidOnLfsObjectsProjects < Gitlab::Database::Migration[2.3] - milestone '18.5' + milestone '18.6' restrict_gitlab_migration gitlab_schema: :gitlab_main -- GitLab From 87c76b03d3911aecf3d3cae1990b1e7c6f7bc11f Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Wed, 22 Oct 2025 09:12:13 -0700 Subject: [PATCH 11/12] Remove extraneous #limit --- .../backfill_oid_on_lfs_objects_projects.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb index bdc813d2b91e88..164b96901fae44 100644 --- a/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb +++ b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb @@ -18,7 +18,7 @@ def perform WHERE "lfs_objects_projects"."oid" IS NULL AND "lfs_objects_projects"."lfs_object_id" = "lfs_objects"."id" - AND "lfs_objects_projects"."id" IN (#{sub_batch.select(:id).limit(sub_batch_size).to_sql}) + AND "lfs_objects_projects"."id" IN (#{sub_batch.select(:id).to_sql}) SQL ) end -- GitLab From ece3e83ecf1af77b0a3575ba3309ef4d935a663e Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Wed, 22 Oct 2025 09:27:13 -0700 Subject: [PATCH 12/12] Extract subquery of lfs_objects_projects ids --- .../backfill_oid_on_lfs_objects_projects.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb index 164b96901fae44..b9259e964b7e2f 100644 --- a/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb +++ b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb @@ -8,6 +8,8 @@ class BackfillOidOnLfsObjectsProjects < BatchedMigrationJob def perform each_sub_batch do |sub_batch| + lfs_obj_proj_ids = sub_batch.select(:id).map(&:id).join(', ') + connection.execute( <<~SQL UPDATE "lfs_objects_projects" @@ -18,7 +20,7 @@ def perform WHERE "lfs_objects_projects"."oid" IS NULL AND "lfs_objects_projects"."lfs_object_id" = "lfs_objects"."id" - AND "lfs_objects_projects"."id" IN (#{sub_batch.select(:id).to_sql}) + AND "lfs_objects_projects"."id" IN (#{lfs_obj_proj_ids}) SQL ) end -- GitLab