diff --git a/db/docs/batched_background_migrations/backfill_oid_on_lfs_objects_projects.yml b/db/docs/batched_background_migrations/backfill_oid_on_lfs_objects_projects.yml new file mode 100644 index 0000000000000000000000000000000000000000..7023f878c3814c03acd521bea14887e99de1e7bf --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_oid_on_lfs_objects_projects.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillOidOnLfsObjectsProjects +description: Backfill lfs_objects_projects.oid from related lfs_object +feature_category: source_code_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/207128 +milestone: '18.6' +queued_migration_version: 20251001143946 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20251001143946_queue_backfill_oid_on_lfs_objects_projects.rb b/db/post_migrate/20251001143946_queue_backfill_oid_on_lfs_objects_projects.rb new file mode 100644 index 0000000000000000000000000000000000000000..9ecae47ce624a1273de9a0635d2d95f0bc1f2ff7 --- /dev/null +++ b/db/post_migrate/20251001143946_queue_backfill_oid_on_lfs_objects_projects.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class QueueBackfillOidOnLfsObjectsProjects < Gitlab::Database::Migration[2.3] + milestone '18.6' + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + MIGRATION = "BackfillOidOnLfsObjectsProjects" + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :lfs_objects_projects, + :id, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :lfs_objects_projects, :id, []) + end +end diff --git a/db/schema_migrations/20251001143946 b/db/schema_migrations/20251001143946 new file mode 100644 index 0000000000000000000000000000000000000000..80b010f34d32dfdae16fc20fcf99488a762eced6 --- /dev/null +++ b/db/schema_migrations/20251001143946 @@ -0,0 +1 @@ +7e591665aaf165e6dbac1d73cb1ee05451688b2ec3cf340cb16d982e955e9fd3 \ No newline at end of file diff --git a/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb new file mode 100644 index 0000000000000000000000000000000000000000..b9259e964b7e2ff7f4c0463110acba0a9ec4cee1 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillOidOnLfsObjectsProjects < BatchedMigrationJob + operation_name :backfill_lfs_objects_projects_oid + feature_category :source_code_management + + def perform + each_sub_batch do |sub_batch| + lfs_obj_proj_ids = sub_batch.select(:id).map(&:id).join(', ') + + connection.execute( + <<~SQL + UPDATE "lfs_objects_projects" + SET + "oid" = "lfs_objects"."oid" + FROM + "lfs_objects" + WHERE + "lfs_objects_projects"."oid" IS NULL + AND "lfs_objects_projects"."lfs_object_id" = "lfs_objects"."id" + AND "lfs_objects_projects"."id" IN (#{lfs_obj_proj_ids}) + SQL + ) + end + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects_spec.rb b/spec/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..1eede9308f8fc634c27eb79f7eb0ba9e4e771bdb --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_oid_on_lfs_objects_projects_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillOidOnLfsObjectsProjects, feature_category: :source_code_management do + let(:connection) { ApplicationRecord.connection } + + let(:projects_table) { table(:projects) } + let(:lfs_objects_table) { table(:lfs_objects) } + let(:lfs_objects_projects_table) { table(:lfs_objects_projects) } + let(:lfs_object_size) { 20 } + + let(:organization) { table(:organizations).create!(name: 'organization', path: 'organization') } + + let(:namespace) { table(:namespaces).create!(name: 'ns1', path: 'ns1', organization_id: organization.id) } + + let(:project) do + projects_table.create!( + namespace_id: namespace.id, + project_namespace_id: namespace.id, + organization_id: organization.id + ) + end + + let(:lfs_object) do + lfs_objects_table.create!( + oid: 'f2b0a1e7550e9b718dafc9b525a04879a766de62e4fbdfc46593d47f7ab74636', + size: lfs_object_size + ) + end + + let!(:lfs_objects_project) do + lfs_objects_projects_table.create!(project_id: project.id, lfs_object_id: lfs_object.id, repository_type: 0) + end + + let(:job_params) do + { + batch_table: :lfs_objects_projects, + batch_column: :id, + pause_ms: 0, + sub_batch_size: QueueBackfillOidOnLfsObjectsProjects::SUB_BATCH_SIZE, + connection: connection + } + end + + let(:migration) { described_class.new(**job_params) } + + it "backfills oid from the lfs_objects" do + expect(lfs_objects_project.oid).to be_nil + + migration.perform + + expect(lfs_objects_project.reload.oid).to eq(lfs_object.oid) + end +end diff --git a/spec/migrations/20251001143946_queue_backfill_oid_on_lfs_objects_projects_spec.rb b/spec/migrations/20251001143946_queue_backfill_oid_on_lfs_objects_projects_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..1ae310b4a23f570b45662f7a423387a51f9879fb --- /dev/null +++ b/spec/migrations/20251001143946_queue_backfill_oid_on_lfs_objects_projects_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillOidOnLfsObjectsProjects, migration: :gitlab_main, feature_category: :source_code_management do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + gitlab_schema: :gitlab_main, + table_name: :lfs_objects_projects, + column_name: :id, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end +end