diff --git a/db/docs/batched_background_migrations/backfill_system_note_metadata_namespace_id.yml b/db/docs/batched_background_migrations/backfill_system_note_metadata_namespace_id.yml new file mode 100644 index 0000000000000000000000000000000000000000..15b9e00efbad920fabc0d6f346fdb779b45589ba --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_system_note_metadata_namespace_id.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillSystemNoteMetadataNamespaceId +description: Backfills namespace_id for every system_note_metadata record +feature_category: team_planning +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/209714 +milestone: '18.6' +queued_migration_version: 20251021184303 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20251021184303_queue_backfill_system_note_metadata_namespace_id.rb b/db/post_migrate/20251021184303_queue_backfill_system_note_metadata_namespace_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..1b8784003e6630add08364fce3e256886a413dcc --- /dev/null +++ b/db/post_migrate/20251021184303_queue_backfill_system_note_metadata_namespace_id.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class QueueBackfillSystemNoteMetadataNamespaceId < Gitlab::Database::Migration[2.3] + milestone '18.6' + restrict_gitlab_migration gitlab_schema: :gitlab_main_org + + MIGRATION = "BackfillSystemNoteMetadataNamespaceId" + BATCH_SIZE = 30_000 + SUB_BATCH_SIZE = 250 + + def up + queue_batched_background_migration( + MIGRATION, + :system_note_metadata, + :id, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :system_note_metadata, :id, []) + end +end diff --git a/db/schema_migrations/20251021184303 b/db/schema_migrations/20251021184303 new file mode 100644 index 0000000000000000000000000000000000000000..0637ce0439794fa105fe03b3a982c60d30426965 --- /dev/null +++ b/db/schema_migrations/20251021184303 @@ -0,0 +1 @@ +94512c6e7aad948c253d50290a4f3719339b32aeb8ffc999784417f871a57e29 \ No newline at end of file diff --git a/lib/gitlab/background_migration/backfill_system_note_metadata_namespace_id.rb b/lib/gitlab/background_migration/backfill_system_note_metadata_namespace_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..2657697c1d6539a0208c6f2577b1f71c770a0c93 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_system_note_metadata_namespace_id.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillSystemNoteMetadataNamespaceId < BatchedMigrationJob + operation_name :set_namespace_id_on_system_note_metadata_records + feature_category :team_planning + + def perform + each_sub_batch do |sub_batch| + connection.execute( + <<~SQL + WITH relation AS MATERIALIZED ( + #{sub_batch.select(:id, :note_id).limit(sub_batch_size).to_sql} + ), relation_with_namespace_id AS MATERIALIZED ( + SELECT "relation".*, COALESCE("projects"."project_namespace_id", "notes"."namespace_id") AS namespace_id + FROM "relation" INNER JOIN "notes" ON "notes"."id" = "relation"."note_id" + LEFT JOIN "projects" ON "projects"."id" = "notes"."project_id" + LIMIT #{sub_batch_size} + ) + UPDATE "system_note_metadata" + SET "namespace_id" = "relation_with_namespace_id"."namespace_id" + FROM "relation_with_namespace_id" + WHERE "system_note_metadata"."id" = "relation_with_namespace_id"."id" + SQL + ) + end + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_system_note_metadata_namespace_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_system_note_metadata_namespace_id_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..fc1ea46785ac3877b560e25844450a25da8031c2 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_system_note_metadata_namespace_id_spec.rb @@ -0,0 +1,180 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillSystemNoteMetadataNamespaceId, feature_category: :team_planning do + # rubocop:disable RSpec/MultipleMemoizedHelpers -- Necessary for backfill setup + let(:namespaces) { table(:namespaces) } + let(:notes) { table(:notes) } + let(:projects) { table(:projects) } + let(:system_note_metadata) { table(:system_note_metadata) } + let(:users) { table(:users) } + let(:organization) { table(:organizations).create!(name: 'organization', path: 'organization') } + let(:project_namespace1) do + table(:namespaces).create!(name: 'project1', path: 'project1', organization_id: organization.id) + end + + let(:project_namespace2) do + table(:namespaces).create!(name: 'project2', path: 'project2', organization_id: organization.id) + end + + let(:group_namespace1) do + table(:namespaces).create!(name: 'group1', path: 'group1', organization_id: organization.id) + end + + let(:group_namespace2) do + table(:namespaces).create!(name: 'group2', path: 'group2', organization_id: organization.id) + end + + let(:project1) do + table(:projects).create!( + namespace_id: group_namespace1.id, + project_namespace_id: project_namespace1.id, + organization_id: organization.id + ) + end + + let(:project2) do + table(:projects).create!( + namespace_id: group_namespace2.id, + project_namespace_id: project_namespace2.id, + organization_id: organization.id + ) + end + + let(:user) do + users.create!( + username: 'john_doe', email: 'johndoe@gitlab.com', projects_limit: 10, organization_id: organization.id + ) + end + + let(:note1) do + notes.create!( + project_id: project1.id, + namespace_id: nil, + noteable_type: 'Issue', + noteable_id: 1, + author_id: user.id + ) + end + + let(:note_metadata1) { system_note_metadata.create!(note_id: note1.id) } + let(:note2) do + notes.create!( + project_id: project2.id, + namespace_id: group_namespace2.id, # Wrong namespace so we can test precedence with project_id column + noteable_type: 'Issue', + noteable_id: 2, + author_id: user.id + ) + end + + let(:note_metadata2) { system_note_metadata.create!(note_id: note2.id) } + let(:note3) do + notes.create!( + project_id: nil, + namespace_id: group_namespace1.id, + noteable_type: 'Issue', + noteable_id: 3, + author_id: user.id + ) + end + + let(:note_metadata3) { system_note_metadata.create!(note_id: note3.id) } + let(:note4) do + notes.create!( + project_id: nil, + namespace_id: group_namespace2.id, + noteable_type: 'Issue', + noteable_id: 4, + author_id: user.id + ) + end + + let(:note_metadata4) { system_note_metadata.create!(note_id: note4.id) } + let(:note5) do + notes.create!( + project_id: project1.id, + namespace_id: project_namespace1.id, + noteable_type: 'Issue', + noteable_id: 5, + author_id: user.id + ) + end + + let(:note_metadata5) { system_note_metadata.create!(note_id: note5.id, namespace_id: project_namespace1.id) } + + let(:note6) do + notes.create!( + project_id: project2.id, + namespace_id: group_namespace2.id, + noteable_type: 'Issue', + noteable_id: 6, + author_id: user.id + ) + end + + let(:note_metadata6) { system_note_metadata.create!(note_id: note6.id, namespace_id: group_namespace2.id) } + + describe '#perform' do + let(:migration) do + start_id, end_id = system_note_metadata.pick('MIN(id), MAX(id)') + + described_class.new( + start_id: start_id, + end_id: end_id, + batch_table: :system_note_metadata, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + job_arguments: [], + connection: ApplicationRecord.connection + ) + end + + subject(:migrate) { migration.perform } + + before do + system_note_metadata.connection.execute(<<~SQL) + -- Necessary as the constraint won't allow new invalid records to be created + ALTER TABLE system_note_metadata DROP CONSTRAINT check_9135b6f0b6; + -- Also disabling the trigger that will set the correct value in the column + ALTER TABLE system_note_metadata DISABLE TRIGGER ALL; + SQL + + note_metadata1 + note_metadata2 + note_metadata3 + note_metadata4 + note_metadata5 + + system_note_metadata.connection.execute(<<~SQL) + ALTER TABLE system_note_metadata + ADD CONSTRAINT check_9135b6f0b6 CHECK (namespace_id IS NOT NULL) NOT VALID; + + ALTER TABLE system_note_metadata ENABLE TRIGGER ALL; + SQL + end + + it 'updates records in batches' do + expect do + migrate + end.to make_queries_matching(/UPDATE\s+"system_note_metadata"/, 3) + end + + it 'sets correct namespace_id in every record' do + expect { migrate }.to change { note_metadata1.reload.namespace_id }.from(nil).to(project_namespace1.id).and( + change { note_metadata2.reload.namespace_id }.from(nil).to(project_namespace2.id) + ).and( + change { note_metadata3.reload.namespace_id }.from(nil).to(group_namespace1.id) + ).and( + change { note_metadata4.reload.namespace_id }.from(nil).to(group_namespace2.id) + ).and( + not_change { note_metadata5.reload.namespace_id }.from(project_namespace1.id) + ).and( + change { note_metadata6.reload.namespace_id }.from(group_namespace2.id).to(project_namespace2.id) + ) + end + end + # rubocop:enable RSpec/MultipleMemoizedHelpers +end diff --git a/spec/migrations/20251021184303_queue_backfill_system_note_metadata_namespace_id_spec.rb b/spec/migrations/20251021184303_queue_backfill_system_note_metadata_namespace_id_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..164ffe6b5be0c59aa2fef45a475767ba49162560 --- /dev/null +++ b/spec/migrations/20251021184303_queue_backfill_system_note_metadata_namespace_id_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillSystemNoteMetadataNamespaceId, migration: :gitlab_main_org, feature_category: :team_planning do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + gitlab_schema: :gitlab_main_org, + table_name: :system_note_metadata, + column_name: :id, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end +end