diff --git a/db/docs/batched_background_migrations/backfill_namespace_details_description_fields.yml b/db/docs/batched_background_migrations/backfill_namespace_details_description_fields.yml new file mode 100644 index 0000000000000000000000000000000000000000..936d52bf665fc3a061240ced27498bae5635215e --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_namespace_details_description_fields.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillNamespaceDetailsDescriptionFields +description: backfill the namespace_details table with the description, description_html and cached_markdown_version +feature_category: groups_and_projects +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/209801 +milestone: '18.6' +queued_migration_version: 20251022120652 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20251022120652_queue_backfill_namespace_details_description_fields.rb b/db/post_migrate/20251022120652_queue_backfill_namespace_details_description_fields.rb new file mode 100644 index 0000000000000000000000000000000000000000..1fa5de7fa834a25d0435340cf69c7c10b4e2c28a --- /dev/null +++ b/db/post_migrate/20251022120652_queue_backfill_namespace_details_description_fields.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +class QueueBackfillNamespaceDetailsDescriptionFields < Gitlab::Database::Migration[2.3] + milestone '18.6' + restrict_gitlab_migration gitlab_schema: :gitlab_main + + MIGRATION = "BackfillNamespaceDetailsDescriptionFields" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + # Check if the migration already exists to avoid re-queueing + if Gitlab::Database::BackgroundMigration::BatchedMigration.for_configuration( + :gitlab_main, + MIGRATION, + :namespace_details, + :namespace_id, + [], + include_compatible: true + ).exists? + Gitlab::AppLogger.warn "Batched background migration not enqueued because it already exists: " \ + "job_class_name: BackfillNamespaceDetailsDescriptionFields, + table_name: namespace_details, column_name: namespace_id, " \ + "job_arguments: []" + return + end + + queue_batched_background_migration( + MIGRATION, + :namespace_details, + :namespace_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :namespace_details, :namespace_id, []) + end +end diff --git a/db/schema_migrations/20251022120652 b/db/schema_migrations/20251022120652 new file mode 100644 index 0000000000000000000000000000000000000000..2bdb306471113f16e49f5faf42fbe8bb636c0dc3 --- /dev/null +++ b/db/schema_migrations/20251022120652 @@ -0,0 +1 @@ +a4cabdf794253708135b9fdc00116c3a1098afeadaeddf50c9dab76ddd89de05 \ No newline at end of file diff --git a/lib/gitlab/background_migration/backfill_namespace_details_description_fields.rb b/lib/gitlab/background_migration/backfill_namespace_details_description_fields.rb new file mode 100644 index 0000000000000000000000000000000000000000..49c954579430ff1f9d472cf4ac36f98cc7919f9a --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_details_description_fields.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillNamespaceDetailsDescriptionFields < BatchedMigrationJob + operation_name :backfill_namespace_details_description_fields + feature_category :groups_and_projects + + def perform + each_sub_batch do |sub_batch| + backfill_namespace_details_fields(sub_batch) + end + end + + private + + def backfill_namespace_details_fields(relation) + connection.execute(<<~SQL) + UPDATE namespace_details + SET description = CASE WHEN namespace_details.description IS NULL THEN namespaces.description ELSE namespace_details.description END, + description_html = CASE WHEN namespace_details.description_html IS NULL THEN namespaces.description_html ELSE namespace_details.description_html END, + cached_markdown_version = CASE WHEN namespace_details.cached_markdown_version IS NULL THEN namespaces.cached_markdown_version ELSE namespace_details.cached_markdown_version END + FROM namespaces + WHERE namespace_details.namespace_id = namespaces.id + AND namespace_details.namespace_id IN (#{relation.select(:namespace_id).to_sql}) + AND ( + (namespace_details.description IS NULL AND namespaces.description IS NOT NULL) + OR (namespace_details.description_html IS NULL AND namespaces.description_html IS NOT NULL) + OR (namespace_details.cached_markdown_version IS NULL AND namespaces.cached_markdown_version IS NOT NULL) + ) + SQL + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_details_description_fields_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_details_description_fields_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..f26fe7e2f9f5a2f84df0af2c63fbd5812a2332b3 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_namespace_details_description_fields_spec.rb @@ -0,0 +1,180 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceDetailsDescriptionFields, feature_category: :groups_and_projects do + let(:organizations) { table(:organizations) } + let(:namespaces) { table(:namespaces) } + let(:namespace_details) { table(:namespace_details) } + let!(:organization) { organizations.create!(name: 'Test Org', path: 'test-org') } + + subject(:perform_migration) do + described_class.new( + start_id: namespace_details.minimum(:namespace_id), + end_id: namespace_details.maximum(:namespace_id), + batch_table: :namespace_details, + batch_column: :namespace_id, + sub_batch_size: 10, + pause_ms: 0, + connection: ApplicationRecord.connection + ).perform + end + + describe '#perform' do + context 'when namespace has description fields and namespace_details do not' do + # these columns are added to ignore_columns :description, :description_html, :cached_markdown_version + # need to manually populate these fields + let!(:namespace1) do + namespaces.create!( + name: 'namespace-1', + path: 'namespace-1', + type: 'Group', + organization_id: organization.id + ) + end + + let!(:namespace_detail1) do + namespace_details.insert({ + namespace_id: namespace1.id, + description: nil, + description_html: nil, + cached_markdown_version: nil, + created_at: Time.current, + updated_at: Time.current + }) + + ApplicationRecord.connection.execute(<<~SQL) + UPDATE namespaces + SET description = 'Test description 1', + description_html = '

Test description 1

', + cached_markdown_version = 1 + WHERE id = #{namespace1.id} + SQL + namespace_details.find(namespace1.id) + end + + let!(:namespace2) do + namespaces.create!( + name: 'namespace-2', + path: 'namespace-2', + type: 'Group', + organization_id: organization.id + ) + end + + let!(:namespace_detail2) do + namespace_details.insert({ + namespace_id: namespace2.id, + description: nil, + description_html: nil, + cached_markdown_version: nil, + created_at: Time.current, + updated_at: Time.current + }) + ApplicationRecord.connection.execute(<<~SQL) + UPDATE namespaces + SET description = 'Test description 2', + description_html = '

Test description 2

', + cached_markdown_version = 3 + WHERE id = #{namespace2.id} + SQL + namespace_details.find(namespace2.id) + end + + it 'backfills namespace_details with description fields from namespaces' do + perform_migration + + detail1 = namespace_details.find(namespace1.id) + detail2 = namespace_details.find(namespace2.id) + + expect(detail1.description).to eq('Test description 1') + expect(detail1.description_html).to eq('

Test description 1

') + expect(detail1.cached_markdown_version).to eq(1) + + expect(detail2.description).to eq('Test description 2') + expect(detail2.description_html).to eq('

Test description 2

') + expect(detail2.cached_markdown_version).to eq(3) + end + end + + context 'when namespace_details already have description fields' do + let!(:namespace) do + namespaces.create!( + name: 'namespace-existing', + path: 'namespace-existing', + type: 'Group', + organization_id: organization.id + ) + end + + let!(:namespace_detail) do + ApplicationRecord.connection.execute(<<~SQL) + UPDATE namespaces + SET description = 'New description', + description_html = '

New description

', + cached_markdown_version = 999999 + WHERE id = #{namespace.id} + SQL + + ApplicationRecord.connection.execute(<<~SQL) + UPDATE namespace_details + SET description = 'Existing description', + description_html = '

Existing description

', + cached_markdown_version = 111111 + WHERE namespace_id = #{namespace.id} + SQL + namespace_details.find(namespace.id) + end + + it 'does not overwrite existing namespace_details fields' do + perform_migration + + detail = namespace_details.find(namespace.id) + + expect(detail.description).to eq('Existing description') + expect(detail.description_html).to eq('

Existing description

') + expect(detail.cached_markdown_version).to eq(111111) + end + end + + context 'when namespace_details has partial fields' do + let!(:namespace) do + namespaces.create!( + name: 'namespace-partial-details', + path: 'namespace-partial-details', + type: 'Group', + organization_id: organization.id + ) + end + + let!(:namespace_detail) do + ApplicationRecord.connection.execute(<<~SQL) + UPDATE namespaces + SET description = 'Updated description', + description_html = '

Updated description

', + cached_markdown_version = 777777 + WHERE id = #{namespace.id} + SQL + + ApplicationRecord.connection.execute(<<~SQL) + UPDATE namespace_details + SET description = 'Existing description', + description_html = NULL, + cached_markdown_version = NULL + WHERE namespace_id = #{namespace.id} + SQL + namespace_details.find(namespace.id) + end + + it 'only backfills the null fields' do + perform_migration + + detail = namespace_details.find(namespace.id) + + expect(detail.description).to eq('Existing description') + expect(detail.description_html).to eq('

Updated description

') + expect(detail.cached_markdown_version).to eq(777777) + end + end + end +end diff --git a/spec/migrations/20251022120652_queue_backfill_namespace_details_description_fields_spec.rb b/spec/migrations/20251022120652_queue_backfill_namespace_details_description_fields_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..e8550de53f573000d03374994d67e3275fd4cf49 --- /dev/null +++ b/spec/migrations/20251022120652_queue_backfill_namespace_details_description_fields_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillNamespaceDetailsDescriptionFields, migration: :gitlab_main, feature_category: :groups_and_projects do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :namespace_details, + column_name: :namespace_id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main, + job_arguments: [] + ) + } + end + end +end