From 9b3aca1dcc133d3dd1f79fdecdfe5dacc8991c88 Mon Sep 17 00:00:00 2001 From: Shubham Kumar Date: Wed, 22 Oct 2025 16:13:42 +0200 Subject: [PATCH 1/2] Backfill namespace_details from namespaces Changelog: other --- ...l_namespace_details_description_fields.yml | 8 + ...ll_namespace_details_description_fields.rb | 42 ++++ db/schema_migrations/20251022120652 | 1 + ...ll_namespace_details_description_fields.rb | 35 ++++ ...mespace_details_description_fields_spec.rb | 180 ++++++++++++++++++ ...mespace_details_description_fields_spec.rb | 28 +++ 6 files changed, 294 insertions(+) create mode 100644 db/docs/batched_background_migrations/backfill_namespace_details_description_fields.yml create mode 100644 db/post_migrate/20251022120652_queue_backfill_namespace_details_description_fields.rb create mode 100644 db/schema_migrations/20251022120652 create mode 100644 lib/gitlab/background_migration/backfill_namespace_details_description_fields.rb create mode 100644 spec/lib/gitlab/background_migration/backfill_namespace_details_description_fields_spec.rb create mode 100644 spec/migrations/20251022120652_queue_backfill_namespace_details_description_fields_spec.rb diff --git a/db/docs/batched_background_migrations/backfill_namespace_details_description_fields.yml b/db/docs/batched_background_migrations/backfill_namespace_details_description_fields.yml new file mode 100644 index 00000000000000..2766ba959a2352 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_namespace_details_description_fields.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillNamespaceDetailsDescriptionFields +description: backfill the namespace_details table with the description, description_html and cached_markdown_version +feature_category: groups_and_projects +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1 +milestone: '18.6' +queued_migration_version: 20251022120652 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20251022120652_queue_backfill_namespace_details_description_fields.rb b/db/post_migrate/20251022120652_queue_backfill_namespace_details_description_fields.rb new file mode 100644 index 00000000000000..1fa5de7fa834a2 --- /dev/null +++ b/db/post_migrate/20251022120652_queue_backfill_namespace_details_description_fields.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +class QueueBackfillNamespaceDetailsDescriptionFields < Gitlab::Database::Migration[2.3] + milestone '18.6' + restrict_gitlab_migration gitlab_schema: :gitlab_main + + MIGRATION = "BackfillNamespaceDetailsDescriptionFields" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + # Check if the migration already exists to avoid re-queueing + if Gitlab::Database::BackgroundMigration::BatchedMigration.for_configuration( + :gitlab_main, + MIGRATION, + :namespace_details, + :namespace_id, + [], + include_compatible: true + ).exists? + Gitlab::AppLogger.warn "Batched background migration not enqueued because it already exists: " \ + "job_class_name: BackfillNamespaceDetailsDescriptionFields, + table_name: namespace_details, column_name: namespace_id, " \ + "job_arguments: []" + return + end + + queue_batched_background_migration( + MIGRATION, + :namespace_details, + :namespace_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :namespace_details, :namespace_id, []) + end +end diff --git a/db/schema_migrations/20251022120652 b/db/schema_migrations/20251022120652 new file mode 100644 index 00000000000000..2bdb306471113f --- /dev/null +++ b/db/schema_migrations/20251022120652 @@ -0,0 +1 @@ +a4cabdf794253708135b9fdc00116c3a1098afeadaeddf50c9dab76ddd89de05 \ No newline at end of file diff --git a/lib/gitlab/background_migration/backfill_namespace_details_description_fields.rb b/lib/gitlab/background_migration/backfill_namespace_details_description_fields.rb new file mode 100644 index 00000000000000..49c954579430ff --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_details_description_fields.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillNamespaceDetailsDescriptionFields < BatchedMigrationJob + operation_name :backfill_namespace_details_description_fields + feature_category :groups_and_projects + + def perform + each_sub_batch do |sub_batch| + backfill_namespace_details_fields(sub_batch) + end + end + + private + + def backfill_namespace_details_fields(relation) + connection.execute(<<~SQL) + UPDATE namespace_details + SET description = CASE WHEN namespace_details.description IS NULL THEN namespaces.description ELSE namespace_details.description END, + description_html = CASE WHEN namespace_details.description_html IS NULL THEN namespaces.description_html ELSE namespace_details.description_html END, + cached_markdown_version = CASE WHEN namespace_details.cached_markdown_version IS NULL THEN namespaces.cached_markdown_version ELSE namespace_details.cached_markdown_version END + FROM namespaces + WHERE namespace_details.namespace_id = namespaces.id + AND namespace_details.namespace_id IN (#{relation.select(:namespace_id).to_sql}) + AND ( + (namespace_details.description IS NULL AND namespaces.description IS NOT NULL) + OR (namespace_details.description_html IS NULL AND namespaces.description_html IS NOT NULL) + OR (namespace_details.cached_markdown_version IS NULL AND namespaces.cached_markdown_version IS NOT NULL) + ) + SQL + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_details_description_fields_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_details_description_fields_spec.rb new file mode 100644 index 00000000000000..f26fe7e2f9f5a2 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_namespace_details_description_fields_spec.rb @@ -0,0 +1,180 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceDetailsDescriptionFields, feature_category: :groups_and_projects do + let(:organizations) { table(:organizations) } + let(:namespaces) { table(:namespaces) } + let(:namespace_details) { table(:namespace_details) } + let!(:organization) { organizations.create!(name: 'Test Org', path: 'test-org') } + + subject(:perform_migration) do + described_class.new( + start_id: namespace_details.minimum(:namespace_id), + end_id: namespace_details.maximum(:namespace_id), + batch_table: :namespace_details, + batch_column: :namespace_id, + sub_batch_size: 10, + pause_ms: 0, + connection: ApplicationRecord.connection + ).perform + end + + describe '#perform' do + context 'when namespace has description fields and namespace_details do not' do + # these columns are added to ignore_columns :description, :description_html, :cached_markdown_version + # need to manually populate these fields + let!(:namespace1) do + namespaces.create!( + name: 'namespace-1', + path: 'namespace-1', + type: 'Group', + organization_id: organization.id + ) + end + + let!(:namespace_detail1) do + namespace_details.insert({ + namespace_id: namespace1.id, + description: nil, + description_html: nil, + cached_markdown_version: nil, + created_at: Time.current, + updated_at: Time.current + }) + + ApplicationRecord.connection.execute(<<~SQL) + UPDATE namespaces + SET description = 'Test description 1', + description_html = '

Test description 1

', + cached_markdown_version = 1 + WHERE id = #{namespace1.id} + SQL + namespace_details.find(namespace1.id) + end + + let!(:namespace2) do + namespaces.create!( + name: 'namespace-2', + path: 'namespace-2', + type: 'Group', + organization_id: organization.id + ) + end + + let!(:namespace_detail2) do + namespace_details.insert({ + namespace_id: namespace2.id, + description: nil, + description_html: nil, + cached_markdown_version: nil, + created_at: Time.current, + updated_at: Time.current + }) + ApplicationRecord.connection.execute(<<~SQL) + UPDATE namespaces + SET description = 'Test description 2', + description_html = '

Test description 2

', + cached_markdown_version = 3 + WHERE id = #{namespace2.id} + SQL + namespace_details.find(namespace2.id) + end + + it 'backfills namespace_details with description fields from namespaces' do + perform_migration + + detail1 = namespace_details.find(namespace1.id) + detail2 = namespace_details.find(namespace2.id) + + expect(detail1.description).to eq('Test description 1') + expect(detail1.description_html).to eq('

Test description 1

') + expect(detail1.cached_markdown_version).to eq(1) + + expect(detail2.description).to eq('Test description 2') + expect(detail2.description_html).to eq('

Test description 2

') + expect(detail2.cached_markdown_version).to eq(3) + end + end + + context 'when namespace_details already have description fields' do + let!(:namespace) do + namespaces.create!( + name: 'namespace-existing', + path: 'namespace-existing', + type: 'Group', + organization_id: organization.id + ) + end + + let!(:namespace_detail) do + ApplicationRecord.connection.execute(<<~SQL) + UPDATE namespaces + SET description = 'New description', + description_html = '

New description

', + cached_markdown_version = 999999 + WHERE id = #{namespace.id} + SQL + + ApplicationRecord.connection.execute(<<~SQL) + UPDATE namespace_details + SET description = 'Existing description', + description_html = '

Existing description

', + cached_markdown_version = 111111 + WHERE namespace_id = #{namespace.id} + SQL + namespace_details.find(namespace.id) + end + + it 'does not overwrite existing namespace_details fields' do + perform_migration + + detail = namespace_details.find(namespace.id) + + expect(detail.description).to eq('Existing description') + expect(detail.description_html).to eq('

Existing description

') + expect(detail.cached_markdown_version).to eq(111111) + end + end + + context 'when namespace_details has partial fields' do + let!(:namespace) do + namespaces.create!( + name: 'namespace-partial-details', + path: 'namespace-partial-details', + type: 'Group', + organization_id: organization.id + ) + end + + let!(:namespace_detail) do + ApplicationRecord.connection.execute(<<~SQL) + UPDATE namespaces + SET description = 'Updated description', + description_html = '

Updated description

', + cached_markdown_version = 777777 + WHERE id = #{namespace.id} + SQL + + ApplicationRecord.connection.execute(<<~SQL) + UPDATE namespace_details + SET description = 'Existing description', + description_html = NULL, + cached_markdown_version = NULL + WHERE namespace_id = #{namespace.id} + SQL + namespace_details.find(namespace.id) + end + + it 'only backfills the null fields' do + perform_migration + + detail = namespace_details.find(namespace.id) + + expect(detail.description).to eq('Existing description') + expect(detail.description_html).to eq('

Updated description

') + expect(detail.cached_markdown_version).to eq(777777) + end + end + end +end diff --git a/spec/migrations/20251022120652_queue_backfill_namespace_details_description_fields_spec.rb b/spec/migrations/20251022120652_queue_backfill_namespace_details_description_fields_spec.rb new file mode 100644 index 00000000000000..e8550de53f5730 --- /dev/null +++ b/spec/migrations/20251022120652_queue_backfill_namespace_details_description_fields_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillNamespaceDetailsDescriptionFields, migration: :gitlab_main, feature_category: :groups_and_projects do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :namespace_details, + column_name: :namespace_id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main, + job_arguments: [] + ) + } + end + end +end -- GitLab From 7dbd433400386b205686527ee7fcb22ddace9f29 Mon Sep 17 00:00:00 2001 From: Shubham Kumar Date: Wed, 22 Oct 2025 16:18:50 +0200 Subject: [PATCH 2/2] Apply 1 suggestion(s) to 1 file(s) --- .../backfill_namespace_details_description_fields.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/docs/batched_background_migrations/backfill_namespace_details_description_fields.yml b/db/docs/batched_background_migrations/backfill_namespace_details_description_fields.yml index 2766ba959a2352..936d52bf665fc3 100644 --- a/db/docs/batched_background_migrations/backfill_namespace_details_description_fields.yml +++ b/db/docs/batched_background_migrations/backfill_namespace_details_description_fields.yml @@ -2,7 +2,7 @@ migration_job_name: BackfillNamespaceDetailsDescriptionFields description: backfill the namespace_details table with the description, description_html and cached_markdown_version feature_category: groups_and_projects -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1 +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/209801 milestone: '18.6' queued_migration_version: 20251022120652 finalized_by: # version of the migration that finalized this BBM -- GitLab