From 3faf0a21f865f096ba13a263b1e2b93ba08b63ff Mon Sep 17 00:00:00 2001 From: Arturo Herrero Date: Tue, 21 Oct 2025 15:23:56 +0200 Subject: [PATCH 1/4] Active Context: Add AdHocIndexingWorker Changelog: other EE: true --- config/sidekiq_queues.yml | 2 + .../ai/active_context/code/repository.rb | 2 +- .../code/ad_hoc_indexing_worker.rb | 55 +++++++++ ee/app/workers/all_queues.yml | 10 ++ .../code/ad_hoc_indexing_worker_spec.rb | 115 ++++++++++++++++++ 5 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 ee/app/workers/ai/active_context/code/ad_hoc_indexing_worker.rb create mode 100644 ee/spec/workers/ai/active_context/code/ad_hoc_indexing_worker_spec.rb diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 9f2e704d5f589c..c3ca9543a2e7c2 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -29,6 +29,8 @@ - 1 - - admin_emails - 1 +- - ai_active_context_code_ad_hoc_indexing + - 1 - - ai_active_context_code_mark_repository_as_ready_event - 1 - - ai_active_context_code_process_pending_enabled_namespace_event diff --git a/ee/app/models/ai/active_context/code/repository.rb b/ee/app/models/ai/active_context/code/repository.rb index eb74a2957f4beb..8408d308fd2efc 100644 --- a/ee/app/models/ai/active_context/code/repository.rb +++ b/ee/app/models/ai/active_context/code/repository.rb @@ -20,7 +20,6 @@ class Repository < ApplicationRecord foreign_key: 'connection_id', optional: true, inverse_of: :repositories validates :project, presence: true - validates :enabled_namespace, presence: true, on: :create validates :active_context_connection, presence: true, on: :create validates :state, presence: true validates :last_commit, length: { maximum: 64 }, allow_blank: true @@ -56,6 +55,7 @@ class Repository < ApplicationRecord } scope :ready_with_active_connection, -> { ready.with_active_connection } + scope :for_project, ->(project_id) { with_active_connection.where(project_id: project_id) } def empty? project.empty_repo? diff --git a/ee/app/workers/ai/active_context/code/ad_hoc_indexing_worker.rb b/ee/app/workers/ai/active_context/code/ad_hoc_indexing_worker.rb new file mode 100644 index 00000000000000..1561e06604f462 --- /dev/null +++ b/ee/app/workers/ai/active_context/code/ad_hoc_indexing_worker.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Ai + module ActiveContext + module Code + class AdHocIndexingWorker + include ApplicationWorker + include Gitlab::Utils::StrongMemoize + prepend ::Geo::SkipSecondary + + feature_category :global_search + deduplicate :until_executed + data_consistency :sticky + urgency :low + idempotent! + defer_on_database_health_signal :gitlab_main, [:p_ai_active_context_code_repositories], 10.minutes + + def perform(project_id) + return false unless ::Ai::ActiveContext::Collections::Code.indexing? + + project = Project.find_by_id(project_id) + return false unless project + return false unless project_eligible_for_indexing?(project) + return false if Ai::ActiveContext::Code::Repository.for_project(project.id).exists? + + repository = create_repository_record(project) + RepositoryIndexWorker.perform_async(repository.id) + end + + private + + def project_eligible_for_indexing?(project) + return false unless project.project_setting.duo_features_enabled + return false if Feature.disabled?(:active_context_code_index_project, project) + + true + end + + def create_repository_record(project) + Ai::ActiveContext::Code::Repository.create( + project_id: project.id, + enabled_namespace_id: nil, + connection_id: active_connection.id, + state: :pending + ) + end + + def active_connection + Ai::ActiveContext::Connection.active + end + strong_memoize_attr :active_connection + end + end + end +end diff --git a/ee/app/workers/all_queues.yml b/ee/app/workers/all_queues.yml index cda2e6e11ecf29..d0ce9725611562 100644 --- a/ee/app/workers/all_queues.yml +++ b/ee/app/workers/all_queues.yml @@ -1433,6 +1433,16 @@ :idempotent: false :tags: [] :queue_namespace: +- :name: ai_active_context_code_ad_hoc_indexing + :worker_name: Ai::ActiveContext::Code::AdHocIndexingWorker + :feature_category: :global_search + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] + :queue_namespace: - :name: ai_active_context_code_mark_repository_as_ready_event :worker_name: Ai::ActiveContext::Code::MarkRepositoryAsReadyEventWorker :feature_category: :global_search diff --git a/ee/spec/workers/ai/active_context/code/ad_hoc_indexing_worker_spec.rb b/ee/spec/workers/ai/active_context/code/ad_hoc_indexing_worker_spec.rb new file mode 100644 index 00000000000000..f005daf511ee6d --- /dev/null +++ b/ee/spec/workers/ai/active_context/code/ad_hoc_indexing_worker_spec.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ai::ActiveContext::Code::AdHocIndexingWorker, feature_category: :global_search do + let_it_be(:project) { create(:project) } + let_it_be(:connection) do + create(:ai_active_context_connection, adapter_class: ActiveContext::Databases::Elasticsearch::Adapter) + end + + subject(:perform) { described_class.new.perform(project.id) } + + describe '#perform' do + context 'when indexing is disabled' do + before do + allow(::Ai::ActiveContext::Collections::Code).to receive(:indexing?).and_return(false) + end + + it 'returns false and does not process' do + expect(perform).to be false + expect(Ai::ActiveContext::Code::Repository).not_to receive(:create) + end + end + + context 'when indexing is enabled' do + before do + allow(::Ai::ActiveContext::Collections::Code).to receive(:indexing?).and_return(true) + end + + context 'with invalid project_id' do + subject(:perform) { described_class.new.perform(non_existing_record_id) } + + it 'returns false when project does not exist' do + expect(perform).to be false + end + end + + context 'when project exists' do + before do + project.reload.project_setting.update!(duo_features_enabled: true) + end + + context 'when project is not eligible for indexing' do + context 'when duo_features_enabled is false' do + before do + project.reload.project_setting.update!(duo_features_enabled: false) + end + + it 'returns false and does not create repository record' do + expect { perform }.not_to change { Ai::ActiveContext::Code::Repository.count } + expect(perform).to be false + end + end + + context 'when active_context_code_index_project feature flag is disabled' do + before do + stub_feature_flags(active_context_code_index_project: false) + end + + it 'returns false and does not create repository record' do + expect { perform }.not_to change { Ai::ActiveContext::Code::Repository.count } + expect(perform).to be false + end + end + end + + context 'when project is eligible for indexing' do + before do + stub_feature_flags(active_context_code_index_project: project) + end + + context 'when repository record already exists' do + let!(:existing_repository) do + create(:ai_active_context_code_repository, project: project, connection_id: connection.id) + end + + it 'returns false and does not create duplicate repository record' do + expect { perform }.not_to change { Ai::ActiveContext::Code::Repository.count } + expect(perform).to be false + end + end + + context 'when repository record does not exist' do + it 'creates repository record and enqueues RepositoryIndexWorker' do + expect(Ai::ActiveContext::Code::RepositoryIndexWorker).to receive(:perform_async) + + expect { perform }.to change { Ai::ActiveContext::Code::Repository.count }.by(1) + + repository = Ai::ActiveContext::Code::Repository.last + expect(repository.project_id).to eq(project.id) + expect(repository.enabled_namespace_id).to be_nil + expect(repository.connection_id).to eq(connection.id) + expect(repository.state).to eq('pending') + end + + it 'calls RepositoryIndexWorker with correct repository id' do + expect(Ai::ActiveContext::Code::RepositoryIndexWorker).to receive(:perform_async) do |repository_id| + repository = Ai::ActiveContext::Code::Repository.find(repository_id) + expect(repository.project_id).to eq(project.id) + end + + perform + end + end + end + end + + it_behaves_like 'worker with data consistency', described_class, data_consistency: :sticky + + it_behaves_like 'an idempotent worker' do + let(:job_args) { [project.id] } + end + end + end +end -- GitLab From f724671989dfc0b0c20d06c4d2e5ab878a020edf Mon Sep 17 00:00:00 2001 From: Arturo Herrero Date: Wed, 22 Oct 2025 13:13:58 +0200 Subject: [PATCH 2/4] Reorder eligibility checks for efficiency --- ee/app/workers/ai/active_context/code/ad_hoc_indexing_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/app/workers/ai/active_context/code/ad_hoc_indexing_worker.rb b/ee/app/workers/ai/active_context/code/ad_hoc_indexing_worker.rb index 1561e06604f462..d31b428f0f16d0 100644 --- a/ee/app/workers/ai/active_context/code/ad_hoc_indexing_worker.rb +++ b/ee/app/workers/ai/active_context/code/ad_hoc_indexing_worker.rb @@ -20,8 +20,8 @@ def perform(project_id) project = Project.find_by_id(project_id) return false unless project - return false unless project_eligible_for_indexing?(project) return false if Ai::ActiveContext::Code::Repository.for_project(project.id).exists? + return false unless project_eligible_for_indexing?(project) repository = create_repository_record(project) RepositoryIndexWorker.perform_async(repository.id) -- GitLab From d57dbcc747a7a3d6d80559e906feedf95868351a Mon Sep 17 00:00:00 2001 From: Arturo Herrero Date: Thu, 23 Oct 2025 19:29:53 +0200 Subject: [PATCH 3/4] Set enabled namespace to repositories --- .../ai/active_context/code/repository.rb | 1 + .../code/ad_hoc_indexing_worker.rb | 11 ++++++-- .../code/ad_hoc_indexing_worker_spec.rb | 25 ++++++++++++++++--- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/ee/app/models/ai/active_context/code/repository.rb b/ee/app/models/ai/active_context/code/repository.rb index 8408d308fd2efc..fc1bf936f89290 100644 --- a/ee/app/models/ai/active_context/code/repository.rb +++ b/ee/app/models/ai/active_context/code/repository.rb @@ -20,6 +20,7 @@ class Repository < ApplicationRecord foreign_key: 'connection_id', optional: true, inverse_of: :repositories validates :project, presence: true + validates :enabled_namespace, presence: true, on: :create validates :active_context_connection, presence: true, on: :create validates :state, presence: true validates :last_commit, length: { maximum: 64 }, allow_blank: true diff --git a/ee/app/workers/ai/active_context/code/ad_hoc_indexing_worker.rb b/ee/app/workers/ai/active_context/code/ad_hoc_indexing_worker.rb index d31b428f0f16d0..274ef9eb227758 100644 --- a/ee/app/workers/ai/active_context/code/ad_hoc_indexing_worker.rb +++ b/ee/app/workers/ai/active_context/code/ad_hoc_indexing_worker.rb @@ -30,8 +30,9 @@ def perform(project_id) private def project_eligible_for_indexing?(project) - return false unless project.project_setting.duo_features_enabled return false if Feature.disabled?(:active_context_code_index_project, project) + return false unless project.project_setting.duo_features_enabled + return false unless enabled_namespace_for_project(project) true end @@ -39,12 +40,18 @@ def project_eligible_for_indexing?(project) def create_repository_record(project) Ai::ActiveContext::Code::Repository.create( project_id: project.id, - enabled_namespace_id: nil, + enabled_namespace_id: enabled_namespace_for_project(project).id, connection_id: active_connection.id, state: :pending ) end + def enabled_namespace_for_project(project) + strong_memoize_with(:enabled_namespace_for_project, project) do + Ai::ActiveContext::Code::EnabledNamespace.find_enabled_namespace(active_connection, project.root_namespace) + end + end + def active_connection Ai::ActiveContext::Connection.active end diff --git a/ee/spec/workers/ai/active_context/code/ad_hoc_indexing_worker_spec.rb b/ee/spec/workers/ai/active_context/code/ad_hoc_indexing_worker_spec.rb index f005daf511ee6d..3a4228d2ad1d55 100644 --- a/ee/spec/workers/ai/active_context/code/ad_hoc_indexing_worker_spec.rb +++ b/ee/spec/workers/ai/active_context/code/ad_hoc_indexing_worker_spec.rb @@ -3,11 +3,16 @@ require 'spec_helper' RSpec.describe Ai::ActiveContext::Code::AdHocIndexingWorker, feature_category: :global_search do - let_it_be(:project) { create(:project) } + let_it_be(:namespace) { create(:group) } + let_it_be(:project) { create(:project, namespace: namespace) } let_it_be(:connection) do create(:ai_active_context_connection, adapter_class: ActiveContext::Databases::Elasticsearch::Adapter) end + let_it_be(:enabled_namespace) do + create(:ai_active_context_code_enabled_namespace, namespace: project.namespace, connection_id: connection.id) + end + subject(:perform) { described_class.new.perform(project.id) } describe '#perform' do @@ -17,8 +22,8 @@ end it 'returns false and does not process' do + expect { perform }.not_to change { Ai::ActiveContext::Code::Repository.count } expect(perform).to be false - expect(Ai::ActiveContext::Code::Repository).not_to receive(:create) end end @@ -69,9 +74,21 @@ stub_feature_flags(active_context_code_index_project: project) end + context 'when no enabled namespace exists for the project' do + before do + enabled_namespace.destroy! + end + + it 'returns false and does not create repository record' do + expect { perform }.not_to change { Ai::ActiveContext::Code::Repository.count } + expect(perform).to be false + end + end + context 'when repository record already exists' do let!(:existing_repository) do - create(:ai_active_context_code_repository, project: project, connection_id: connection.id) + create(:ai_active_context_code_repository, + project: project, connection_id: connection.id, enabled_namespace: enabled_namespace) end it 'returns false and does not create duplicate repository record' do @@ -88,7 +105,7 @@ repository = Ai::ActiveContext::Code::Repository.last expect(repository.project_id).to eq(project.id) - expect(repository.enabled_namespace_id).to be_nil + expect(repository.enabled_namespace_id).to eq(enabled_namespace.id) expect(repository.connection_id).to eq(connection.id) expect(repository.state).to eq('pending') end -- GitLab From 55af8287242e9a3e1913190ad43bd1b74b828a9d Mon Sep 17 00:00:00 2001 From: Arturo Herrero Date: Thu, 23 Oct 2025 19:30:47 +0200 Subject: [PATCH 4/4] Remove process_pending_enabled_namespace cron job scheduling --- .../active_context/code/scheduling_service.rb | 5 --- ee/lib/ee/gitlab/event_store.rb | 3 -- ee/spec/lib/ee/gitlab/event_store_spec.rb | 1 - .../code/scheduling_service_spec.rb | 33 ------------------- 4 files changed, 42 deletions(-) diff --git a/ee/app/services/ai/active_context/code/scheduling_service.rb b/ee/app/services/ai/active_context/code/scheduling_service.rb index 49708a1e7f1d1e..193f8d4b298b5c 100644 --- a/ee/app/services/ai/active_context/code/scheduling_service.rb +++ b/ee/app/services/ai/active_context/code/scheduling_service.rb @@ -7,11 +7,6 @@ class SchedulingService include Gitlab::Scheduling::TaskExecutor TASKS = { - process_pending_enabled_namespace: { - period: 30.minutes, - if: -> { ::Ai::ActiveContext::Code::EnabledNamespace.pending.with_active_connection.exists? }, - dispatch: { event: ProcessPendingEnabledNamespaceEvent } - }, index_repository: { period: 10.minutes, if: -> { ::Ai::ActiveContext::Code::Repository.pending.with_active_connection.exists? }, diff --git a/ee/lib/ee/gitlab/event_store.rb b/ee/lib/ee/gitlab/event_store.rb index 1ef0ce2e40b124..6073ad01502b2a 100644 --- a/ee/lib/ee/gitlab/event_store.rb +++ b/ee/lib/ee/gitlab/event_store.rb @@ -217,9 +217,6 @@ def subscribe_to_active_context_code_events(store) store.subscribe ::Ai::ActiveContext::Code::MarkRepositoryAsReadyEventWorker, to: ::Ai::ActiveContext::Code::MarkRepositoryAsReadyEvent - store.subscribe ::Ai::ActiveContext::Code::ProcessPendingEnabledNamespaceEventWorker, - to: ::Ai::ActiveContext::Code::ProcessPendingEnabledNamespaceEvent - store.subscribe ::Ai::ActiveContext::Code::SaasInitialIndexingEventWorker, to: ::Ai::ActiveContext::Code::SaasInitialIndexingEvent end diff --git a/ee/spec/lib/ee/gitlab/event_store_spec.rb b/ee/spec/lib/ee/gitlab/event_store_spec.rb index 79fd9924d5041d..bf9aec684f4d7b 100644 --- a/ee/spec/lib/ee/gitlab/event_store_spec.rb +++ b/ee/spec/lib/ee/gitlab/event_store_spec.rb @@ -9,7 +9,6 @@ expect(instance.subscriptions.keys).to match_array([ Ai::ActiveContext::Code::MarkRepositoryAsReadyEvent, - Ai::ActiveContext::Code::ProcessPendingEnabledNamespaceEvent, Ai::ActiveContext::Code::SaasInitialIndexingEvent, ::Ci::JobArtifactsDeletedEvent, ::Ci::JobSecurityScanCompletedEvent, diff --git a/ee/spec/services/ai/active_context/code/scheduling_service_spec.rb b/ee/spec/services/ai/active_context/code/scheduling_service_spec.rb index 4654e87668bb54..b425a034b03030 100644 --- a/ee/spec/services/ai/active_context/code/scheduling_service_spec.rb +++ b/ee/spec/services/ai/active_context/code/scheduling_service_spec.rb @@ -232,39 +232,6 @@ def schema end describe 'tasks' do - describe 'process_pending_enabled_namespace' do - before do - allow(Gitlab::EventStore).to receive(:publish) - end - - context 'when there are pending namespaces to process' do - before do - allow(Ai::ActiveContext::Code::EnabledNamespace) - .to receive_message_chain(:pending, :with_active_connection, :exists?).and_return(true) - end - - it 'publishes ProcessPendingEnabledNamespaceEvent' do - described_class.new(:process_pending_enabled_namespace).execute - - expect(Gitlab::EventStore).to have_received(:publish) - .with(an_instance_of(Ai::ActiveContext::Code::ProcessPendingEnabledNamespaceEvent)) - end - end - - context 'when there are no pending namespaces to process' do - before do - allow(Ai::ActiveContext::Code::EnabledNamespace) - .to receive_message_chain(:pending, :with_active_connection, :exists?).and_return(false) - end - - it 'does not publish the event' do - described_class.new(:process_pending_enabled_namespace).execute - - expect(Gitlab::EventStore).not_to have_received(:publish) - end - end - end - describe 'index_repository' do context 'when there are repositories to process' do before do -- GitLab