diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 9f2e704d5f589c9bae719954b4b562e2a8b67bdf..c3ca9543a2e7c294662df19cdd5daa77b23de382 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -29,6 +29,8 @@ - 1 - - admin_emails - 1 +- - ai_active_context_code_ad_hoc_indexing + - 1 - - ai_active_context_code_mark_repository_as_ready_event - 1 - - ai_active_context_code_process_pending_enabled_namespace_event diff --git a/ee/app/models/ai/active_context/code/repository.rb b/ee/app/models/ai/active_context/code/repository.rb index eb74a2957f4beb8bde92c4df1dbfdc04c3d02268..fc1bf936f89290e51989a2789ee24c17c42180ba 100644 --- a/ee/app/models/ai/active_context/code/repository.rb +++ b/ee/app/models/ai/active_context/code/repository.rb @@ -56,6 +56,7 @@ class Repository < ApplicationRecord } scope :ready_with_active_connection, -> { ready.with_active_connection } + scope :for_project, ->(project_id) { with_active_connection.where(project_id: project_id) } def empty? project.empty_repo? diff --git a/ee/app/services/ai/active_context/code/scheduling_service.rb b/ee/app/services/ai/active_context/code/scheduling_service.rb index 49708a1e7f1d1e9b867d2462a14c5760de40c763..193f8d4b298b5c1f922814093c4fa36168aa2b3f 100644 --- a/ee/app/services/ai/active_context/code/scheduling_service.rb +++ b/ee/app/services/ai/active_context/code/scheduling_service.rb @@ -7,11 +7,6 @@ class SchedulingService include Gitlab::Scheduling::TaskExecutor TASKS = { - process_pending_enabled_namespace: { - period: 30.minutes, - if: -> { ::Ai::ActiveContext::Code::EnabledNamespace.pending.with_active_connection.exists? }, - dispatch: { event: ProcessPendingEnabledNamespaceEvent } - }, index_repository: { period: 10.minutes, if: -> { ::Ai::ActiveContext::Code::Repository.pending.with_active_connection.exists? }, diff --git a/ee/app/workers/ai/active_context/code/ad_hoc_indexing_worker.rb b/ee/app/workers/ai/active_context/code/ad_hoc_indexing_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..274ef9eb2277583c2bad8b912c15e7c0177ab6d0 --- /dev/null +++ b/ee/app/workers/ai/active_context/code/ad_hoc_indexing_worker.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Ai + module ActiveContext + module Code + class AdHocIndexingWorker + include ApplicationWorker + include Gitlab::Utils::StrongMemoize + prepend ::Geo::SkipSecondary + + feature_category :global_search + deduplicate :until_executed + data_consistency :sticky + urgency :low + idempotent! + defer_on_database_health_signal :gitlab_main, [:p_ai_active_context_code_repositories], 10.minutes + + def perform(project_id) + return false unless ::Ai::ActiveContext::Collections::Code.indexing? + + project = Project.find_by_id(project_id) + return false unless project + return false if Ai::ActiveContext::Code::Repository.for_project(project.id).exists? + return false unless project_eligible_for_indexing?(project) + + repository = create_repository_record(project) + RepositoryIndexWorker.perform_async(repository.id) + end + + private + + def project_eligible_for_indexing?(project) + return false if Feature.disabled?(:active_context_code_index_project, project) + return false unless project.project_setting.duo_features_enabled + return false unless enabled_namespace_for_project(project) + + true + end + + def create_repository_record(project) + Ai::ActiveContext::Code::Repository.create( + project_id: project.id, + enabled_namespace_id: enabled_namespace_for_project(project).id, + connection_id: active_connection.id, + state: :pending + ) + end + + def enabled_namespace_for_project(project) + strong_memoize_with(:enabled_namespace_for_project, project) do + Ai::ActiveContext::Code::EnabledNamespace.find_enabled_namespace(active_connection, project.root_namespace) + end + end + + def active_connection + Ai::ActiveContext::Connection.active + end + strong_memoize_attr :active_connection + end + end + end +end diff --git a/ee/app/workers/all_queues.yml b/ee/app/workers/all_queues.yml index cda2e6e11ecf299f3f2c8473d86327eec9f841d3..d0ce9725611562553e44221a9c47262ec340c569 100644 --- a/ee/app/workers/all_queues.yml +++ b/ee/app/workers/all_queues.yml @@ -1433,6 +1433,16 @@ :idempotent: false :tags: [] :queue_namespace: +- :name: ai_active_context_code_ad_hoc_indexing + :worker_name: Ai::ActiveContext::Code::AdHocIndexingWorker + :feature_category: :global_search + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] + :queue_namespace: - :name: ai_active_context_code_mark_repository_as_ready_event :worker_name: Ai::ActiveContext::Code::MarkRepositoryAsReadyEventWorker :feature_category: :global_search diff --git a/ee/lib/ee/gitlab/event_store.rb b/ee/lib/ee/gitlab/event_store.rb index 1ef0ce2e40b124fc74a6460b99c189ba76b4ed21..6073ad01502b2ad60ddca79f9cd372f6a94a7b10 100644 --- a/ee/lib/ee/gitlab/event_store.rb +++ b/ee/lib/ee/gitlab/event_store.rb @@ -217,9 +217,6 @@ def subscribe_to_active_context_code_events(store) store.subscribe ::Ai::ActiveContext::Code::MarkRepositoryAsReadyEventWorker, to: ::Ai::ActiveContext::Code::MarkRepositoryAsReadyEvent - store.subscribe ::Ai::ActiveContext::Code::ProcessPendingEnabledNamespaceEventWorker, - to: ::Ai::ActiveContext::Code::ProcessPendingEnabledNamespaceEvent - store.subscribe ::Ai::ActiveContext::Code::SaasInitialIndexingEventWorker, to: ::Ai::ActiveContext::Code::SaasInitialIndexingEvent end diff --git a/ee/spec/lib/ee/gitlab/event_store_spec.rb b/ee/spec/lib/ee/gitlab/event_store_spec.rb index 79fd9924d5041d255b73e6701ca0ddba8f422670..bf9aec684f4d7b778fd074e9d634d7dcf26abe5e 100644 --- a/ee/spec/lib/ee/gitlab/event_store_spec.rb +++ b/ee/spec/lib/ee/gitlab/event_store_spec.rb @@ -9,7 +9,6 @@ expect(instance.subscriptions.keys).to match_array([ Ai::ActiveContext::Code::MarkRepositoryAsReadyEvent, - Ai::ActiveContext::Code::ProcessPendingEnabledNamespaceEvent, Ai::ActiveContext::Code::SaasInitialIndexingEvent, ::Ci::JobArtifactsDeletedEvent, ::Ci::JobSecurityScanCompletedEvent, diff --git a/ee/spec/services/ai/active_context/code/scheduling_service_spec.rb b/ee/spec/services/ai/active_context/code/scheduling_service_spec.rb index 4654e87668bb54837ef289f0d70d6cb1b5c03ca7..b425a034b03030a35278d8fd28ef391d7c084dac 100644 --- a/ee/spec/services/ai/active_context/code/scheduling_service_spec.rb +++ b/ee/spec/services/ai/active_context/code/scheduling_service_spec.rb @@ -232,39 +232,6 @@ def schema end describe 'tasks' do - describe 'process_pending_enabled_namespace' do - before do - allow(Gitlab::EventStore).to receive(:publish) - end - - context 'when there are pending namespaces to process' do - before do - allow(Ai::ActiveContext::Code::EnabledNamespace) - .to receive_message_chain(:pending, :with_active_connection, :exists?).and_return(true) - end - - it 'publishes ProcessPendingEnabledNamespaceEvent' do - described_class.new(:process_pending_enabled_namespace).execute - - expect(Gitlab::EventStore).to have_received(:publish) - .with(an_instance_of(Ai::ActiveContext::Code::ProcessPendingEnabledNamespaceEvent)) - end - end - - context 'when there are no pending namespaces to process' do - before do - allow(Ai::ActiveContext::Code::EnabledNamespace) - .to receive_message_chain(:pending, :with_active_connection, :exists?).and_return(false) - end - - it 'does not publish the event' do - described_class.new(:process_pending_enabled_namespace).execute - - expect(Gitlab::EventStore).not_to have_received(:publish) - end - end - end - describe 'index_repository' do context 'when there are repositories to process' do before do diff --git a/ee/spec/workers/ai/active_context/code/ad_hoc_indexing_worker_spec.rb b/ee/spec/workers/ai/active_context/code/ad_hoc_indexing_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..3a4228d2ad1d55c934eed41ce09bbf6ec08f3a4a --- /dev/null +++ b/ee/spec/workers/ai/active_context/code/ad_hoc_indexing_worker_spec.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ai::ActiveContext::Code::AdHocIndexingWorker, feature_category: :global_search do + let_it_be(:namespace) { create(:group) } + let_it_be(:project) { create(:project, namespace: namespace) } + let_it_be(:connection) do + create(:ai_active_context_connection, adapter_class: ActiveContext::Databases::Elasticsearch::Adapter) + end + + let_it_be(:enabled_namespace) do + create(:ai_active_context_code_enabled_namespace, namespace: project.namespace, connection_id: connection.id) + end + + subject(:perform) { described_class.new.perform(project.id) } + + describe '#perform' do + context 'when indexing is disabled' do + before do + allow(::Ai::ActiveContext::Collections::Code).to receive(:indexing?).and_return(false) + end + + it 'returns false and does not process' do + expect { perform }.not_to change { Ai::ActiveContext::Code::Repository.count } + expect(perform).to be false + end + end + + context 'when indexing is enabled' do + before do + allow(::Ai::ActiveContext::Collections::Code).to receive(:indexing?).and_return(true) + end + + context 'with invalid project_id' do + subject(:perform) { described_class.new.perform(non_existing_record_id) } + + it 'returns false when project does not exist' do + expect(perform).to be false + end + end + + context 'when project exists' do + before do + project.reload.project_setting.update!(duo_features_enabled: true) + end + + context 'when project is not eligible for indexing' do + context 'when duo_features_enabled is false' do + before do + project.reload.project_setting.update!(duo_features_enabled: false) + end + + it 'returns false and does not create repository record' do + expect { perform }.not_to change { Ai::ActiveContext::Code::Repository.count } + expect(perform).to be false + end + end + + context 'when active_context_code_index_project feature flag is disabled' do + before do + stub_feature_flags(active_context_code_index_project: false) + end + + it 'returns false and does not create repository record' do + expect { perform }.not_to change { Ai::ActiveContext::Code::Repository.count } + expect(perform).to be false + end + end + end + + context 'when project is eligible for indexing' do + before do + stub_feature_flags(active_context_code_index_project: project) + end + + context 'when no enabled namespace exists for the project' do + before do + enabled_namespace.destroy! + end + + it 'returns false and does not create repository record' do + expect { perform }.not_to change { Ai::ActiveContext::Code::Repository.count } + expect(perform).to be false + end + end + + context 'when repository record already exists' do + let!(:existing_repository) do + create(:ai_active_context_code_repository, + project: project, connection_id: connection.id, enabled_namespace: enabled_namespace) + end + + it 'returns false and does not create duplicate repository record' do + expect { perform }.not_to change { Ai::ActiveContext::Code::Repository.count } + expect(perform).to be false + end + end + + context 'when repository record does not exist' do + it 'creates repository record and enqueues RepositoryIndexWorker' do + expect(Ai::ActiveContext::Code::RepositoryIndexWorker).to receive(:perform_async) + + expect { perform }.to change { Ai::ActiveContext::Code::Repository.count }.by(1) + + repository = Ai::ActiveContext::Code::Repository.last + expect(repository.project_id).to eq(project.id) + expect(repository.enabled_namespace_id).to eq(enabled_namespace.id) + expect(repository.connection_id).to eq(connection.id) + expect(repository.state).to eq('pending') + end + + it 'calls RepositoryIndexWorker with correct repository id' do + expect(Ai::ActiveContext::Code::RepositoryIndexWorker).to receive(:perform_async) do |repository_id| + repository = Ai::ActiveContext::Code::Repository.find(repository_id) + expect(repository.project_id).to eq(project.id) + end + + perform + end + end + end + end + + it_behaves_like 'worker with data consistency', described_class, data_consistency: :sticky + + it_behaves_like 'an idempotent worker' do + let(:job_args) { [project.id] } + end + end + end +end