diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 3e764c38432da9e15fea9e285e151302b68ceec0..b9491019b18ac6c368b506a6214ca46d917466ac 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -951,10 +951,14 @@ - 1 - - security_analyzer_namespace_statuses_process_group_deleted_events - 1 +- - security_analyzer_namespace_statuses_recalculate + - 1 - - security_analyzers_status_process_archived_events - 1 - - security_analyzers_status_process_deleted_events - 1 +- - security_analyzers_status_process_group_archived_events + - 1 - - security_analyzers_status_process_group_transfer_events - 1 - - security_analyzers_status_process_project_transfer_events @@ -963,6 +967,8 @@ - 1 - - security_analyzers_status_setting_changed_update - 1 +- - security_analyzers_status_update_archived_analyzer_status + - 1 - - security_attributes_cleanup_batch - 1 - - security_attributes_cleanup_project_to_security_attribute diff --git a/ee/app/services/security/analyzers_status/update_archived_service.rb b/ee/app/services/security/analyzers_status/update_archived_service.rb index d306b40aaad925f7eb8b0496e906d11dbaefeb79..9c80f1501a380029534125142a141a7f5dbfa59d 100644 --- a/ee/app/services/security/analyzers_status/update_archived_service.rb +++ b/ee/app/services/security/analyzers_status/update_archived_service.rb @@ -22,7 +22,7 @@ def execute attr_reader :project def update_analyzer_statuses - project.analyzer_statuses.update!(archived: project.archived) + project.analyzer_statuses.update!(archived: project.self_or_ancestors_archived?) end end end diff --git a/ee/app/workers/all_queues.yml b/ee/app/workers/all_queues.yml index 285c6d16717c1d0c20983a95906e1b7d250ec59e..77bbdb8653e58b57ff216ea6cfe94402045c9493 100644 --- a/ee/app/workers/all_queues.yml +++ b/ee/app/workers/all_queues.yml @@ -3566,6 +3566,16 @@ :idempotent: true :tags: [] :queue_namespace: +- :name: security_analyzer_namespace_statuses_recalculate + :worker_name: Security::AnalyzerNamespaceStatuses::RecalculateWorker + :feature_category: :security_asset_inventories + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] + :queue_namespace: - :name: security_analyzers_status_process_archived_events :worker_name: Security::AnalyzersStatus::ProcessArchivedEventsWorker :feature_category: :security_asset_inventories @@ -3586,6 +3596,16 @@ :idempotent: true :tags: [] :queue_namespace: +- :name: security_analyzers_status_process_group_archived_events + :worker_name: Security::AnalyzersStatus::ProcessGroupArchivedEventsWorker + :feature_category: :security_asset_inventories + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] + :queue_namespace: - :name: security_analyzers_status_process_group_transfer_events :worker_name: Security::AnalyzersStatus::ProcessGroupTransferEventsWorker :feature_category: :security_asset_inventories @@ -3626,6 +3646,16 @@ :idempotent: true :tags: [] :queue_namespace: +- :name: security_analyzers_status_update_archived_analyzer_status + :worker_name: Security::AnalyzersStatus::UpdateArchivedAnalyzerStatusWorker + :feature_category: :security_asset_inventories + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] + :queue_namespace: - :name: security_attributes_cleanup_batch :worker_name: Security::Attributes::CleanupBatchWorker :feature_category: :security_asset_inventories diff --git a/ee/app/workers/security/analyzer_namespace_statuses/recalculate_worker.rb b/ee/app/workers/security/analyzer_namespace_statuses/recalculate_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..0f2c87747b598e1afcafa137de59daacfbf6569e --- /dev/null +++ b/ee/app/workers/security/analyzer_namespace_statuses/recalculate_worker.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Security + module AnalyzerNamespaceStatuses + class RecalculateWorker + include ApplicationWorker + + idempotent! + data_consistency :sticky + deduplicate :until_executing, including_scheduled: true + feature_category :security_asset_inventories + + def perform(group_id) + group = Group.find_by_id(group_id) + return unless group.present? + + AnalyzerNamespaceStatuses::RecalculateService.execute(group) + end + end + end +end diff --git a/ee/app/workers/security/analyzers_status/process_group_archived_events_worker.rb b/ee/app/workers/security/analyzers_status/process_group_archived_events_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..64a0940cdea945a3838a2b4c9d09bcfb22164da0 --- /dev/null +++ b/ee/app/workers/security/analyzers_status/process_group_archived_events_worker.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Security + module AnalyzersStatus + class ProcessGroupArchivedEventsWorker + include Gitlab::EventStore::Subscriber + + idempotent! + deduplicate :until_executing, including_scheduled: true + data_consistency :sticky + feature_category :security_asset_inventories + + def handle_event(event) + return unless event.data[:group_id] + + namespace_id = event.data[:group_id] + project_count = 0 + + cursor = { current_id: namespace_id, depth: [namespace_id] } + iterator = Gitlab::Database::NamespaceEachBatch.new(namespace_class: Group, cursor: cursor) + + iterator.each_batch(of: 100) do |ids, _new_cursor| + Project.in_namespace(ids).each_batch(of: 100) do |project_batch| + bulk_schedule_project_statuses_worker(project_batch) + project_count += project_batch.size + end + end + + delay = calculate_recalculation_delay(project_count) + AnalyzerNamespaceStatuses::RecalculateWorker.perform_in(delay, namespace_id) + end + + private + + def bulk_schedule_project_statuses_worker(projects) + UpdateArchivedAnalyzerStatusWorker.bulk_perform_async_with_contexts( + projects, + arguments_proc: ->(project) { project.id }, + context_proc: ->(project) { { project: project } } + ) + end + + def calculate_recalculation_delay(project_count) + # Examples: + # 100 projects: ~7 minutes + # 1,000 projects: ~22 minutes + # 10,000 projects: ~137 minutes (~2.3 hours) + + base_delay = 5.minutes + delay = base_delay + ((project_count**0.7) / 7).minutes + + [delay, 6.hours].min + end + end + end +end diff --git a/ee/app/workers/security/analyzers_status/update_archived_analyzer_status_worker.rb b/ee/app/workers/security/analyzers_status/update_archived_analyzer_status_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..cba9750888c9f181ee6b26d21dc31917bee437a9 --- /dev/null +++ b/ee/app/workers/security/analyzers_status/update_archived_analyzer_status_worker.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Security + module AnalyzersStatus + class UpdateArchivedAnalyzerStatusWorker + include ApplicationWorker + + idempotent! + data_consistency :sticky + feature_category :security_asset_inventories + sidekiq_options retry: 2 + sidekiq_retry_in { 2.minutes.seconds.to_i } + + def perform(project_id) + UpdateArchivedService.execute(Project.find_by_id(project_id)) + end + end + end +end diff --git a/ee/lib/ee/gitlab/event_store.rb b/ee/lib/ee/gitlab/event_store.rb index 8a35a8220cdf815c826117e577c300f55fd2d499..ec0ca64b58adfd42944c6c6e88d91c0a19beb7e3 100644 --- a/ee/lib/ee/gitlab/event_store.rb +++ b/ee/lib/ee/gitlab/event_store.rb @@ -146,6 +146,9 @@ def register_threat_insights_subscribers(store) to: ::Groups::GroupTransferedEvent store.subscribe ::Security::AnalyzersStatus::ProcessProjectTransferEventsWorker, to: ::Projects::ProjectTransferedEvent + store.subscribe ::Security::AnalyzersStatus::ProcessGroupArchivedEventsWorker, + to: ::Namespaces::Groups::GroupArchivedEvent + store.subscribe ::Security::AnalyzerNamespaceStatuses::ProcessGroupDeletedEventsWorker, to: ::Groups::GroupDeletedEvent diff --git a/ee/spec/services/security/analyzers_status/update_archived_service_spec.rb b/ee/spec/services/security/analyzers_status/update_archived_service_spec.rb index 12db334a5b60bc830a030b8801a1ec287037c113..f452a07f8af66b14bed2a0ba90da9ca103314a4b 100644 --- a/ee/spec/services/security/analyzers_status/update_archived_service_spec.rb +++ b/ee/spec/services/security/analyzers_status/update_archived_service_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Security::AnalyzersStatus::UpdateArchivedService, feature_category: :vulnerability_management do +RSpec.describe Security::AnalyzersStatus::UpdateArchivedService, feature_category: :security_asset_inventories do describe '.execute' do it 'instantiates a new service object and calls execute' do expect_next_instance_of(described_class, :project) do |instance| @@ -61,10 +61,47 @@ end end - context 'when the project and analyzer_status archived state match' do + context 'when the project parent group is archived' do + let_it_be(:group) { create(:group, :archived) } + let_it_be_with_reload(:project) { create(:project, group: group, archived: false) } + + it 'sets the analyzer_status record to be archived even though project is not directly archived' do + expect { update_archived } + .to change { analyzer_status.reload.archived }.from(false).to(true) + end + end + + context 'when the project ancestor group is archived' do + let_it_be(:root_group) { create(:group, :archived) } + let_it_be(:group) { create(:group, parent: root_group) } + let_it_be_with_reload(:project) { create(:project, group: group, archived: false) } + + it 'sets the analyzer_status record to be archived due to ancestor being archived' do + expect { update_archived } + .to change { analyzer_status.reload.archived }.from(false).to(true) + end + end + + context 'when the project is unarchived and parent group is also unarchived' do + let_it_be(:group) { create(:group) } + let_it_be_with_reload(:project) { create(:project, group: group, archived: false) } + + before do + analyzer_status.update!(archived: true) + end + + it 'sets the analyzer_status record to not be archived' do + expect { update_archived } + .to change { analyzer_status.reload.archived }.from(true).to(false) + end + end + + context 'when the project and analyzer_status archived state match based on ancestry' do + let_it_be(:group) { create(:group, :archived) } + let_it_be_with_reload(:project) { create(:project, group: group, archived: false) } + before do analyzer_status.update!(archived: true) - project.update!(archived: true) end it 'does not change the analyzer_status record archived state' do diff --git a/ee/spec/workers/security/analyzer_namespace_statuses/recalculate_worker_spec.rb b/ee/spec/workers/security/analyzer_namespace_statuses/recalculate_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..85ec4cd0c36edf51e13b6b52c7637fcae3035cbb --- /dev/null +++ b/ee/spec/workers/security/analyzer_namespace_statuses/recalculate_worker_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Security::AnalyzerNamespaceStatuses::RecalculateWorker, feature_category: :security_asset_inventories do + let_it_be(:group) { create(:group) } + let_it_be(:group_id) { group.id } + + subject(:run_worker) { described_class.new.perform(group_id) } + + describe '#perform' do + before do + allow(Security::AnalyzerNamespaceStatuses::RecalculateService).to receive(:execute) + end + + context 'when there is no group associated with the id' do + let(:group_id) { non_existing_record_id } + + it 'does not call the service layer logic' do + run_worker + + expect(Security::AnalyzerNamespaceStatuses::RecalculateService).not_to have_received(:execute) + end + end + + context 'when there is a group associated with the id' do + it 'calls the UpdateArchivedService' do + run_worker + + expect(Security::AnalyzerNamespaceStatuses::RecalculateService) + .to have_received(:execute).with(group) + end + end + end +end diff --git a/ee/spec/workers/security/analyzers_status/process_group_archived_events_worker_spec.rb b/ee/spec/workers/security/analyzers_status/process_group_archived_events_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..467061ddbf6b9cff2c812f6e43863a0444b1f19c --- /dev/null +++ b/ee/spec/workers/security/analyzers_status/process_group_archived_events_worker_spec.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Security::AnalyzersStatus::ProcessGroupArchivedEventsWorker, feature_category: :security_asset_inventories, type: :job do + let_it_be(:group) { create(:group) } + let_it_be(:subgroup) { create(:group, parent: group) } + let_it_be(:project_1) { create(:project, group: group) } + let_it_be(:project_2) { create(:project, group: group) } + let_it_be(:project_3) { create(:project, group: subgroup) } + + let(:worker) { described_class.new } + let(:event) do + ::Namespaces::Groups::GroupArchivedEvent.new(data: { + group_id: group.id, + root_namespace_id: group.id + }) + end + + describe '#handle_event' do + subject(:handle_event) { worker.handle_event(event) } + + it 'schedules UpdateArchivedAnalyzerStatusWorker for each project in the group and subgroups' do + project_ids = [] + context_projects = [] + + expect(Security::AnalyzersStatus::UpdateArchivedAnalyzerStatusWorker) + .to receive(:bulk_perform_async_with_contexts).at_least(:once) do |projects, arguments_proc:, context_proc:| + project_ids += projects.map(&arguments_proc) + context_projects += projects.map(&context_proc) + end + + handle_event + + expect(project_ids).to contain_exactly(project_1.id, project_2.id, project_3.id) + expect(context_projects).to contain_exactly( + { project: project_1 }, + { project: project_2 }, + { project: project_3 } + ) + end + + it 'schedules RecalculateWorker with delay based on 3 projects' do + expect(Security::AnalyzerNamespaceStatuses::RecalculateWorker) + .to receive(:perform_in) do |delay, namespace_id| + expect(delay).to be_within(5.minutes).of(6.minutes) + expect(namespace_id).to eq(group.id) + end + + handle_event + end + + context 'with 1000 projects' do + before do + allow(Security::AnalyzersStatus::UpdateArchivedAnalyzerStatusWorker) + .to receive(:bulk_perform_async_with_contexts) + + allow_next_instance_of(Gitlab::Database::NamespaceEachBatch) do |instance| + allow(instance).to receive(:each_batch).and_yield([group.id], nil) + end + + allow(Project).to receive_message_chain(:in_namespace, :each_batch) do |&block| + 10.times do # Simulate 10 batches of 100 projects each to reach 1000 total + batch = instance_double(ActiveRecord::Relation, size: 100) + block.call(batch) + end + end + end + + it 'schedules RecalculateWorker with delay based on 1000 projects' do + expect(Security::AnalyzerNamespaceStatuses::RecalculateWorker) + .to receive(:perform_in) do |delay, namespace_id| + expect(delay).to be_within(21.minutes).of(22.minutes) + expect(namespace_id).to eq(group.id) + end + + handle_event + end + end + end +end diff --git a/ee/spec/workers/security/analyzers_status/update_archived_analyzer_status_worker_spec.rb b/ee/spec/workers/security/analyzers_status/update_archived_analyzer_status_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..416d84f2cad7bc843b9f72427ebd794881fe97cf --- /dev/null +++ b/ee/spec/workers/security/analyzers_status/update_archived_analyzer_status_worker_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Security::AnalyzersStatus::UpdateArchivedAnalyzerStatusWorker, feature_category: :security_asset_inventories do + let(:project_id) { non_existing_record_id } + + subject(:run_worker) { described_class.new.perform(project_id) } + + describe '#perform' do + before do + allow(Security::AnalyzersStatus::UpdateArchivedService).to receive(:execute) + end + + it 'calls the UpdateArchivedService' do + run_worker + + expect(Security::AnalyzersStatus::UpdateArchivedService) + .to have_received(:execute).with(Project.find_by_id(project_id)) + end + end + + include_examples 'an idempotent worker' do + let_it_be(:project) { create(:project) } + + let(:job_args) { project.id } + end +end