diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index dee2d65874d0b836dca2100a17c5ece3db440095..5180acd01327925ef5b9bc1d90e1bd36eaa029a9 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -296,6 +296,16 @@ :idempotent: true :tags: [] :queue_namespace: :cronjob +- :name: cronjob:authn_data_retention_authentication_events + :worker_name: Authn::DataRetention::AuthenticationEventsWorker + :feature_category: :system_access + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] + :queue_namespace: :cronjob - :name: cronjob:authn_oauth_access_token_cleanup :worker_name: Authn::OauthAccessTokenCleanupWorker :feature_category: :system_access diff --git a/app/workers/authn/data_retention/authentication_events_worker.rb b/app/workers/authn/data_retention/authentication_events_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..a49d6a6063ea9e21afbcfe4cc1bc6a4546566af8 --- /dev/null +++ b/app/workers/authn/data_retention/authentication_events_worker.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Authn + module DataRetention + class AuthenticationEventsWorker + include ApplicationWorker + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext -- no metadata available + + RETENTION_PERIOD = 1.year + MAX_RUNTIME = 3.minutes + REQUEUE_DELAY = 3.minutes + ITERATION_DELAY = 0.1 + BATCH_SIZE = 10_000 + SUB_BATCH_SIZE = 1_000 + + feature_category :system_access + concurrency_limit -> { 1 } + idempotent! + deduplicate :until_executing, including_scheduled: true + data_consistency :sticky + defer_on_database_health_signal :gitlab_main, [:authentication_events], 10.minutes + + def perform + return unless Feature.enabled?(:cleanup_authentication_events, :instance) + + runtime_limiter = Gitlab::Metrics::RuntimeLimiter.new(MAX_RUNTIME) + + AuthenticationEvent.each_batch(of: BATCH_SIZE) do |batch| + batch.each_batch(of: SUB_BATCH_SIZE) do |sub_batch| + # rubocop: disable CodeReuse/ActiveRecord -- created_at column is not indexed, intended for use only in this worker. + sub_batch.where(created_at: ...RETENTION_PERIOD.ago).delete_all + # rubocop: enable CodeReuse/ActiveRecord + + sleep ITERATION_DELAY + end + + self.class.perform_in(REQUEUE_DELAY) && break if runtime_limiter.over_time? + end + end + end + end +end diff --git a/config/feature_flags/gitlab_com_derisk/cleanup_authentication_events.yml b/config/feature_flags/gitlab_com_derisk/cleanup_authentication_events.yml new file mode 100644 index 0000000000000000000000000000000000000000..f94ddf3eb1620b1c95aa75b5bc9231d210f23648 --- /dev/null +++ b/config/feature_flags/gitlab_com_derisk/cleanup_authentication_events.yml @@ -0,0 +1,10 @@ +--- +name: cleanup_authentication_events +description: +feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/545007 +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/196134 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/552348 +milestone: '18.2' +group: group::authentication +type: gitlab_com_derisk +default_enabled: false diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index ef9bfb86ce81912c6065470c10a5fc374a63f629..6023c9901f47be01ae5c0d749b71aef8080c8885 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -777,6 +777,9 @@ Settings.cron_jobs['authn_oauth_access_token_cleanup_worker'] ||= {} Settings.cron_jobs['authn_oauth_access_token_cleanup_worker']['cron'] ||= '0 0 1 * *' Settings.cron_jobs['authn_oauth_access_token_cleanup_worker']['job_class'] = 'Authn::OauthAccessTokenCleanupWorker' +Settings.cron_jobs['authn_data_retention_authentication_events_worker'] ||= {} +Settings.cron_jobs['authn_data_retention_authentication_events_worker']['cron'] ||= '0 2 * * *' +Settings.cron_jobs['authn_data_retention_authentication_events_worker']['job_class'] = 'Authn::DataRetention::AuthenticationEventsWorker' Gitlab.ee do Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker'] ||= {} diff --git a/spec/workers/authn/data_retention/authentication_events_worker_spec.rb b/spec/workers/authn/data_retention/authentication_events_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..b274222956f6303e0a6796a66c8cb7474761384c --- /dev/null +++ b/spec/workers/authn/data_retention/authentication_events_worker_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Authn::DataRetention::AuthenticationEventsWorker, feature_category: :system_access do + describe '#perform' do + subject(:worker) { described_class.new } + + let_it_be_with_reload(:old_event_1) { create(:authentication_event, created_at: 2.years.ago) } + let_it_be_with_reload(:old_event_2) { create(:authentication_event, created_at: 13.months.ago) } + let_it_be_with_reload(:valid_event) { create(:authentication_event, created_at: 6.months.ago) } + + it 'deletes expired authentication events' do + expect { worker.perform }.to change { AuthenticationEvent.count }.by(-2) + + expect(AuthenticationEvent.ids).to contain_exactly(valid_event.id) + end + + context 'with FF disabled' do + before do + stub_feature_flags(cleanup_authentication_events: false) + end + + it 'does not delete expired authentication events' do + expect { worker.perform }.not_to change { AuthenticationEvent.count } + end + end + + context 'with batches' do + before do + stub_const("#{described_class}::SUB_BATCH_SIZE", 1) + end + + it 'performs deletes in multiple batches' do + sql_queries = ActiveRecord::QueryRecorder.new { worker.perform }.log + + delete_statement_count = sql_queries.count { |query| query.start_with?('DELETE FROM "authentication_events"') } + + expect(delete_statement_count).to eq(3) + expect(AuthenticationEvent.ids).to contain_exactly(valid_event.id) + end + end + + context 'when runtime limit is reached' do + before do + stub_const("#{described_class}::BATCH_SIZE", 2) + stub_const("#{described_class}::SUB_BATCH_SIZE", 1) + + allow_next_instance_of(Gitlab::Metrics::RuntimeLimiter) do |runtime_limiter| + allow(runtime_limiter).to receive(:over_time?).and_return(true) + end + end + + it 'reschedules the worker' do + expect(described_class).to receive(:perform_in).with(3.minutes).twice + + worker.perform + end + end + end +end