From 8efcaac0025bcfce2f5a5c59adba193e269ede49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Tue, 28 Jan 2020 18:13:21 +0100 Subject: [PATCH 1/3] Ensure mail delivery is re-enabled outside of lib/gitlab/seeder.rb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- db/fixtures/development/04_labels.rb | 4 +- db/fixtures/development/06_teams.rb | 8 +-- db/fixtures/development/07_milestones.rb | 4 +- db/fixtures/development/10_merge_requests.rb | 5 +- db/fixtures/development/11_keys.rb | 4 +- db/fixtures/development/12_snippets.rb | 5 +- db/fixtures/development/14_pipelines.rb | 4 +- .../development/16_protected_branches.rb | 4 +- db/fixtures/development/17_cycle_analytics.rb | 4 +- db/fixtures/development/19_environments.rb | 4 +- db/fixtures/development/23_spam_logs.rb | 8 +-- db/fixtures/development/24_forks.rb | 6 +- .../development/26_container_images.rb | 4 +- ee/db/fixtures/development/20_burndown.rb | 4 +- .../development/20_vulnerabilities.rb | 4 +- .../development/25_downstream_pipelines.rb | 4 +- ee/db/fixtures/development/26_packages.rb | 4 +- lib/gitlab/seeder.rb | 63 +++++-------------- lib/tasks/gitlab/seed.rake | 42 +++++++------ 19 files changed, 76 insertions(+), 109 deletions(-) diff --git a/db/fixtures/development/04_labels.rb b/db/fixtures/development/04_labels.rb index 21d552c89f5cb2..51f539f2e35e01 100644 --- a/db/fixtures/development/04_labels.rb +++ b/db/fixtures/development/04_labels.rb @@ -36,14 +36,14 @@ def seed! end end -Gitlab::Seeder.quiet do +Gitlab::Seeder.quiet do |seeder| puts "\nGenerating group labels" Group.all.find_each do |group| Gitlab::Seeder::GroupLabels.new(group).seed! end puts "\nGenerating project labels" - Project.not_mass_generated.find_each do |project| + seeder.not_mass_generated_projects.find_each do |project| Gitlab::Seeder::ProjectLabels.new(project).seed! end end diff --git a/db/fixtures/development/06_teams.rb b/db/fixtures/development/06_teams.rb index 94d3aa59710262..f165152909883c 100644 --- a/db/fixtures/development/06_teams.rb +++ b/db/fixtures/development/06_teams.rb @@ -1,9 +1,9 @@ require './spec/support/sidekiq_middleware' Sidekiq::Testing.inline! do - Gitlab::Seeder.quiet do + Gitlab::Seeder.quiet do |seeder| Group.all.each do |group| - User.not_mass_generated.sample(4).each do |user| + seeder.not_mass_generated_users.sample(4).each do |user| if group.add_user(user, Gitlab::Access.values.sample).persisted? print '.' else @@ -12,8 +12,8 @@ end end - Project.not_mass_generated.each do |project| - User.not_mass_generated.sample(4).each do |user| + seeder.not_mass_generated_projects.each do |project| + seeder.not_mass_generated_users.sample(4).each do |user| if project.add_role(user, Gitlab::Access.sym_options.keys.sample) print '.' else diff --git a/db/fixtures/development/07_milestones.rb b/db/fixtures/development/07_milestones.rb index 880a1211c03894..a2d96a4a1e4f83 100644 --- a/db/fixtures/development/07_milestones.rb +++ b/db/fixtures/development/07_milestones.rb @@ -1,7 +1,7 @@ require './spec/support/sidekiq_middleware' -Gitlab::Seeder.quiet do - Project.not_mass_generated.each do |project| +Gitlab::Seeder.quiet do |seeder| + seeder.not_mass_generated_projects.each do |project| 5.times do |i| milestone_params = { title: "v#{i}.0", diff --git a/db/fixtures/development/10_merge_requests.rb b/db/fixtures/development/10_merge_requests.rb index 9157045a7fdc4e..dd955ea0d50aa7 100644 --- a/db/fixtures/development/10_merge_requests.rb +++ b/db/fixtures/development/10_merge_requests.rb @@ -1,13 +1,12 @@ require './spec/support/sidekiq_middleware' -Gitlab::Seeder.quiet do +Gitlab::Seeder.quiet do |seeder| # Limit the number of merge requests per project to avoid long seeds MAX_NUM_MERGE_REQUESTS = 10 - projects = Project + projects = seeder.not_mass_generated_projects .non_archived .with_merge_requests_enabled - .not_mass_generated .reject(&:empty_repo?) projects.each do |project| diff --git a/db/fixtures/development/11_keys.rb b/db/fixtures/development/11_keys.rb index eeee2388d0158a..9bf1046c051ad5 100644 --- a/db/fixtures/development/11_keys.rb +++ b/db/fixtures/development/11_keys.rb @@ -4,12 +4,12 @@ # Creating keys runs a gitlab-shell worker. Since we may not have the right # gitlab-shell path set (yet) we need to disable this for these fixtures. Sidekiq::Testing.disable! do - Gitlab::Seeder.quiet do + Gitlab::Seeder.quiet do |seeder| # We want to run `add_to_shell` immediately instead of after the commit, so # that it falls under `Sidekiq::Testing.disable!`. Key.skip_callback(:commit, :after, :add_to_shell) - User.not_mass_generated.first(10).each do |user| + seeder.not_mass_generated_users.first(10).each do |user| key = "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt#{user.id + 100}6k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=" key = user.keys.create( diff --git a/db/fixtures/development/12_snippets.rb b/db/fixtures/development/12_snippets.rb index 3fa172c8f0f69c..8f2260a3ee147b 100644 --- a/db/fixtures/development/12_snippets.rb +++ b/db/fixtures/development/12_snippets.rb @@ -1,6 +1,6 @@ require './spec/support/sidekiq_middleware' -Gitlab::Seeder.quiet do +Gitlab::Seeder.quiet do |seeder| content =< do - where.not("path LIKE '#{MASS_INSERT_PROJECT_START}%'") - end - end + def self.not_mass_generated_users + User.where.not("username LIKE '#{MASS_INSERT_USER_START}%'") # rubocop:disable CodeReuse/ActiveRecord end - module UserSeed - extend ActiveSupport::Concern - - included do - scope :not_mass_generated, -> do - where.not("username LIKE '#{MASS_INSERT_USER_START}%'") - end - end + def self.not_mass_generated_projects + Project.where.not("path LIKE '#{MASS_INSERT_PROJECT_START}%'") # rubocop:disable CodeReuse/ActiveRecord end def self.with_mass_insert(size, model) @@ -72,47 +64,22 @@ def self.estimated_time_message(size) end end + using MuteNotifications + using DeliverNever + def self.quiet # Disable database insertion logs so speed isn't limited by ability to print to console old_logger = ActiveRecord::Base.logger ActiveRecord::Base.logger = nil - # Additional seed logic for models. - Project.include(ProjectSeed) - User.include(UserSeed) - - mute_notifications - mute_mailer - SeedFu.quiet = true - yield + yield self SeedFu.quiet = false ActiveRecord::Base.logger = old_logger puts "\nOK".color(:green) end - - def self.without_gitaly_timeout - # Remove Gitaly timeout - old_timeout = Gitlab::CurrentSettings.current_application_settings.gitaly_timeout_default - Gitlab::CurrentSettings.current_application_settings.update_columns(gitaly_timeout_default: 0) - # Otherwise we still see the default value when running seed_fu - ApplicationSetting.expire - - yield - ensure - Gitlab::CurrentSettings.current_application_settings.update_columns(gitaly_timeout_default: old_timeout) - ApplicationSetting.expire - end - - def self.mute_notifications - NotificationService.prepend(MuteNotifications) - end - - def self.mute_mailer - ActionMailer::MessageDelivery.prepend(DeliverNever) - end end end # :nocov: diff --git a/lib/tasks/gitlab/seed.rake b/lib/tasks/gitlab/seed.rake index d758280ba69def..61cd79caee40de 100644 --- a/lib/tasks/gitlab/seed.rake +++ b/lib/tasks/gitlab/seed.rake @@ -4,33 +4,35 @@ namespace :gitlab do task :issues, [:project_full_path, :backfill_weeks, :average_issues_per_week] => :environment do |t, args| args.with_defaults(backfill_weeks: 5, average_issues_per_week: 2) - projects = - if args.project_full_path - project = Project.find_by_full_path(args.project_full_path) + Gitlab::Seeder.quiet do |seeder| + projects = + if args.project_full_path + project = Project.find_by_full_path(args.project_full_path) - unless project - error_message = "Project '#{args.project_full_path}' does not exist!" - potential_projects = Project.search(args.project_full_path) + unless project + error_message = "Project '#{args.project_full_path}' does not exist!" + potential_projects = Project.search(args.project_full_path) - if potential_projects.present? - error_message += " Did you mean '#{potential_projects.first.full_path}'?" + if potential_projects.present? + error_message += " Did you mean '#{potential_projects.first.full_path}'?" + end + + puts error_message.color(:red) + exit 1 end - puts error_message.color(:red) - exit 1 + [project] + else + seeder.not_mass_generated_projects.find_each end - [project] - else - Project.not_mass_generated.find_each + projects.each do |project| + puts "\nSeeding issues for the '#{project.full_path}' project" + seeder = Quality::Seeders::Issues.new(project: project) + issues_created = seeder.seed(backfill_weeks: args.backfill_weeks.to_i, + average_issues_per_week: args.average_issues_per_week.to_i) + puts "\n#{issues_created} issues created!" end - - projects.each do |project| - puts "\nSeeding issues for the '#{project.full_path}' project" - seeder = Quality::Seeders::Issues.new(project: project) - issues_created = seeder.seed(backfill_weeks: args.backfill_weeks.to_i, - average_issues_per_week: args.average_issues_per_week.to_i) - puts "\n#{issues_created} issues created!" end end end -- GitLab From 36e745f73f8047a011740cd6bbfc4ddb3a020db8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Tue, 28 Jan 2020 18:24:07 +0100 Subject: [PATCH 2/3] Fix RemoteMirror#mark_as_failed! notification test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- spec/models/remote_mirror_spec.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/models/remote_mirror_spec.rb b/spec/models/remote_mirror_spec.rb index 79d45da8a1e035..72a3e1cfb37d8d 100644 --- a/spec/models/remote_mirror_spec.rb +++ b/spec/models/remote_mirror_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -describe RemoteMirror, :mailer do +describe RemoteMirror do include GitHelpers describe 'URL validation' do @@ -189,10 +189,10 @@ remote_mirror.project.add_maintainer(user) end - it 'notifies the project maintainers', :sidekiq_might_not_need_inline do - perform_enqueued_jobs { subject } + it 'schedule a notification for the remote mirror', :sidekiq_might_not_need_inline do + expect(RemoteMirrorNotificationWorker).to receive(:perform_async).with(remote_mirror.id) - should_email(user) + subject end end end -- GitLab From e2c3a4ccb44e356fa8a6aa94472ebde69cfec3fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 30 Jan 2020 13:30:03 +0100 Subject: [PATCH 3/3] Debug the Seeder module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/gitlab/seeder.rb | 13 +++++++++++++ spec/workers/new_note_worker_spec.rb | 4 ++++ 2 files changed, 17 insertions(+) diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 49a96caf109718..809c5aad5635f3 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -64,6 +64,9 @@ def self.estimated_time_message(size) end end + puts "Seeder required here!" + caller.each { |l| puts l } + using MuteNotifications using DeliverNever @@ -74,8 +77,18 @@ def self.quiet SeedFu.quiet = true + p "In Seeder.quiet before yield" + p NotificationService.method(:new_note) + p NotificationService.method(:new_note).source_location + p NotificationService.method(:new_note).source + yield self + p "In Seeder.quiet after yield" + p NotificationService.method(:new_note) + p NotificationService.method(:new_note).source_location + p NotificationService.method(:new_note).source + SeedFu.quiet = false ActiveRecord::Base.logger = old_logger puts "\nOK".color(:green) diff --git a/spec/workers/new_note_worker_spec.rb b/spec/workers/new_note_worker_spec.rb index ae62237960a70d..42e326b1a0b78b 100644 --- a/spec/workers/new_note_worker_spec.rb +++ b/spec/workers/new_note_worker_spec.rb @@ -8,6 +8,10 @@ it "calls NotificationService#new_note" do expect_next_instance_of(NotificationService) do |service| + p "In expect_next_instance_of(NotificationService)" + p service.method(:new_note) + p service.method(:new_note).source_location + p service.method(:new_note).source expect(service).to receive(:new_note).with(note) end -- GitLab