From a2fe5f618689c168d969ec2aa1f44bdeef53c67a Mon Sep 17 00:00:00 2001 From: David Kim Date: Thu, 6 Oct 2022 19:00:03 +1030 Subject: [PATCH 1/9] Create merge_request_diff asynchronously Changelog: changed --- app/models/merge_request.rb | 4 +- .../merge_requests/after_create_service.rb | 2 + app/services/merge_requests/create_service.rb | 7 +++- .../async_merge_request_diff_creation.yml | 8 ++++ spec/models/merge_request_spec.rb | 28 ++++++++++++++ .../merge_requests/create_service_spec.rb | 37 ++++++++++++++++--- 6 files changed, 78 insertions(+), 8 deletions(-) create mode 100644 config/feature_flags/development/async_merge_request_diff_creation.yml diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index fb20d91fa20b02..a9367f9b5aff9a 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -136,7 +136,7 @@ def merge_request_diff before_validation :set_draft_status - after_create :ensure_merge_request_diff + after_create :ensure_merge_request_diff, unless: :async_merge_request_diff_creation after_update :clear_memoized_shas after_update :reload_diff_if_branch_changed after_commit :ensure_metrics, on: [:create, :update], unless: :importing? @@ -146,6 +146,8 @@ def merge_request_diff # It allows us to close or modify broken merge requests attr_accessor :allow_broken + attr_accessor :async_merge_request_diff_creation + # Temporary fields to store compare vars # when creating new merge request attr_accessor :can_be_created, :compare_commits, :diff_options, :compare diff --git a/app/services/merge_requests/after_create_service.rb b/app/services/merge_requests/after_create_service.rb index 9d12eb80eb6c79..20b32dbc2a08ea 100644 --- a/app/services/merge_requests/after_create_service.rb +++ b/app/services/merge_requests/after_create_service.rb @@ -5,6 +5,8 @@ class AfterCreateService < MergeRequests::BaseService include Gitlab::Utils::StrongMemoize def execute(merge_request) + merge_request.ensure_merge_request_diff + prepare_for_mergeability(merge_request) prepare_merge_request(merge_request) end diff --git a/app/services/merge_requests/create_service.rb b/app/services/merge_requests/create_service.rb index 8e0f72eb380872..8930c024276801 100644 --- a/app/services/merge_requests/create_service.rb +++ b/app/services/merge_requests/create_service.rb @@ -34,7 +34,12 @@ def before_create(merge_request) # callback (e.g. after_create), a database transaction will be # open while the Gitaly RPC waits. To avoid an idle in transaction # timeout, we do this before we attempt to save the merge request. - merge_request.eager_fetch_ref! + + if Feature.enabled?(:async_merge_request_diff_creation, current_user) + merge_request.async_merge_request_diff_creation = true + else + merge_request.eager_fetch_ref! + end end def set_projects! diff --git a/config/feature_flags/development/async_merge_request_diff_creation.yml b/config/feature_flags/development/async_merge_request_diff_creation.yml new file mode 100644 index 00000000000000..472bd0aba56085 --- /dev/null +++ b/config/feature_flags/development/async_merge_request_diff_creation.yml @@ -0,0 +1,8 @@ +--- +name: async_merge_request_diff_creation +introduced_by_url: +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/376759 +milestone: '15.5' +type: development +group: group::code review +default_enabled: false diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb index 32518b867cb9ab..a91fb948d28947 100644 --- a/spec/models/merge_request_spec.rb +++ b/spec/models/merge_request_spec.rb @@ -278,6 +278,34 @@ end describe 'callbacks' do + describe '#ensure_merge_request_diff' do + let(:merge_request) { build(:merge_request) } + + context 'when async_merge_request_diff_creation is true' do + before do + merge_request.async_merge_request_diff_creation = true + end + + it 'does not merge request diff after save' do + merge_request.save! + + expect(merge_request.merge_request_diff).to be_empty + end + end + + context 'when async_merge_request_diff_creation is false' do + before do + merge_request.async_merge_request_diff_creation = false + end + + it 'creates merge request diff after save' do + merge_request.save! + + expect(merge_request.merge_request_diff).not_to be_empty + end + end + end + describe '#ensure_merge_request_metrics' do let(:merge_request) { create(:merge_request) } diff --git a/spec/services/merge_requests/create_service_spec.rb b/spec/services/merge_requests/create_service_spec.rb index 0bc8258af428b1..989c9fb84d54a6 100644 --- a/spec/services/merge_requests/create_service_spec.rb +++ b/spec/services/merge_requests/create_service_spec.rb @@ -495,15 +495,40 @@ project.add_developer(user) end - it 'creates the merge request', :sidekiq_might_not_need_inline do - expect_next_instance_of(MergeRequest) do |instance| - expect(instance).to receive(:eager_fetch_ref!).and_call_original + context 'when async_merge_request_diff_creation is enabled' do + before do + stub_feature_flags(async_merge_request_diff_creation: true) end - merge_request = described_class.new(project: project, current_user: user, params: opts).execute + it 'creates the merge request', :sidekiq_might_not_need_inline do + expect_next_instance_of(MergeRequest) do |instance| + expect(instance).not_to receive(:eager_fetch_ref!) + end - expect(merge_request).to be_persisted - expect(merge_request.iid).to be > 0 + merge_request = described_class.new(project: project, current_user: user, params: opts).execute + + expect(merge_request).to be_persisted + expect(merge_request.iid).to be > 0 + expect(merge_request.merge_request_diff).not_to be_empty + end + end + + context 'when async_merge_request_diff_creation is disabled' do + before do + stub_feature_flags(async_merge_request_diff_creation: false) + end + + it 'creates the merge request', :sidekiq_might_not_need_inline do + expect_next_instance_of(MergeRequest) do |instance| + expect(instance).to receive(:eager_fetch_ref!).and_call_original + end + + merge_request = described_class.new(project: project, current_user: user, params: opts).execute + + expect(merge_request).to be_persisted + expect(merge_request.iid).to be > 0 + expect(merge_request.merge_request_diff).not_to be_empty + end end it 'does not create the merge request when the target project is archived' do -- GitLab From c011db965a840494b543f380c15cec830a51820f Mon Sep 17 00:00:00 2001 From: David Kim Date: Mon, 10 Oct 2022 18:33:42 +1030 Subject: [PATCH 2/9] Inline sidekiq for some specs --- .../analytics/cycle_analytics/filters_and_data_spec.rb | 4 ++-- .../user_creates_merge_request_with_blocking_mrs_spec.rb | 2 +- .../merge_request/user_sets_approval_rules_spec.rb | 2 +- .../features/merge_request/user_sets_approvers_spec.rb | 4 ++-- .../models/analytics/cycle_analytics/group_level_spec.rb | 2 +- spec/features/cycle_analytics_spec.rb | 8 ++++---- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ee/spec/features/groups/analytics/cycle_analytics/filters_and_data_spec.rb b/ee/spec/features/groups/analytics/cycle_analytics/filters_and_data_spec.rb index cf671548306c3f..83d78734df155e 100644 --- a/ee/spec/features/groups/analytics/cycle_analytics/filters_and_data_spec.rb +++ b/ee/spec/features/groups/analytics/cycle_analytics/filters_and_data_spec.rb @@ -291,7 +291,7 @@ def vsa_stages(selected_group) end end - context 'with lots of data', :js do + context 'with lots of data', :js, :sidekiq_inline do let(:issue) { create(:issue, project: project) } around do |example| @@ -332,7 +332,7 @@ def vsa_stages(selected_group) { title: 'Test', description: 'Total test time for all commits/merges', events_count: 0, time: "-" } ] - it 'each stage with events will display the stage events list when selected', :sidekiq_might_not_need_inline do + it 'each stage with events will display the stage events list when selected' do stages_without_data.each do |stage| select_stage(stage[:title]) expect(page).not_to have_selector('[data-testid="vsa-stage-event"]') diff --git a/ee/spec/features/merge_request/user_creates_merge_request_with_blocking_mrs_spec.rb b/ee/spec/features/merge_request/user_creates_merge_request_with_blocking_mrs_spec.rb index 76b94702c09333..a1eca344efeaa7 100644 --- a/ee/spec/features/merge_request/user_creates_merge_request_with_blocking_mrs_spec.rb +++ b/ee/spec/features/merge_request/user_creates_merge_request_with_blocking_mrs_spec.rb @@ -17,7 +17,7 @@ stub_licensed_features(blocking_merge_requests: true) end - it 'creates a merge request with a blocking MR' do + it 'creates a merge request with a blocking MR', :sidekiq_inline do other_mr = create(:merge_request) other_mr.target_project.team.add_maintainer(user) diff --git a/ee/spec/features/merge_request/user_sets_approval_rules_spec.rb b/ee/spec/features/merge_request/user_sets_approval_rules_spec.rb index 668d22fb3680f8..38321b14fcd811 100644 --- a/ee/spec/features/merge_request/user_sets_approval_rules_spec.rb +++ b/ee/spec/features/merge_request/user_sets_approval_rules_spec.rb @@ -65,7 +65,7 @@ def page_rule_names end end - it "persists hidden groups that author has no access to when creating MR" do + it "persists hidden groups that author has no access to when creating MR", :sidekiq_inline do click_on("Create merge request") wait_for_requests diff --git a/ee/spec/features/merge_request/user_sets_approvers_spec.rb b/ee/spec/features/merge_request/user_sets_approvers_spec.rb index 40ec8aed5a9673..0ea51f8b046d6f 100644 --- a/ee/spec/features/merge_request/user_sets_approvers_spec.rb +++ b/ee/spec/features/merge_request/user_sets_approvers_spec.rb @@ -69,7 +69,7 @@ sign_in(user) end - it 'allows setting groups as approvers' do + it 'allows setting groups as approvers', :sidekiq_inline do visit project_new_merge_request_path(project, merge_request: { target_branch: 'master', source_branch: 'feature' }) open_modal(text: 'Add approval rule') @@ -97,7 +97,7 @@ expect(page).to have_selector("img[alt='#{other_user.name}']") end - it 'allows delete approvers group when it is set in project' do + it 'allows delete approvers group when it is set in project', :sidekiq_inline do approver = create :user project.add_developer(approver) group.add_developer(approver) diff --git a/ee/spec/models/analytics/cycle_analytics/group_level_spec.rb b/ee/spec/models/analytics/cycle_analytics/group_level_spec.rb index 2192b96175cf76..bd6cd997959fb0 100644 --- a/ee/spec/models/analytics/cycle_analytics/group_level_spec.rb +++ b/ee/spec/models/analytics/cycle_analytics/group_level_spec.rb @@ -38,7 +38,7 @@ ::Dora::DailyMetrics::RefreshWorker.new.perform(environment.id, pipeline.created_at.to_date.to_s) end - it 'returns medians for each stage for a specific group' do + it 'returns medians for each stage for a specific group', :sidekiq_inline do expect(subject.summary.map { |summary| summary[:value] }).to contain_exactly('0.1', '1', '1') end end diff --git a/spec/features/cycle_analytics_spec.rb b/spec/features/cycle_analytics_spec.rb index 488a4f842975c7..8de4c66c62f65b 100644 --- a/spec/features/cycle_analytics_spec.rb +++ b/spec/features/cycle_analytics_spec.rb @@ -56,7 +56,7 @@ def set_daterange(from_date, to_date) end end - context "when there's value stream analytics data" do + context "when there's value stream analytics data", :sidekiq_inline do # NOTE: in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68595 travel back # 5 days in time before we create data for these specs, to mitigate some flakiness # So setting the date range to be the last 2 days should skip past the existing data @@ -103,7 +103,7 @@ def set_daterange(from_date, to_date) end end - it 'shows data on each stage', :sidekiq_might_not_need_inline, quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338332' do + it 'shows data on each stage', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338332' do expect_issue_to_be_present click_stage('Plan') @@ -207,11 +207,11 @@ def go_to_next_page wait_for_requests end - it 'does not show the commit stats' do + it 'does not show the commit stats', :sidekiq_inline do expect(page.find(metrics_selector)).not_to have_selector("#commits") end - it 'does not show restricted stages', :aggregate_failures do + it 'does not show restricted stages', :aggregate_failures, :sidekiq_inline do expect(find(stage_table_selector)).to have_content(issue.title) expect(page).to have_selector('.gl-path-nav-list-item', text: 'Issue') -- GitLab From 27e6dc4292d9d12c6062ad33d107477d68395587 Mon Sep 17 00:00:00 2001 From: Danger bot Date: Tue, 11 Oct 2022 07:56:45 +0000 Subject: [PATCH 3/9] Apply 1 suggestion(s) to 1 file(s) --- .../development/async_merge_request_diff_creation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/feature_flags/development/async_merge_request_diff_creation.yml b/config/feature_flags/development/async_merge_request_diff_creation.yml index 472bd0aba56085..eefbbf26aa8fff 100644 --- a/config/feature_flags/development/async_merge_request_diff_creation.yml +++ b/config/feature_flags/development/async_merge_request_diff_creation.yml @@ -1,6 +1,6 @@ --- name: async_merge_request_diff_creation -introduced_by_url: +introduced_by_url: "https://gitlab.com/gitlab-org/gitlab/-/merge_requests/100390" rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/376759 milestone: '15.5' type: development -- GitLab From 0fe8f6633f1e85abaecd404a44685ca9745a0e6f Mon Sep 17 00:00:00 2001 From: David Kim Date: Wed, 12 Oct 2022 16:40:25 +1030 Subject: [PATCH 4/9] Address various code review comments --- app/models/merge_request.rb | 4 ++-- app/services/merge_requests/create_service.rb | 2 +- spec/models/merge_request_spec.rb | 4 ++-- spec/services/merge_requests/create_service_spec.rb | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index a9367f9b5aff9a..c55f44e7ba251d 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -136,7 +136,7 @@ def merge_request_diff before_validation :set_draft_status - after_create :ensure_merge_request_diff, unless: :async_merge_request_diff_creation + after_create :ensure_merge_request_diff, unless: :skip_ensure_merge_request_diff after_update :clear_memoized_shas after_update :reload_diff_if_branch_changed after_commit :ensure_metrics, on: [:create, :update], unless: :importing? @@ -146,7 +146,7 @@ def merge_request_diff # It allows us to close or modify broken merge requests attr_accessor :allow_broken - attr_accessor :async_merge_request_diff_creation + attr_accessor :skip_ensure_merge_request_diff # Temporary fields to store compare vars # when creating new merge request diff --git a/app/services/merge_requests/create_service.rb b/app/services/merge_requests/create_service.rb index 8930c024276801..9303b0c4e517ce 100644 --- a/app/services/merge_requests/create_service.rb +++ b/app/services/merge_requests/create_service.rb @@ -36,7 +36,7 @@ def before_create(merge_request) # timeout, we do this before we attempt to save the merge request. if Feature.enabled?(:async_merge_request_diff_creation, current_user) - merge_request.async_merge_request_diff_creation = true + merge_request.skip_ensure_merge_request_diff = true else merge_request.eager_fetch_ref! end diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb index a91fb948d28947..5e5b308fdc5312 100644 --- a/spec/models/merge_request_spec.rb +++ b/spec/models/merge_request_spec.rb @@ -283,7 +283,7 @@ context 'when async_merge_request_diff_creation is true' do before do - merge_request.async_merge_request_diff_creation = true + merge_request.skip_ensure_merge_request_diff = true end it 'does not merge request diff after save' do @@ -295,7 +295,7 @@ context 'when async_merge_request_diff_creation is false' do before do - merge_request.async_merge_request_diff_creation = false + merge_request.skip_ensure_merge_request_diff = false end it 'creates merge request diff after save' do diff --git a/spec/services/merge_requests/create_service_spec.rb b/spec/services/merge_requests/create_service_spec.rb index 989c9fb84d54a6..0892b91032bfbc 100644 --- a/spec/services/merge_requests/create_service_spec.rb +++ b/spec/services/merge_requests/create_service_spec.rb @@ -500,7 +500,7 @@ stub_feature_flags(async_merge_request_diff_creation: true) end - it 'creates the merge request', :sidekiq_might_not_need_inline do + it 'creates the merge request', :sidekiq_inline do expect_next_instance_of(MergeRequest) do |instance| expect(instance).not_to receive(:eager_fetch_ref!) end @@ -518,7 +518,7 @@ stub_feature_flags(async_merge_request_diff_creation: false) end - it 'creates the merge request', :sidekiq_might_not_need_inline do + it 'creates the merge request' do expect_next_instance_of(MergeRequest) do |instance| expect(instance).to receive(:eager_fetch_ref!).and_call_original end -- GitLab From 68c335ba822494b065f65774606561312ef41622 Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Wed, 19 Oct 2022 00:59:31 +0000 Subject: [PATCH 5/9] Apply 1 suggestion(s) to 1 file(s) --- spec/models/merge_request_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb index 5e5b308fdc5312..6b49fc80c58c77 100644 --- a/spec/models/merge_request_spec.rb +++ b/spec/models/merge_request_spec.rb @@ -286,7 +286,7 @@ merge_request.skip_ensure_merge_request_diff = true end - it 'does not merge request diff after save' do + it 'does not create a merge_request_diff after create' do merge_request.save! expect(merge_request.merge_request_diff).to be_empty -- GitLab From 1791e505a44b14152dc5b8049ac11584f0ab8bbb Mon Sep 17 00:00:00 2001 From: Kerri Miller Date: Wed, 19 Oct 2022 00:59:37 +0000 Subject: [PATCH 6/9] Apply 1 suggestion(s) to 1 file(s) --- spec/models/merge_request_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb index 6b49fc80c58c77..21bba0c270d0e4 100644 --- a/spec/models/merge_request_spec.rb +++ b/spec/models/merge_request_spec.rb @@ -298,7 +298,7 @@ merge_request.skip_ensure_merge_request_diff = false end - it 'creates merge request diff after save' do + it 'creates merge_request_diff after create' do merge_request.save! expect(merge_request.merge_request_diff).not_to be_empty -- GitLab From b2c9b634c3a0aa775596190904278ca7d729c467 Mon Sep 17 00:00:00 2001 From: David Kim Date: Wed, 19 Oct 2022 11:32:53 +1030 Subject: [PATCH 7/9] Add comment regarding skip_ensure_merge_request_diff --- app/models/merge_request.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index c55f44e7ba251d..0f47e6d39366ff 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -146,6 +146,8 @@ def merge_request_diff # It allows us to close or modify broken merge requests attr_accessor :allow_broken + # Temporary flag to skip merge_request_diff creation on create. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/100390 attr_accessor :skip_ensure_merge_request_diff # Temporary fields to store compare vars -- GitLab From cc32f5a5a5066b42a27c589ae2308102f229a02e Mon Sep 17 00:00:00 2001 From: David Kim Date: Wed, 19 Oct 2022 13:48:53 +1030 Subject: [PATCH 8/9] Fix broken specs --- spec/requests/projects/cycle_analytics_events_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/requests/projects/cycle_analytics_events_spec.rb b/spec/requests/projects/cycle_analytics_events_spec.rb index 65540f86d34763..370febf82ff49e 100644 --- a/spec/requests/projects/cycle_analytics_events_spec.rb +++ b/spec/requests/projects/cycle_analytics_events_spec.rb @@ -9,7 +9,7 @@ let(:project) { create(:project, :repository, public_builds: false) } let(:issue) { create(:issue, project: project, created_at: 2.days.ago) } - describe 'GET /:namespace/:project/value_stream_analytics/events/issues' do + describe 'GET /:namespace/:project/value_stream_analytics/events/issues', :sidekiq_inline do let(:first_issue_iid) { project.issues.sort_by_attribute(:created_desc).pick(:iid).to_s } let(:first_mr_iid) { project.merge_requests.sort_by_attribute(:created_desc).pick(:iid).to_s } @@ -65,7 +65,7 @@ expect(json_response['events'].first['iid']).to eq(first_mr_iid) end - it 'lists the staging events', :sidekiq_inline do + it 'lists the staging events' do get project_cycle_analytics_staging_path(project, format: :json) expect(json_response['events']).not_to be_empty -- GitLab From 4711a6a575936864332a63ebb66c23b82a840025 Mon Sep 17 00:00:00 2001 From: David Kim Date: Wed, 19 Oct 2022 15:04:58 +1030 Subject: [PATCH 9/9] Update milestone --- .../development/async_merge_request_diff_creation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/feature_flags/development/async_merge_request_diff_creation.yml b/config/feature_flags/development/async_merge_request_diff_creation.yml index eefbbf26aa8fff..8e4bdd13b2b4f9 100644 --- a/config/feature_flags/development/async_merge_request_diff_creation.yml +++ b/config/feature_flags/development/async_merge_request_diff_creation.yml @@ -2,7 +2,7 @@ name: async_merge_request_diff_creation introduced_by_url: "https://gitlab.com/gitlab-org/gitlab/-/merge_requests/100390" rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/376759 -milestone: '15.5' +milestone: '15.6' type: development group: group::code review default_enabled: false -- GitLab