From 2f95bad11728c7e937df86d2a78eb31c116a9b5c Mon Sep 17 00:00:00 2001 From: Avielle Wolfe Date: Wed, 15 Oct 2025 16:04:06 +0200 Subject: [PATCH 1/6] Persist job inputs spec in Ci::Build options field When a job has inputs defined, we need to: 1. Store the input spec definitions in the options field 2. Create Ci::JobInput records with default values Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/557879 --- lib/gitlab/ci/pipeline/seed/build.rb | 21 +++++ .../lib/gitlab/ci/pipeline/seed/build_spec.rb | 79 +++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 37c73699249b5e..108dd5e3847321 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -19,6 +19,7 @@ def initialize(context, attributes, stages_for_needs_lookup) @job_variables = @seed_attributes.delete(:job_variables) @execution_config_attribute = @seed_attributes.delete(:execution_config) @root_variables_inheritance = @seed_attributes.delete(:root_variables_inheritance) { true } + @inputs = @seed_attributes.delete(:inputs) @using_rules = attributes.key?(:rules) @using_only = attributes.key?(:only) @@ -71,6 +72,7 @@ def attributes .deep_merge(rules_attributes) .deep_merge(allow_failure_criteria_attributes) .deep_merge(@cache.cache_attributes) + .deep_merge(inputs_attributes) .deep_merge(runner_tags) .deep_merge(build_execution_config_attribute) .deep_merge(scoped_user_id_attribute) @@ -238,6 +240,25 @@ def allow_failure_criteria_attributes { options: { allow_failure_criteria: nil } } end + def inputs_attributes + return {} unless @inputs + + { + inputs_attributes: build_inputs_attributes, + options: { inputs: @inputs } + } + end + + def build_inputs_attributes + @inputs.map do |name, spec| + { + project: @pipeline.project, + name: name.to_s, + value: spec[:default] + } + end + end + def calculate_yaml_variables! @seed_attributes[:yaml_variables] = Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index 4e6e53efd0032b..eeeed688a7f3c0 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -475,6 +475,85 @@ end end + context 'with job:inputs' do + let(:attributes) do + { + name: 'rspec', + ref: 'master', + inputs: { + string_input: { + type: 'string', + default: 'default value' + }, + number_input: { + type: 'number', + default: 666 + } + } + } + end + + # Remove when the FF `stop_writing_builds_metadata` is removed. + it { is_expected.not_to have_key(:options) } + + it 'assigns inputs to job definition options' do + expect(seed_attributes[:temp_job_definition]).to have_attributes( + config: a_hash_including( + options: a_hash_including( + inputs: { + string_input: { + type: 'string', + default: 'default value' + }, + number_input: { + type: 'number', + default: 666 + } + } + ) + ) + ) + end + + it 'assigns inputs_attributes for creating input records' do + expect(seed_attributes[:inputs_attributes]).to match_array([ + { + project: pipeline.project, + name: 'string_input', + value: 'default value' + }, + { + project: pipeline.project, + name: 'number_input', + value: 666 + } + ]) + end + + context 'when the FF stop_writing_builds_metadata is disabled' do + before do + stub_feature_flags(stop_writing_builds_metadata: false) + end + + it 'includes inputs in the options hash' do + is_expected.to include( + options: a_hash_including( + inputs: { + string_input: { + type: 'string', + default: 'default value' + }, + number_input: { + type: 'number', + default: 666 + } + } + ) + ) + end + end + end + context 'with cache:key' do let(:attributes) do { -- GitLab From c20fb611d5db35ebb25d9bd3c618ef3f94aa52cb Mon Sep 17 00:00:00 2001 From: Avielle Wolfe Date: Wed, 15 Oct 2025 16:04:32 +0200 Subject: [PATCH 2/6] Rename input_type to type in job inputs spec Changes the job inputs spec field from input_type to type for cleaner JSON structure. This affects: - BuildRunnerPresenter: reads spec[:type] instead of spec[:input_type] - Related specs updated to use type field The backtick workaround will be used in JSON schema validation to handle the type reserved keyword. --- app/presenters/ci/build_runner_presenter.rb | 2 +- spec/lib/api/entities/ci/job_request/response_spec.rb | 2 +- spec/presenters/ci/build_runner_presenter_spec.rb | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/presenters/ci/build_runner_presenter.rb b/app/presenters/ci/build_runner_presenter.rb index de50abf3da0729..b69934e3db012a 100644 --- a/app/presenters/ci/build_runner_presenter.rb +++ b/app/presenters/ci/build_runner_presenter.rb @@ -51,7 +51,7 @@ def runner_inputs key: name, value: { content: input_value, - type: spec[:input_type] + type: spec[:type] } } end diff --git a/spec/lib/api/entities/ci/job_request/response_spec.rb b/spec/lib/api/entities/ci/job_request/response_spec.rb index 2346f78abec2fb..4514733407b44f 100644 --- a/spec/lib/api/entities/ci/job_request/response_spec.rb +++ b/spec/lib/api/entities/ci/job_request/response_spec.rb @@ -7,7 +7,7 @@ let_it_be(:job) do create( :ci_build, runner: runner, - options: { inputs: { test_input: { input_type: 'string', default: 'test' } } } + options: { inputs: { test_input: { type: 'string', default: 'test' } } } ) end diff --git a/spec/presenters/ci/build_runner_presenter_spec.rb b/spec/presenters/ci/build_runner_presenter_spec.rb index 951c367dcf25a0..4275257973d08d 100644 --- a/spec/presenters/ci/build_runner_presenter_spec.rb +++ b/spec/presenters/ci/build_runner_presenter_spec.rb @@ -374,19 +374,19 @@ let(:inputs_spec) do { string_input: { - input_type: 'string', + type: 'string', default: 'default value one' }, array_input: { - input_type: 'array', + type: 'array', default: ['default array'] }, boolean_input: { - input_type: 'boolean', + type: 'boolean', default: false }, number_input: { - input_type: 'number', + type: 'number', default: 666 } } -- GitLab From 50b7a8f791d114a0f3c8deade0c7d31154e4fc00 Mon Sep 17 00:00:00 2001 From: Avielle Wolfe Date: Wed, 15 Oct 2025 16:12:27 +0200 Subject: [PATCH 3/6] Default job input type to string Ensures that job inputs without an explicit type field default to 'string' type, matching the behavior of CI config inputs. --- lib/gitlab/ci/pipeline/seed/build.rb | 11 +++++-- .../lib/gitlab/ci/pipeline/seed/build_spec.rb | 29 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 108dd5e3847321..617dfc576cd4b6 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -245,12 +245,19 @@ def inputs_attributes { inputs_attributes: build_inputs_attributes, - options: { inputs: @inputs } + options: { inputs: normalized_inputs } } end + def normalized_inputs + @inputs.transform_values do |spec| + spec[:type] ||= 'string' + spec + end + end + def build_inputs_attributes - @inputs.map do |name, spec| + normalized_inputs.map do |name, spec| { project: @pipeline.project, name: name.to_s, diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index eeeed688a7f3c0..c8a26978b8c484 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -552,6 +552,35 @@ ) end end + + context 'when input has no type specified' do + let(:attributes) do + { + name: 'rspec', + ref: 'master', + inputs: { + untyped_input: { + default: 'some value' + } + } + } + end + + it 'defaults to string type' do + expect(seed_attributes[:temp_job_definition]).to have_attributes( + config: a_hash_including( + options: a_hash_including( + inputs: { + untyped_input: { + type: 'string', + default: 'some value' + } + } + ) + ) + ) + end + end end context 'with cache:key' do -- GitLab From f6321c8966f4f7c79651c390079e3bdce89de8eb Mon Sep 17 00:00:00 2001 From: Avielle Wolfe Date: Thu, 16 Oct 2025 13:39:09 +0200 Subject: [PATCH 4/6] Simplify inputs processing in Seed::Build Validation will be done during the YAML config processing. All Seed::Build needs to do is prepare the inputs for storage in the database. --- lib/gitlab/ci/pipeline/seed/build.rb | 13 ++------ .../lib/gitlab/ci/pipeline/seed/build_spec.rb | 33 ++----------------- 2 files changed, 5 insertions(+), 41 deletions(-) diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 617dfc576cd4b6..a15c1783cf156d 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -245,22 +245,15 @@ def inputs_attributes { inputs_attributes: build_inputs_attributes, - options: { inputs: normalized_inputs } + options: { inputs: @inputs } } end - def normalized_inputs - @inputs.transform_values do |spec| - spec[:type] ||= 'string' - spec - end - end - def build_inputs_attributes - normalized_inputs.map do |name, spec| + @inputs.map do |name, spec| { project: @pipeline.project, - name: name.to_s, + name: name, value: spec[:default] } end diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index c8a26978b8c484..25768083a37e74 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -519,12 +519,12 @@ expect(seed_attributes[:inputs_attributes]).to match_array([ { project: pipeline.project, - name: 'string_input', + name: :string_input, value: 'default value' }, { project: pipeline.project, - name: 'number_input', + name: :number_input, value: 666 } ]) @@ -552,35 +552,6 @@ ) end end - - context 'when input has no type specified' do - let(:attributes) do - { - name: 'rspec', - ref: 'master', - inputs: { - untyped_input: { - default: 'some value' - } - } - } - end - - it 'defaults to string type' do - expect(seed_attributes[:temp_job_definition]).to have_attributes( - config: a_hash_including( - options: a_hash_including( - inputs: { - untyped_input: { - type: 'string', - default: 'some value' - } - } - ) - ) - ) - end - end end context 'with cache:key' do -- GitLab From 821f10f1c501de2415d17cf35f64539ea2b7f8e4 Mon Sep 17 00:00:00 2001 From: Avielle Wolfe Date: Fri, 17 Oct 2025 17:22:36 +0200 Subject: [PATCH 5/6] Add BulkInsertSafe to Ci::JobInput This change addresses potential N+1 query issues when creating pipelines with multiple jobs that have inputs. By including BulkInsertSafe, job inputs are now inserted in bulk during pipeline creation rather than one-by-one. Also adds a code comment explaining why inputs are stored in two places (options[:inputs] for definitions, ci_job_inputs for values). --- app/models/ci/job_input.rb | 1 + lib/gitlab/ci/pipeline/seed/build.rb | 6 ++++++ spec/models/ci/job_input_spec.rb | 8 ++++++++ 3 files changed, 15 insertions(+) diff --git a/app/models/ci/job_input.rb b/app/models/ci/job_input.rb index 5015e8a39e6b39..7d033c914271de 100644 --- a/app/models/ci/job_input.rb +++ b/app/models/ci/job_input.rb @@ -3,6 +3,7 @@ module Ci class JobInput < Ci::ApplicationRecord include Ci::Partitionable + include BulkInsertSafe MAX_VALUE_SIZE = ::Gitlab::Ci::Config::Interpolation::Access::MAX_ACCESS_BYTESIZE diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index a15c1783cf156d..101280fbc96d42 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -243,6 +243,12 @@ def allow_failure_criteria_attributes def inputs_attributes return {} unless @inputs + # We store inputs in two places: + # 1. options[:inputs] - The input definitions (type, default, etc.) used for: + # - Populating the UI form on the job retry page + # - Validating input values when a job is retried + # 2. inputs_attributes - The actual input values (stored in ci_job_inputs) used for: + # - Job execution (first run uses defaults, retries use user-submitted values) { inputs_attributes: build_inputs_attributes, options: { inputs: @inputs } diff --git a/spec/models/ci/job_input_spec.rb b/spec/models/ci/job_input_spec.rb index 22352950d4de07..92d0a4e56726de 100644 --- a/spec/models/ci/job_input_spec.rb +++ b/spec/models/ci/job_input_spec.rb @@ -14,6 +14,14 @@ let!(:parent) { model.project } end + it_behaves_like 'a BulkInsertSafe model', described_class do + let(:valid_items_for_bulk_insertion) do + build_list(:ci_job_input, 10, job: job, project: project) + end + + let(:invalid_items_for_bulk_insertion) { [] } # name and partition_id are NOT NULL at database level + end + describe 'associations' do it { is_expected.to belong_to(:job) } it { is_expected.to belong_to(:project) } -- GitLab From c2f75ee8195310d70dee8e78d35df1383460c03d Mon Sep 17 00:00:00 2001 From: Avielle Wolfe Date: Wed, 22 Oct 2025 15:47:12 +0200 Subject: [PATCH 6/6] Skip creating Ci::JobInput records for defaults To reduce database growth, we now skip creating Ci::JobInput records when jobs use default input values. Records are only created when a job is retried with user-submitted values. --- app/models/ci/job_input.rb | 9 +++++++++ lib/gitlab/ci/pipeline/seed/build.rb | 17 ----------------- spec/lib/gitlab/ci/pipeline/seed/build_spec.rb | 15 --------------- 3 files changed, 9 insertions(+), 32 deletions(-) diff --git a/app/models/ci/job_input.rb b/app/models/ci/job_input.rb index 7d033c914271de..394b81c9300e7d 100644 --- a/app/models/ci/job_input.rb +++ b/app/models/ci/job_input.rb @@ -1,6 +1,15 @@ # frozen_string_literal: true module Ci + # Stores input values for CI jobs. + # + # Records are only persisted when a job is retried with user-submitted input values. + # On first run, jobs use default values from the input spec stored in options[:inputs], + # and no Ci::JobInput records are created. This avoids storing unnecessary data since + # most jobs use default values. + # + # The fallback logic to default values is implemented in: + # - BuildRunnerPresenter#runner_inputs (for job execution) class JobInput < Ci::ApplicationRecord include Ci::Partitionable include BulkInsertSafe diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 101280fbc96d42..cc1f465e93ba0c 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -243,28 +243,11 @@ def allow_failure_criteria_attributes def inputs_attributes return {} unless @inputs - # We store inputs in two places: - # 1. options[:inputs] - The input definitions (type, default, etc.) used for: - # - Populating the UI form on the job retry page - # - Validating input values when a job is retried - # 2. inputs_attributes - The actual input values (stored in ci_job_inputs) used for: - # - Job execution (first run uses defaults, retries use user-submitted values) { - inputs_attributes: build_inputs_attributes, options: { inputs: @inputs } } end - def build_inputs_attributes - @inputs.map do |name, spec| - { - project: @pipeline.project, - name: name, - value: spec[:default] - } - end - end - def calculate_yaml_variables! @seed_attributes[:yaml_variables] = Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index 25768083a37e74..6a3d5dce981412 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -515,21 +515,6 @@ ) end - it 'assigns inputs_attributes for creating input records' do - expect(seed_attributes[:inputs_attributes]).to match_array([ - { - project: pipeline.project, - name: :string_input, - value: 'default value' - }, - { - project: pipeline.project, - name: :number_input, - value: 666 - } - ]) - end - context 'when the FF stop_writing_builds_metadata is disabled' do before do stub_feature_flags(stop_writing_builds_metadata: false) -- GitLab