From a9105b63f59accb79075261ec706c19fb8e195ac Mon Sep 17 00:00:00 2001
From: lma-git <lma@gitlab.com>
Date: Tue, 5 Aug 2025 09:34:37 -0700
Subject: [PATCH 1/3] Update metadata read methods to use new job definition
 model

Updates the CI::Metadatable module's metadata reader method so that
it reads from job_definition if exists. This affects the fields
`options` and `yaml_variables`. Other fields will be updated in next
MR iterations.

Introduces the feature flag:
`read_from_new_ci_destinations`

This is a first iteration in our effort to move a job's getter/setter
methods into Ci::Metadatable.
---
 app/models/concerns/ci/metadatable.rb         | 17 +++++++---
 .../read_from_new_ci_destinations.yml         | 12 +++++++
 spec/models/ci/build_spec.rb                  | 33 ++++++++++++++++---
 3 files changed, 53 insertions(+), 9 deletions(-)
 create mode 100644 config/feature_flags/gitlab_com_derisk/read_from_new_ci_destinations.yml

diff --git a/app/models/concerns/ci/metadatable.rb b/app/models/concerns/ci/metadatable.rb
index 5c9a285c5c9fd3..a4a7b04f424e0a 100644
--- a/app/models/concerns/ci/metadatable.rb
+++ b/app/models/concerns/ci/metadatable.rb
@@ -7,6 +7,7 @@ module Ci
   #
   module Metadatable
     extend ActiveSupport::Concern
+    include Gitlab::Utils::StrongMemoize
 
     included do
       has_one :metadata,
@@ -85,11 +86,11 @@ def degenerate!
     end
 
     def options
-      read_metadata_attribute(:options, :config_options, {})
+      read_metadata_attribute(:options, :config_options, :options, {})
     end
 
     def yaml_variables
-      read_metadata_attribute(:yaml_variables, :config_variables, [])
+      read_metadata_attribute(:yaml_variables, :config_variables, :yaml_variables, [])
     end
 
     def options=(value)
@@ -152,14 +153,22 @@ def artifacts_paths
 
     private
 
-    def read_metadata_attribute(legacy_key, metadata_key, default_value = nil)
-      read_attribute(legacy_key) || metadata&.read_attribute(metadata_key) || default_value
+    def read_metadata_attribute(legacy_key, metadata_key, job_definition_key, default_value = nil)
+      (legacy_key && read_attribute(legacy_key)) ||
+        (read_from_new_destination? && job_definition&.config && job_definition.config[job_definition_key]) ||
+        metadata&.read_attribute(metadata_key) ||
+        default_value
     end
 
     def write_metadata_attribute(legacy_key, metadata_key, value)
       ensure_metadata.write_attribute(metadata_key, value)
       write_attribute(legacy_key, nil)
     end
+
+    def read_from_new_destination?
+      Feature.enabled?(:read_from_new_ci_destinations, project)
+    end
+    strong_memoize_attr :read_from_new_destination?
   end
 end
 
diff --git a/config/feature_flags/gitlab_com_derisk/read_from_new_ci_destinations.yml b/config/feature_flags/gitlab_com_derisk/read_from_new_ci_destinations.yml
new file mode 100644
index 00000000000000..bb570c47a5dd7d
--- /dev/null
+++ b/config/feature_flags/gitlab_com_derisk/read_from_new_ci_destinations.yml
@@ -0,0 +1,12 @@
+---
+name: read_from_new_ci_destinations
+description: >
+  Controls whether to read job processing data from ci_job_definitions, new columns in ci_builds and
+  new association table for jobs->environment, or from the legacy ci_builds_metadata when disabled.
+feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/552057
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/199971
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/552063
+milestone: '18.3'
+group: group::ci platform
+type: gitlab_com_derisk
+default_enabled: false
diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb
index 3af55863ce669e..6652336174b0f4 100644
--- a/spec/models/ci/build_spec.rb
+++ b/spec/models/ci/build_spec.rb
@@ -4944,16 +4944,18 @@ def run_job_without_exception
 
   describe '#read_metadata_attribute' do
     let(:build) { create(:ci_build, :degenerated, pipeline: pipeline) }
-    let(:build_options) { { key: "build" } }
-    let(:metadata_options) { { key: "metadata" } }
-    let(:default_options) { { key: "default" } }
+    let(:build_options) { { key: 'build' } }
+    let(:job_definition_options) { { key: 'definition' } }
+    let(:metadata_options) { { key: 'metadata' } }
+    let(:default_options) { { key: 'default' } }
 
-    subject { build.send(:read_metadata_attribute, :options, :config_options, default_options) }
+    subject { build.send(:read_metadata_attribute, :options, :config_options, :options, default_options) }
 
-    context 'when build and metadata options is set' do
+    context 'when all destination options are set' do
       before do
         build.write_attribute(:options, build_options)
         build.ensure_metadata.write_attribute(:config_options, metadata_options)
+        build.build_job_definition.write_attribute(:config, { options: job_definition_options })
       end
 
       it 'prefers build options' do
@@ -4961,6 +4963,27 @@ def run_job_without_exception
       end
     end
 
+    context 'when only job definition and metadata options are set' do
+      before do
+        build.ensure_metadata.write_attribute(:config_options, metadata_options)
+        build.build_job_definition.write_attribute(:config, { options: job_definition_options })
+      end
+
+      it 'returns job definition options' do
+        is_expected.to eq(job_definition_options)
+      end
+
+      context 'when FF `read_from_new_ci_destinations` is disabled' do
+        before do
+          stub_feature_flags(read_from_new_ci_destinations: false)
+        end
+
+        it 'returns metadata options' do
+          is_expected.to eq(metadata_options)
+        end
+      end
+    end
+
     context 'when only metadata options is set' do
       before do
         build.write_attribute(:options, nil)
-- 
GitLab


From 4c6231aae941259ea9d506de414143cb5a583f25 Mon Sep 17 00:00:00 2001
From: lma-git <lma@gitlab.com>
Date: Thu, 7 Aug 2025 13:04:00 -0700
Subject: [PATCH 2/3] Update preloads to include job_definition

Adds :job_definition preload wherever job.metadata
is preloaded.
---
 app/finders/deployments_finder.rb                      | 1 +
 app/graphql/resolvers/ci/all_jobs_resolver.rb          | 4 ++--
 app/graphql/resolvers/ci/runner_jobs_resolver.rb       | 1 +
 app/graphql/types/ci/stage_type.rb                     | 2 +-
 app/models/ci/bridge.rb                                | 1 +
 app/models/ci/build.rb                                 | 1 +
 app/models/ci/pipeline.rb                              | 4 ++--
 app/models/ci/stage.rb                                 | 3 ++-
 app/models/concerns/ci/metadatable.rb                  | 6 +++---
 app/models/deployment.rb                               | 3 ++-
 app/presenters/ci/stage_presenter.rb                   | 4 +++-
 app/serializers/environment_serializer.rb              | 5 +++--
 app/serializers/pipeline_serializer.rb                 | 4 ++--
 app/services/ci/drop_pipeline_service.rb               | 2 +-
 ee/app/services/ee/ci/cancel_pipeline_service.rb       | 2 +-
 lib/api/ci/pipelines.rb                                | 2 +-
 lib/gitlab/data_builder/pipeline.rb                    | 1 +
 spec/models/preloaders/commit_status_preloader_spec.rb | 2 +-
 18 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/app/finders/deployments_finder.rb b/app/finders/deployments_finder.rb
index 1c4036703530d5..a01c578b543706 100644
--- a/app/finders/deployments_finder.rb
+++ b/app/finders/deployments_finder.rb
@@ -196,6 +196,7 @@ def preload_associations(scope)
         job_artifacts: [],
         user: [],
         metadata: [],
+        job_definition: [],
         pipeline: {
           project: {
             route: [],
diff --git a/app/graphql/resolvers/ci/all_jobs_resolver.rb b/app/graphql/resolvers/ci/all_jobs_resolver.rb
index 21bd2248d33be1..111053e76036c6 100644
--- a/app/graphql/resolvers/ci/all_jobs_resolver.rb
+++ b/app/graphql/resolvers/ci/all_jobs_resolver.rb
@@ -61,8 +61,8 @@ def preloads
           previous_stage_jobs_or_needs: [:needs, :pipeline],
           artifacts: [:job_artifacts],
           pipeline: [:user],
-          kind: [:metadata],
-          retryable: [:metadata],
+          kind: [:metadata, :job_definition],
+          retryable: [:metadata, :job_definition],
           project: [{ project: [:route, { namespace: [:route] }] }],
           commit_path: [:pipeline, { project: { namespace: [:route] } }],
           ref_path: [{ project: [:route, { namespace: [:route] }] }],
diff --git a/app/graphql/resolvers/ci/runner_jobs_resolver.rb b/app/graphql/resolvers/ci/runner_jobs_resolver.rb
index f68107e14dc1f0..0f93997a382728 100644
--- a/app/graphql/resolvers/ci/runner_jobs_resolver.rb
+++ b/app/graphql/resolvers/ci/runner_jobs_resolver.rb
@@ -35,6 +35,7 @@ def preloads
           project: [{ project: [:route, { namespace: [:route] }, :project_feature] }],
           detailed_status: [
             :metadata,
+            :job_definition,
             { pipeline: [:merge_request] },
             { project: [:route, { namespace: :route }] }
           ],
diff --git a/app/graphql/types/ci/stage_type.rb b/app/graphql/types/ci/stage_type.rb
index e66ad2ed56269f..bbe264dd594a31 100644
--- a/app/graphql/types/ci/stage_type.rb
+++ b/app/graphql/types/ci/stage_type.rb
@@ -63,7 +63,7 @@ def jobs
       def jobs_for_pipeline(pipeline, stage_ids, include_needs)
         jobs = pipeline.statuses.latest.where(stage_id: stage_ids)
 
-        preloaded_relations = [:project, :metadata, :job_artifacts, :downstream_pipeline]
+        preloaded_relations = [:project, :metadata, :job_definition, :job_artifacts, :downstream_pipeline]
         preloaded_relations << :needs if include_needs
 
         Preloaders::CommitStatusPreloader.new(jobs).execute(preloaded_relations)
diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb
index 7eb65334a9e6fa..a295d98fa25dba 100644
--- a/app/models/ci/bridge.rb
+++ b/app/models/ci/bridge.rb
@@ -88,6 +88,7 @@ class Bridge < Ci::Processable
     def self.with_preloads
       preload(
         :metadata,
+        :job_definition,
         user: [:followers, :followees],
         downstream_pipeline: [project: [:route, { namespace: :route }]],
         project: [:namespace]
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index cfa26db8b8aabe..9f4b8462f35cd9 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -172,6 +172,7 @@ class Build < Ci::Processable
     scope :eager_load_for_api, -> do
       preload(
         :job_artifacts_archive, :ci_stage, :job_artifacts, :runner, :tags, :runner_manager, :metadata,
+        :job_definition,
         pipeline: :project,
         user: [:user_preference, :user_detail, :followees, :followers]
       )
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index f160234c9dc8eb..b7bbdd311775b2 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -1176,11 +1176,11 @@ def latest_report_builds(reports_scope = ::Ci::JobArtifact.all_reports)
     end
 
     def latest_test_report_builds_in_self_and_project_descendants
-      latest_report_builds_in_self_and_project_descendants(Ci::JobArtifact.of_report_type(:test)).preload(:project, :metadata, job_artifacts: :artifact_report)
+      latest_report_builds_in_self_and_project_descendants(Ci::JobArtifact.of_report_type(:test)).preload(:project, :metadata, :job_definition, job_artifacts: :artifact_report)
     end
 
     def latest_test_report_builds
-      latest_report_builds(Ci::JobArtifact.of_report_type(:test)).preload(:project, :metadata, job_artifacts: :artifact_report)
+      latest_report_builds(Ci::JobArtifact.of_report_type(:test)).preload(:project, :metadata, :job_definition, job_artifacts: :artifact_report)
     end
 
     def latest_report_builds_in_self_and_project_descendants(reports_scope = ::Ci::JobArtifact.all_reports)
diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb
index 8761c885496e9e..17b7a8334dad95 100644
--- a/app/models/ci/stage.rb
+++ b/app/models/ci/stage.rb
@@ -218,7 +218,8 @@ def ordered_retried_statuses
     private
 
     def preload_metadata(statuses)
-      relations = [:metadata, :pipeline, { downstream_pipeline: [:user, { project: [:route, { namespace: :route }] }] }]
+      relations = [:metadata, :job_definition, :pipeline,
+        { downstream_pipeline: [:user, { project: [:route, { namespace: :route }] }] }]
 
       Preloaders::CommitStatusPreloader.new(statuses).execute(relations)
 
diff --git a/app/models/concerns/ci/metadatable.rb b/app/models/concerns/ci/metadatable.rb
index a4a7b04f424e0a..9142fbfff1ba24 100644
--- a/app/models/concerns/ci/metadatable.rb
+++ b/app/models/concerns/ci/metadatable.rb
@@ -27,14 +27,14 @@ module Metadatable
       before_validation :ensure_metadata, on: :create
 
       scope :with_project_and_metadata, -> do
-        joins(:metadata).includes(:metadata).preload(:project)
+        joins(:metadata).includes(:metadata, :job_definition).preload(:project)
       end
 
       def self.any_with_exposed_artifacts?
         found_exposed_artifacts = false
 
         # TODO: Remove :project preload when FF `ci_use_job_artifacts_table_for_exposed_artifacts` is removed
-        includes(:project).each_batch do |batch|
+        includes(:project, :job_definition).each_batch do |batch|
           # We only load what we need for `has_exposed_artifacts?`
           records = batch.select(:id, :partition_id, :project_id, :options).to_a
 
@@ -60,7 +60,7 @@ def self.any_with_exposed_artifacts?
       end
 
       def self.select_with_exposed_artifacts
-        includes(:metadata, :job_artifacts_metadata, :project).select(&:has_exposed_artifacts?)
+        includes(:metadata, :job_definition, :job_artifacts_metadata, :project).select(&:has_exposed_artifacts?)
       end
     end
 
diff --git a/app/models/deployment.rb b/app/models/deployment.rb
index 4e40874a8cb45a..dd2d5200af3253 100644
--- a/app/models/deployment.rb
+++ b/app/models/deployment.rb
@@ -60,7 +60,8 @@ class Deployment < ApplicationRecord
     preload({ deployable: { runner: [], tags: [], user: [], job_artifacts_archive: [] } })
   end
   scope :with_environment_page_associations, -> do
-    preload(project: [], environment: [], deployable: [:user, :metadata, :project, { pipeline: [:manual_actions] }])
+    preload(project: [], environment: [],
+      deployable: [:user, :metadata, :job_definition, :project, { pipeline: [:manual_actions] }])
   end
 
   scope :finished_after, ->(date) { where('finished_at >= ?', date) }
diff --git a/app/presenters/ci/stage_presenter.rb b/app/presenters/ci/stage_presenter.rb
index bd5bf08abbcad3..a4444afc1edd1b 100644
--- a/app/presenters/ci/stage_presenter.rb
+++ b/app/presenters/ci/stage_presenter.rb
@@ -4,7 +4,9 @@ module Ci
   class StagePresenter < Gitlab::View::Presenter::Delegated
     presents ::Ci::Stage, as: :stage
 
-    PRELOADED_RELATIONS = [:pipeline, :metadata, :tags, :job_artifacts_archive, :downstream_pipeline].freeze
+    PRELOADED_RELATIONS = [
+      :pipeline, :metadata, :job_definition, :tags, :job_artifacts_archive, :downstream_pipeline
+    ].freeze
 
     def latest_ordered_statuses
       preload_statuses(stage.statuses.latest_ordered)
diff --git a/app/serializers/environment_serializer.rb b/app/serializers/environment_serializer.rb
index e241494baa8229..e36e41007cd153 100644
--- a/app/serializers/environment_serializer.rb
+++ b/app/serializers/environment_serializer.rb
@@ -95,9 +95,10 @@ def deployment_associations
       deployable: {
         user: [],
         metadata: [],
+        job_definition: [],
         pipeline: {
-          manual_actions: [:metadata, :deployment],
-          scheduled_actions: [:metadata],
+          manual_actions: [:metadata, :job_definition, :deployment],
+          scheduled_actions: [:metadata, :job_definition],
           latest_successful_jobs: []
         },
         project: project_associations
diff --git a/app/serializers/pipeline_serializer.rb b/app/serializers/pipeline_serializer.rb
index d194d4d0278a8c..02c1750246f33d 100644
--- a/app/serializers/pipeline_serializer.rb
+++ b/app/serializers/pipeline_serializer.rb
@@ -46,8 +46,8 @@ def preloaded_relations(preload_statuses: true, preload_downstream_statuses: tru
       (:limited_failed_builds if disable_failed_builds),
       {
         **(disable_failed_builds ? {} : { failed_builds: %i[project metadata] }),
-        manual_actions: :metadata,
-        scheduled_actions: :metadata,
+        manual_actions: [:metadata, :job_definition],
+        scheduled_actions: [:metadata, :job_definition],
         merge_request: {
           source_project: [:route, { namespace: :route }],
           target_project: [:route, { namespace: :route }]
diff --git a/app/services/ci/drop_pipeline_service.rb b/app/services/ci/drop_pipeline_service.rb
index 94b664c5a5b460..3c9955ce15ae9d 100644
--- a/app/services/ci/drop_pipeline_service.rb
+++ b/app/services/ci/drop_pipeline_service.rb
@@ -2,7 +2,7 @@
 
 module Ci
   class DropPipelineService
-    PRELOADED_RELATIONS = [:project, :pipeline, :metadata, :deployment, :taggings].freeze
+    PRELOADED_RELATIONS = [:project, :pipeline, :metadata, :job_definition, :deployment, :taggings].freeze
 
     # execute service asynchronously for each cancelable pipeline
     def execute_async_for_all(pipelines, failure_reason, context_user)
diff --git a/ee/app/services/ee/ci/cancel_pipeline_service.rb b/ee/app/services/ee/ci/cancel_pipeline_service.rb
index aa55c1f0f94db7..e803b6024f17e0 100644
--- a/ee/app/services/ee/ci/cancel_pipeline_service.rb
+++ b/ee/app/services/ee/ci/cancel_pipeline_service.rb
@@ -9,7 +9,7 @@ module CancelPipelineService
 
       override :build_preloads
       def build_preloads
-        super + [:metadata]
+        super + [:metadata, :job_definition_instance, :job_definition]
       end
     end
   end
diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb
index e368a70d2af968..05b534403f1df3 100644
--- a/lib/api/ci/pipelines.rb
+++ b/lib/api/ci/pipelines.rb
@@ -167,7 +167,7 @@ class Pipelines < ::API::Base
             .new(current_user: current_user, pipeline: pipeline, params: params)
             .execute
 
-          builds = builds.with_preloads.preload(:metadata, :runner_manager, :ci_stage) # rubocop:disable CodeReuse/ActiveRecord -- preload job.archived?
+          builds = builds.with_preloads.preload(:metadata, :job_definition, :runner_manager, :ci_stage) # rubocop:disable CodeReuse/ActiveRecord -- preload job.archived?
 
           present paginate(builds), with: Entities::Ci::Job
         end
diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb
index 54b305391b0e7d..315ad381972bd7 100644
--- a/lib/gitlab/data_builder/pipeline.rb
+++ b/lib/gitlab/data_builder/pipeline.rb
@@ -53,6 +53,7 @@ def preload_builds(pipeline, association)
               job_artifacts_archive: [],
               user: [],
               metadata: [],
+              job_definition: [],
               ci_stage: []
             }
           }
diff --git a/spec/models/preloaders/commit_status_preloader_spec.rb b/spec/models/preloaders/commit_status_preloader_spec.rb
index a028652577dde3..ac03b21a39f95d 100644
--- a/spec/models/preloaders/commit_status_preloader_spec.rb
+++ b/spec/models/preloaders/commit_status_preloader_spec.rb
@@ -13,7 +13,7 @@
   let_it_be(:generic_commit_status2) { create(:generic_commit_status, pipeline: pipeline) }
 
   describe '#execute' do
-    let(:relations) { %i[pipeline metadata tags job_artifacts_archive { downstream_pipeline: [:user] }] }
+    let(:relations) { %i[pipeline metadata job_definition tags job_artifacts_archive { downstream_pipeline: [:user] }] }
     let(:statuses) { CommitStatus.where(commit_id: pipeline.id).all }
 
     subject(:execute) { described_class.new(statuses).execute(relations) }
-- 
GitLab


From 2e277feed75a1eb322e6fa73af0328244464fecd Mon Sep 17 00:00:00 2001
From: lma-git <lma@gitlab.com>
Date: Fri, 8 Aug 2025 13:08:44 -0700
Subject: [PATCH 3/3] Apply minor suggestions

---
 app/models/concerns/ci/metadatable.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/models/concerns/ci/metadatable.rb b/app/models/concerns/ci/metadatable.rb
index 9142fbfff1ba24..99337fbab7de30 100644
--- a/app/models/concerns/ci/metadatable.rb
+++ b/app/models/concerns/ci/metadatable.rb
@@ -27,7 +27,7 @@ module Metadatable
       before_validation :ensure_metadata, on: :create
 
       scope :with_project_and_metadata, -> do
-        joins(:metadata).includes(:metadata, :job_definition).preload(:project)
+        joins(:metadata).includes(:metadata).preload(:project, :job_definition)
       end
 
       def self.any_with_exposed_artifacts?
@@ -155,7 +155,7 @@ def artifacts_paths
 
     def read_metadata_attribute(legacy_key, metadata_key, job_definition_key, default_value = nil)
       (legacy_key && read_attribute(legacy_key)) ||
-        (read_from_new_destination? && job_definition&.config && job_definition.config[job_definition_key]) ||
+        (read_from_new_destination? && job_definition && job_definition.config[job_definition_key]) ||
         metadata&.read_attribute(metadata_key) ||
         default_value
     end
-- 
GitLab