From 1861e7bb04043da0336e2d0120bf3a66315eec67 Mon Sep 17 00:00:00 2001 From: Gregory Havenga <11164960-ghavenga@users.noreply.gitlab.com> Date: Tue, 21 Oct 2025 15:03:13 +0200 Subject: [PATCH 1/5] Enforce passing of database trigger feature flag for Vulnerability Read Creation In the interest of easier reviewability, this MR takes the responsibility of enforcing that the database trigger for vulnerability_reads creation from https://gitlab.com/gitlab-org/gitlab/-/merge_requests/199517 so that that one can focus on the new vulnerability read upsert service in isolation. This MR ensures that we're safely passing the FF to the DB trigger everywhere applicable by causing tests to fail if the flag is not found in the transaction config of classes applicable to the creation of Vulnerability Reads. --- app/models/sec_application_record.rb | 52 ++++++++++++++++--- ...nforce_vulnerability_read_db_trigger_ff.rb | 30 +++++++++++ 2 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 ee/app/models/concerns/vulnerabilities/enforce_vulnerability_read_db_trigger_ff.rb diff --git a/app/models/sec_application_record.rb b/app/models/sec_application_record.rb index a64990f9ee75f8..3169270241c8dc 100644 --- a/app/models/sec_application_record.rb +++ b/app/models/sec_application_record.rb @@ -1,13 +1,51 @@ # frozen_string_literal: true class SecApplicationRecord < ApplicationRecord - self.abstract_class = true + self.abstract_class = true - connects_to database: { writing: :sec, reading: :sec } if Gitlab::Database.has_config?(:sec) + connects_to database: { writing: :sec, reading: :sec } if Gitlab::Database.has_config?(:sec) - class << self - def backup_model - Vulnerabilities::Backup.descendants.find { |descendant| self == descendant.original_model } - end - end + UnflaggedVulnReadDatabaseTriggerTransaction = Class.new(StandardError) + + class << self + def backup_model + Vulnerabilities::Backup.descendants.find { |descendant| self == descendant.original_model } + end + + #################### + # This transaction code exists to help identify and prevent instances of code that may need to explicitly pass + # a feature flag setting to the vulnerability reads database trigger. + # + # Once transitioned away from the database trigger, we can remove it. + def feature_flagged_transaction_for(projects) + SecApplicationRecord.transaction do + ::SecApplicationRecord.pass_feature_flag_to_vuln_reads_db_trigger(projects) + + yield + end + end + + def db_trigger_flag_not_set? + result = ::SecApplicationRecord.connection.execute( + "SELECT current_setting('vulnerability_management.dont_execute_db_trigger', true);" + ).first['current_setting'] + + result ? result.empty? : result.nil? + end + + def pass_feature_flag_to_vuln_reads_db_trigger(projects) + feature_enabled = if projects.nil? + Feature.enabled?(:turn_off_vulnerability_read_create_db_trigger_function, :instance) + else + Array(projects).all? do |project| + Feature.enabled?( + :turn_off_vulnerability_read_create_db_trigger_function, project) + end + end + + ::SecApplicationRecord.connection.execute("SELECT set_config( + 'vulnerability_management.dont_execute_db_trigger', '#{feature_enabled}', true);") + end + ################## + end end diff --git a/ee/app/models/concerns/vulnerabilities/enforce_vulnerability_read_db_trigger_ff.rb b/ee/app/models/concerns/vulnerabilities/enforce_vulnerability_read_db_trigger_ff.rb new file mode 100644 index 00000000000000..b9d55128ef16d8 --- /dev/null +++ b/ee/app/models/concerns/vulnerabilities/enforce_vulnerability_read_db_trigger_ff.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +# This transaction code exists to help identify and prevent instances of code +# that may need to explicitly pass a feature flag setting to the vulnerability +# reads database trigger. +# +# Once transitioned away from the database trigger, we can remove it. + +module Vulnerabilities + module EnforceVulnerabilityReadDbTriggerFf + UnflaggedVulnReadDatabaseTriggerTransaction = Class.new(StandardError) + + def self.extended(base) + base.define_singleton_method(:transaction) do |*args, **kwargs, &block| + if base.db_trigger_flag_not_set? + if Rails.env.test? + raise UnflaggedVulnReadDatabaseTriggerTransaction, + 'This transaction is not passing the needed feature flag to the vulnerability read db trigger.' + else + Gitlab::AppLogger.warn( + "Sec transaction executed without setting vulnerability read db trigger feature flag!" + ) + end + end + + super(*args, **kwargs, &block) + end + end + end +end -- GitLab From f6a60c1e50872ae1c9bae2a8161ab222dbefaa98 Mon Sep 17 00:00:00 2001 From: Gregory Havenga <11164960-ghavenga@users.noreply.gitlab.com> Date: Tue, 21 Oct 2025 15:03:34 +0200 Subject: [PATCH 2/5] Add EnforceVulnerabilityReadDbTriggerFf to Vulnerability model --- ee/app/models/ee/vulnerability.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/ee/app/models/ee/vulnerability.rb b/ee/app/models/ee/vulnerability.rb index d038afd3ddfcc7..1fd1c3bd53af29 100644 --- a/ee/app/models/ee/vulnerability.rb +++ b/ee/app/models/ee/vulnerability.rb @@ -21,6 +21,7 @@ module Vulnerability include ::VulnerabilityScopes include ::Gitlab::Utils::StrongMemoize include ::Elastic::ApplicationVersionedSearch + extend ::Vulnerabilities::EnforceVulnerabilityReadDbTriggerFf extend ::Gitlab::Utils::Override -- GitLab From eca209072474cbd7be55c99bd82bb56896efcab1 Mon Sep 17 00:00:00 2001 From: Gregory Havenga <11164960-ghavenga@users.noreply.gitlab.com> Date: Tue, 21 Oct 2025 15:06:50 +0200 Subject: [PATCH 3/5] Add EnforceVulnerabilityReadDbTriggerFf to Finding model --- ee/app/models/vulnerabilities/finding.rb | 1195 +++++++++++----------- 1 file changed, 598 insertions(+), 597 deletions(-) diff --git a/ee/app/models/vulnerabilities/finding.rb b/ee/app/models/vulnerabilities/finding.rb index 1fe365e9b608be..8253ddde730544 100644 --- a/ee/app/models/vulnerabilities/finding.rb +++ b/ee/app/models/vulnerabilities/finding.rb @@ -1,603 +1,604 @@ # frozen_string_literal: true module Vulnerabilities - class Finding < ::SecApplicationRecord - include ShaAttribute - include ::Gitlab::Utils::StrongMemoize - include Presentable - include ::VulnerabilityFindingHelpers - include EachBatch - - # https://gitlab.com/groups/gitlab-org/-/epics/3148 - # https://gitlab.com/gitlab-org/gitlab/-/issues/214563#note_370782508 is why the table names are not renamed - self.table_name = 'vulnerability_occurrences' - - FINDINGS_PER_PAGE = 20 - MAX_NUMBER_OF_IDENTIFIERS = 20 - REPORT_TYPES_WITH_LOCATION_IMAGE = %w[container_scanning cluster_image_scanning].freeze - SECRET_DETECTION_DEFAULT_COMMIT_SHA = "0000000" - - AI_ALLOWED_REPORT_TYPES = %w[sast].freeze - - # https://gitlab.com/gitlab-org/gitlab/-/issues/472861 - HIGH_CONFIDENCE_AI_RESOLUTION_CWES = %w[ - CWE-23 - CWE-73 - CWE-78 - CWE-80 - CWE-89 - CWE-116 - CWE-118 - CWE-119 - CWE-120 - CWE-126 - CWE-190 - CWE-200 - CWE-208 - CWE-209 - CWE-272 - CWE-287 - CWE-295 - CWE-297 - CWE-305 - CWE-310 - CWE-311 - CWE-323 - CWE-327 - CWE-328 - CWE-330 - CWE-338 - CWE-345 - CWE-346 - CWE-352 - CWE-362 - CWE-369 - CWE-377 - CWE-378 - CWE-400 - CWE-489 - CWE-521 - CWE-539 - CWE-599 - CWE-611 - CWE-676 - CWE-704 - CWE-754 - CWE-770 - CWE-1004 - CWE-1275 - ].to_set.freeze - - paginates_per FINDINGS_PER_PAGE - - sha_attribute :location_fingerprint - - attr_readonly :initial_pipeline_id - - belongs_to :project, inverse_of: :vulnerability_findings - belongs_to :scanner, class_name: 'Vulnerabilities::Scanner' - belongs_to :primary_identifier, class_name: 'Vulnerabilities::Identifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id' - belongs_to :vulnerability, class_name: 'Vulnerability', inverse_of: :findings, foreign_key: 'vulnerability_id' - has_one :one_vulnerability, class_name: 'Vulnerability', inverse_of: :vulnerability_finding - has_many :state_transitions, through: :vulnerability - has_many :issue_links, through: :vulnerability - has_many :external_issue_links, through: :vulnerability - has_many :merge_request_links, through: :vulnerability - - has_many :finding_identifiers, class_name: 'Vulnerabilities::FindingIdentifier', inverse_of: :finding, foreign_key: 'occurrence_id' - has_many :identifiers, through: :finding_identifiers, class_name: 'Vulnerabilities::Identifier' - - has_many :cve_identifiers, -> { where('LOWER(external_type) = ?', 'cve') }, through: :finding_identifiers, source: :identifier, class_name: 'Vulnerabilities::Identifier' - has_many :cve_enrichments, through: :cve_identifiers, disable_joins: true - - has_one :finding_token_status, class_name: 'Vulnerabilities::FindingTokenStatus', foreign_key: 'vulnerability_occurrence_id', inverse_of: :finding - - has_many :finding_links, class_name: 'Vulnerabilities::FindingLink', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' - - has_many :finding_remediations, class_name: 'Vulnerabilities::FindingRemediation', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' - has_many :remediations, through: :finding_remediations - - has_many :triggered_workflows, class_name: '::Vulnerabilities::TriggeredWorkflow', inverse_of: :vulnerability_occurrence, foreign_key: 'vulnerability_occurrence_id' - - has_one :finding_risk_score, class_name: 'Vulnerabilities::FindingRiskScore', inverse_of: :finding - - # rubocop: disable Rails/InverseOf -- these relations are not present on Ci::Pipeline - belongs_to :initial_finding_pipeline, class_name: '::Ci::Pipeline', foreign_key: 'initial_pipeline_id' - belongs_to :latest_finding_pipeline, class_name: '::Ci::Pipeline', foreign_key: 'latest_pipeline_id' - # rubocop:enable Rails/InverseOf - - has_many :signatures, class_name: 'Vulnerabilities::FindingSignature', inverse_of: :finding - - has_many :vulnerability_flags, class_name: 'Vulnerabilities::Flag', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' - - has_many :feedbacks, class_name: 'Vulnerabilities::Feedback', inverse_of: :finding, primary_key: 'uuid', foreign_key: 'finding_uuid' - - has_one :finding_evidence, class_name: 'Vulnerabilities::Finding::Evidence', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' - - has_many :security_findings, - class_name: 'Security::Finding', - primary_key: :uuid, - foreign_key: :uuid, - inverse_of: :vulnerability_finding - - attribute :config_options, ::Gitlab::Database::Type::IndifferentJsonb.new - - attr_writer :sha - attr_accessor :scan, :found_by_pipeline - - enum :report_type, ::Enums::Vulnerability.report_types - enum :severity, ::Enums::Vulnerability.severity_levels, prefix: :severity - enum :detection_method, ::Enums::Vulnerability.detection_methods - - validates :scanner, presence: true - validates :project, presence: true - validates :uuid, presence: true - - validates :primary_identifier, presence: true - validates :location_fingerprint, presence: true - # Uniqueness validation doesn't work with binary columns, so save this useless query. It is enforce by DB constraint anyway. - # TODO: find out why it fails - # validates :location_fingerprint, presence: true, uniqueness: { scope: [:primary_identifier_id, :scanner_id, :ref, :pipeline_id, :project_id] } - validates :name, presence: true - validates :report_type, presence: true - validates :severity, presence: true - validates :detection_method, presence: true - - validates :metadata_version, presence: true - validates :raw_metadata, presence: true - validates :details, json_schema: { filename: 'vulnerability_finding_details' } - - COLUMN_LENGTH_LIMITS = { - description: 15_000, - solution: 7_000 - }.freeze - - validates :description, length: { maximum: COLUMN_LENGTH_LIMITS[:description] } - validates :solution, length: { maximum: COLUMN_LENGTH_LIMITS[:solution] } - validates :cve, length: { maximum: 48400 } - - delegate :name, :external_id, to: :scanner, prefix: true, allow_nil: true - - scope :report_type, ->(type) { where(report_type: report_types[type]) } - scope :ordered, -> { order(severity: :desc, id: :asc) } - - scope :active, -> { joins(:vulnerability).merge(Vulnerability.active) } - scope :by_vulnerability, ->(vulnerability_id) { where(vulnerability: vulnerability_id) } - scope :ids_by_vulnerability, ->(vulnerability_id) { by_vulnerability(vulnerability_id).pluck(:id) } - scope :by_report_types, ->(values) { where(report_type: values) } - scope :by_projects, ->(values) { where(project_id: values) } - scope :by_scanners, ->(values) { where(scanner_id: values) } - scope :by_severities, ->(values) { where(severity: values) } - scope :by_location_fingerprints, ->(values) { where(location_fingerprint: values) } - scope :by_uuid, ->(uuids) { where(uuid: uuids) } - scope :excluding_uuids, ->(uuids) { where.not(uuid: uuids) } - scope :eager_load_comparison_entities, -> { includes(:scanner, :primary_identifier) } - scope :by_primary_identifiers, ->(identifier_ids) { where(primary_identifier: identifier_ids) } - scope :by_latest_pipeline, ->(pipeline_id) { where(latest_pipeline_id: pipeline_id) } - scope :with_project, -> { includes(:project) } - scope :with_token_status, -> { preload(:finding_token_status) } - scope :with_cve_enrichments, -> { includes(:cve_enrichments) } - scope :with_risk_score, -> { preload(:finding_risk_score) } - - scope :all_preloaded, -> do - preload(:scanner, :identifiers, :feedbacks, project: [:namespace, :project_feature]) - end - - scope :with_false_positive, ->(false_positive) do - flags = ::Vulnerabilities::Flag.arel_table - - where( - false_positive ? 'EXISTS (?)' : 'NOT EXISTS (?)', - ::Vulnerabilities::Flag.select(1).false_positive.where(flags[:vulnerability_occurrence_id].eq(arel_table[:id])) - ) - end - - scope :with_fix_available, ->(fix_available) do - remediation = ::Vulnerabilities::FindingRemediation.arel_table - solution_query = where(fix_available ? 'solution IS NOT NULL' : 'solution IS NULL') - exist_query = where( - fix_available ? 'EXISTS (?)' : 'NOT EXISTS (?)', - ::Vulnerabilities::FindingRemediation.select(1).where(remediation[:vulnerability_occurrence_id].eq(arel_table[:id])) - ) - - fix_available ? solution_query.or(exist_query) : solution_query.and(exist_query) - end - - scope :scoped_project, -> { where('vulnerability_occurrences.project_id = projects.id') } - scope :eager_load_vulnerability_flags, -> { includes(:vulnerability_flags) } - scope :by_location_image, ->(images) do - where(report_type: REPORT_TYPES_WITH_LOCATION_IMAGE) - .where("vulnerability_occurrences.location -> 'image' ?| array[:images]", images: images) - end - scope :by_location_cluster, ->(cluster_ids) do - where(report_type: 'cluster_image_scanning') - .where("vulnerability_occurrences.location -> 'kubernetes_resource' -> 'cluster_id' ?| array[:cluster_ids]", cluster_ids: cluster_ids) - end - scope :by_location_cluster_agent, ->(agent_ids) do - where(report_type: 'cluster_image_scanning') - .where("vulnerability_occurrences.location -> 'kubernetes_resource' -> 'agent_id' ?| array[:agent_ids]", agent_ids: agent_ids) - end - - alias_method :declarative_policy_subject, :project - alias_attribute :finding_details, :details - - def self.counted_by_severity - group(:severity).count.transform_keys do |severity| - severities[severity] - end - end - - # sha can be sourced from a joined pipeline or set from the report - def sha - # Some analysers (like Secret Detection) that produce security findings may perform scans across Git history and - # attach specific commit information to the finding. When this is the case, we _must_ use the commit SHA specified - # in the security report to compute the blob URL, otherwise the URL will link to the incorrect revision of the file. - # - # We also need to ensure we _don't_ use the commit SHA from the report if it's the default placeholder value, - # which is defined in the `secrets` analyzer: - # https://gitlab.com/gitlab-org/security-products/analyzers/secrets/-/blob/7e1e03209495a209308f3e9e96c5a4a0d32e1d55/secret.go#L13-13 - commit_sha = location.dig("commit", "sha") - if !commit_sha || commit_sha == SECRET_DETECTION_DEFAULT_COMMIT_SHA - # Two layers of fallbacks. - commit_sha = @sha || pipeline_branch - end - - commit_sha - end - - def state - if vulnerability.nil? || vulnerability.detected? - 'detected' - elsif vulnerability.resolved? - 'resolved' - elsif vulnerability.dismissed? # fail-safe check for cases when dismissal feedback was lost or was not created - 'dismissed' - else - 'confirmed' - end - end - - def token_status - finding_token_status&.status_before_type_cast || ::Security::TokenStatus::UNKNOWN - end - - def source_code? - source_code.present? - end - - def vulnerable_code(lines: vulnerable_lines) - strong_memoize_with(:vulnerable_code, lines) do - source_code.lines[lines]&.join - end - end - - def self.related_dismissal_feedback - Feedback.where('vulnerability_occurrences.uuid = vulnerability_feedback.finding_uuid') - .for_dismissal - end - private_class_method :related_dismissal_feedback - - def self.dismissed - where('EXISTS (?)', related_dismissal_feedback.select(1)) - end - - def self.undismissed - where('NOT EXISTS (?)', related_dismissal_feedback.select(1)) - end - - def feedback(feedback_type:) - load_feedback.find { |f| f.feedback_type == feedback_type } - end - - def load_feedback - BatchLoader.for(uuid).batch do |uuids, loader| - finding_feedbacks = Vulnerabilities::Feedback.all_preloaded.where(finding_uuid: uuids.uniq) - - uuids.each do |finding_uuid| - loader.call( - finding_uuid, - finding_feedbacks.select { |f| finding_uuid == f.finding_uuid } - ) - end - end - end - - def dismissal_feedback - feedback(feedback_type: 'dismissal') - end - - def issue_feedback - related_issues = vulnerability&.related_issues - related_issues.blank? ? feedback(feedback_type: 'issue') : Vulnerabilities::Feedback.find_by(issue: related_issues.first.id) - end - - def merge_request_feedback - feedback(feedback_type: 'merge_request') - end - - def metadata - strong_memoize(:metadata) do - data = Gitlab::Json.parse(raw_metadata) - - data = {} unless data.is_a?(Hash) - - data - rescue JSON::ParserError - {} - end - end - - def description - super.presence || metadata['description'] - end - - def solution - super.presence || metadata['solution'] || remediations&.first&.dig('summary') - end - - def location - super.presence || metadata.fetch('location', {}) - end - - def file - location['file'] - end - - def image - location['image'] - end - - def links - return metadata.fetch('links', []) if finding_links.load.empty? - - finding_links.as_json(only: [:name, :url]) - end - - def remediations - return metadata['remediations'] unless super.present? - - super.as_json(only: [:summary], methods: [:diff]) - end - - def token_type - return unless metadata['identifiers'] - - metadata['identifiers'].find { |hash| hash['type'] == 'gitleaks_rule_id' }&.dig('value') - end - - def token_value - metadata['raw_source_code_extract'] - end - - def cve_enrichment - return unless cve_enrichments.load.any? - - cve_enrichments.first - end - strong_memoize_attr :cve_enrichment - - def build_evidence_request(data) - return if data.nil? - - { - headers: data.fetch('headers', []).map do |request_header| - { - name: request_header['name'], - value: request_header['value'] - } - end, - method: data['method'], - url: data['url'], - body: data['body'] - } - end - - def build_evidence_response(data) - return if data.nil? - - { - headers: data.fetch('headers', []).map do |header_data| - { - name: header_data['name'], - value: header_data['value'] - } - end, - status_code: data['status_code'], - reason_phrase: data['reason_phrase'], - body: data['body'] - } - end - - def build_evidence_supporting_messages(data) - return [] if data.nil? - - data.map do |message| - { - name: message['name'], - request: build_evidence_request(message['request']), - response: build_evidence_response(message['response']) - } - end - end - - def build_evidence_source(data) - return if data.nil? - - { - id: data['id'], - name: data['name'], - url: data['url'] - } - end - - def evidence - evidence_data = finding_evidence.present? ? finding_evidence.data : metadata['evidence'] - - return if evidence_data.nil? - - { - summary: evidence_data&.dig('summary'), - request: build_evidence_request(evidence_data&.dig('request')), - response: build_evidence_response(evidence_data&.dig('response')), - source: build_evidence_source(evidence_data&.dig('source')), - supporting_messages: build_evidence_supporting_messages(evidence_data&.dig('supporting_messages')) - } - end - - def cve_value - identifiers.find(&:cve?)&.name - end - - def cwe_value - identifiers.find(&:cwe?)&.name - end - - def other_identifier_values - identifiers.select(&:other?).map(&:name) - end - - def assets - metadata.fetch('assets', []).map do |asset_data| - { - name: asset_data['name'], - type: asset_data['type'], - url: asset_data['url'] - } - end - end - - def risk_score - finding_risk_score&.risk_score || 0.0 - end - - alias_method :==, :eql? - - def eql?(other) - return false unless other.is_a?(self.class) - - unless other.report_type == report_type && other.primary_identifier_fingerprint == primary_identifier_fingerprint - return false - end - - if project.licensed_feature_available?(:vulnerability_finding_signatures) - matches_signatures(other.signatures, other.uuid) - else - other.location_fingerprint == location_fingerprint - end - end - - # Array.difference (-) method uses hash and eql? methods to do comparison - def hash - # This is causing N+1 queries whenever we are calling findings, ActiveRecord uses #hash method to make sure the - # array with findings is uniq before preloading. This method is used only in Gitlab::Ci::Reports::Security::VulnerabilityReportsComparer - # where we are normalizing security report findings into instances of Vulnerabilities::Finding, this is why we are using original implementation - # when Finding is persisted and identifiers are not preloaded. - return super if persisted? && !identifiers.loaded? - - report_type.hash ^ location_fingerprint.hash ^ primary_identifier_fingerprint.hash - end - - def severity_value - self.class.severities[self.severity] - end - - # We will eventually have only UUIDv5 values for the `uuid` - # attribute of the finding records. - def uuid_v5 - if Gitlab::UUID.v5?(uuid) - uuid - else - ::Security::VulnerabilityUUID.generate( - report_type: report_type, - primary_identifier_fingerprint: primary_identifier.fingerprint, - location_fingerprint: location_fingerprint, - project_id: project_id - ) - end - end - - def self.pluck_uuids - pluck(:uuid) - end - - def self.pluck_vulnerability_ids - pluck(:vulnerability_id) - end - - def pipeline_branch - last_finding_pipeline&.sha || project.default_branch - end - - def false_positive? - vulnerability_flags.any?(&:false_positive?) - end - - def first_finding_pipeline - initial_finding_pipeline - end - - def last_finding_pipeline - latest_finding_pipeline - end - - def vulnerable_lines - # -1 is a magic number here meaning an explicit value - # for start_line or end_line was not provided. If neither - # were provided we return the entire file contents. - return (0..-1) if (start_line < 0) && (end_line < 0) - - range_start = [start_line, 0].max - range_end = [end_line, range_start].max - - (range_start..range_end) - end - - def start_line - location["start_line"].to_i - 1 - end - - def end_line - location["end_line"].to_i - 1 - end - - def source_code - return "" unless file.present? - - blob = project.repository.blob_at(pipeline_branch, file) - blob.present? ? blob.data : "" - end - strong_memoize_attr :source_code - - def identifier_names - identifiers.pluck(:name) - end - - def ai_explanation_available? - AI_ALLOWED_REPORT_TYPES.include?(report_type) - end - - def ai_resolution_available? - AI_ALLOWED_REPORT_TYPES.include?(report_type) - end - - def ai_resolution_enabled? - ai_resolution_available? && ai_resolution_supported_cwe? - end - - def ai_resolution_supported_cwe? - if ::Feature.enabled?(:ignore_supported_cwe_list_check, project) - true - else - HIGH_CONFIDENCE_AI_RESOLUTION_CWES.include?(cwe_value&.upcase) - end - end - - protected - - def primary_identifier_fingerprint - identifiers.first&.fingerprint - end - end + class Finding < ::SecApplicationRecord + extend EnforceVulnerabilityReadDbTriggerFf + include ShaAttribute + include ::Gitlab::Utils::StrongMemoize + include Presentable + include ::VulnerabilityFindingHelpers + include EachBatch + + # https://gitlab.com/groups/gitlab-org/-/epics/3148 + # https://gitlab.com/gitlab-org/gitlab/-/issues/214563#note_370782508 is why the table names are not renamed + self.table_name = 'vulnerability_occurrences' + + FINDINGS_PER_PAGE = 20 + MAX_NUMBER_OF_IDENTIFIERS = 20 + REPORT_TYPES_WITH_LOCATION_IMAGE = %w[container_scanning cluster_image_scanning].freeze + SECRET_DETECTION_DEFAULT_COMMIT_SHA = "0000000" + + AI_ALLOWED_REPORT_TYPES = %w[sast].freeze + + # https://gitlab.com/gitlab-org/gitlab/-/issues/472861 + HIGH_CONFIDENCE_AI_RESOLUTION_CWES = %w[ + CWE-23 + CWE-73 + CWE-78 + CWE-80 + CWE-89 + CWE-116 + CWE-118 + CWE-119 + CWE-120 + CWE-126 + CWE-190 + CWE-200 + CWE-208 + CWE-209 + CWE-272 + CWE-287 + CWE-295 + CWE-297 + CWE-305 + CWE-310 + CWE-311 + CWE-323 + CWE-327 + CWE-328 + CWE-330 + CWE-338 + CWE-345 + CWE-346 + CWE-352 + CWE-362 + CWE-369 + CWE-377 + CWE-378 + CWE-400 + CWE-489 + CWE-521 + CWE-539 + CWE-599 + CWE-611 + CWE-676 + CWE-704 + CWE-754 + CWE-770 + CWE-1004 + CWE-1275 + ].to_set.freeze + + paginates_per FINDINGS_PER_PAGE + + sha_attribute :location_fingerprint + + attr_readonly :initial_pipeline_id + + belongs_to :project, inverse_of: :vulnerability_findings + belongs_to :scanner, class_name: 'Vulnerabilities::Scanner' + belongs_to :primary_identifier, class_name: 'Vulnerabilities::Identifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id' + belongs_to :vulnerability, class_name: 'Vulnerability', inverse_of: :findings, foreign_key: 'vulnerability_id' + has_one :one_vulnerability, class_name: 'Vulnerability', inverse_of: :vulnerability_finding + has_many :state_transitions, through: :vulnerability + has_many :issue_links, through: :vulnerability + has_many :external_issue_links, through: :vulnerability + has_many :merge_request_links, through: :vulnerability + + has_many :finding_identifiers, class_name: 'Vulnerabilities::FindingIdentifier', inverse_of: :finding, foreign_key: 'occurrence_id' + has_many :identifiers, through: :finding_identifiers, class_name: 'Vulnerabilities::Identifier' + + has_many :cve_identifiers, -> { where('LOWER(external_type) = ?', 'cve') }, through: :finding_identifiers, source: :identifier, class_name: 'Vulnerabilities::Identifier' + has_many :cve_enrichments, through: :cve_identifiers, disable_joins: true + + has_one :finding_token_status, class_name: 'Vulnerabilities::FindingTokenStatus', foreign_key: 'vulnerability_occurrence_id', inverse_of: :finding + + has_many :finding_links, class_name: 'Vulnerabilities::FindingLink', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' + + has_many :finding_remediations, class_name: 'Vulnerabilities::FindingRemediation', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' + has_many :remediations, through: :finding_remediations + + has_many :triggered_workflows, class_name: '::Vulnerabilities::TriggeredWorkflow', inverse_of: :vulnerability_occurrence, foreign_key: 'vulnerability_occurrence_id' + + has_one :finding_risk_score, class_name: 'Vulnerabilities::FindingRiskScore', inverse_of: :finding + + # rubocop: disable Rails/InverseOf -- these relations are not present on Ci::Pipeline + belongs_to :initial_finding_pipeline, class_name: '::Ci::Pipeline', foreign_key: 'initial_pipeline_id' + belongs_to :latest_finding_pipeline, class_name: '::Ci::Pipeline', foreign_key: 'latest_pipeline_id' + # rubocop:enable Rails/InverseOf + + has_many :signatures, class_name: 'Vulnerabilities::FindingSignature', inverse_of: :finding + + has_many :vulnerability_flags, class_name: 'Vulnerabilities::Flag', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' + + has_many :feedbacks, class_name: 'Vulnerabilities::Feedback', inverse_of: :finding, primary_key: 'uuid', foreign_key: 'finding_uuid' + + has_one :finding_evidence, class_name: 'Vulnerabilities::Finding::Evidence', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' + + has_many :security_findings, + class_name: 'Security::Finding', + primary_key: :uuid, + foreign_key: :uuid, + inverse_of: :vulnerability_finding + + attribute :config_options, ::Gitlab::Database::Type::IndifferentJsonb.new + + attr_writer :sha + attr_accessor :scan, :found_by_pipeline + + enum :report_type, ::Enums::Vulnerability.report_types + enum :severity, ::Enums::Vulnerability.severity_levels, prefix: :severity + enum :detection_method, ::Enums::Vulnerability.detection_methods + + validates :scanner, presence: true + validates :project, presence: true + validates :uuid, presence: true + + validates :primary_identifier, presence: true + validates :location_fingerprint, presence: true + # Uniqueness validation doesn't work with binary columns, so save this useless query. It is enforce by DB constraint anyway. + # TODO: find out why it fails + # validates :location_fingerprint, presence: true, uniqueness: { scope: [:primary_identifier_id, :scanner_id, :ref, :pipeline_id, :project_id] } + validates :name, presence: true + validates :report_type, presence: true + validates :severity, presence: true + validates :detection_method, presence: true + + validates :metadata_version, presence: true + validates :raw_metadata, presence: true + validates :details, json_schema: { filename: 'vulnerability_finding_details' } + + COLUMN_LENGTH_LIMITS = { + description: 15_000, + solution: 7_000 + }.freeze + + validates :description, length: { maximum: COLUMN_LENGTH_LIMITS[:description] } + validates :solution, length: { maximum: COLUMN_LENGTH_LIMITS[:solution] } + validates :cve, length: { maximum: 48400 } + + delegate :name, :external_id, to: :scanner, prefix: true, allow_nil: true + + scope :report_type, ->(type) { where(report_type: report_types[type]) } + scope :ordered, -> { order(severity: :desc, id: :asc) } + + scope :active, -> { joins(:vulnerability).merge(Vulnerability.active) } + scope :by_vulnerability, ->(vulnerability_id) { where(vulnerability: vulnerability_id) } + scope :ids_by_vulnerability, ->(vulnerability_id) { by_vulnerability(vulnerability_id).pluck(:id) } + scope :by_report_types, ->(values) { where(report_type: values) } + scope :by_projects, ->(values) { where(project_id: values) } + scope :by_scanners, ->(values) { where(scanner_id: values) } + scope :by_severities, ->(values) { where(severity: values) } + scope :by_location_fingerprints, ->(values) { where(location_fingerprint: values) } + scope :by_uuid, ->(uuids) { where(uuid: uuids) } + scope :excluding_uuids, ->(uuids) { where.not(uuid: uuids) } + scope :eager_load_comparison_entities, -> { includes(:scanner, :primary_identifier) } + scope :by_primary_identifiers, ->(identifier_ids) { where(primary_identifier: identifier_ids) } + scope :by_latest_pipeline, ->(pipeline_id) { where(latest_pipeline_id: pipeline_id) } + scope :with_project, -> { includes(:project) } + scope :with_token_status, -> { preload(:finding_token_status) } + scope :with_cve_enrichments, -> { includes(:cve_enrichments) } + scope :with_risk_score, -> { preload(:finding_risk_score) } + + scope :all_preloaded, -> do + preload(:scanner, :identifiers, :feedbacks, project: [:namespace, :project_feature]) + end + + scope :with_false_positive, ->(false_positive) do + flags = ::Vulnerabilities::Flag.arel_table + + where( + false_positive ? 'EXISTS (?)' : 'NOT EXISTS (?)', + ::Vulnerabilities::Flag.select(1).false_positive.where(flags[:vulnerability_occurrence_id].eq(arel_table[:id])) + ) + end + + scope :with_fix_available, ->(fix_available) do + remediation = ::Vulnerabilities::FindingRemediation.arel_table + solution_query = where(fix_available ? 'solution IS NOT NULL' : 'solution IS NULL') + exist_query = where( + fix_available ? 'EXISTS (?)' : 'NOT EXISTS (?)', + ::Vulnerabilities::FindingRemediation.select(1).where(remediation[:vulnerability_occurrence_id].eq(arel_table[:id])) + ) + + fix_available ? solution_query.or(exist_query) : solution_query.and(exist_query) + end + + scope :scoped_project, -> { where('vulnerability_occurrences.project_id = projects.id') } + scope :eager_load_vulnerability_flags, -> { includes(:vulnerability_flags) } + scope :by_location_image, ->(images) do + where(report_type: REPORT_TYPES_WITH_LOCATION_IMAGE) + .where("vulnerability_occurrences.location -> 'image' ?| array[:images]", images: images) + end + scope :by_location_cluster, ->(cluster_ids) do + where(report_type: 'cluster_image_scanning') + .where("vulnerability_occurrences.location -> 'kubernetes_resource' -> 'cluster_id' ?| array[:cluster_ids]", cluster_ids: cluster_ids) + end + scope :by_location_cluster_agent, ->(agent_ids) do + where(report_type: 'cluster_image_scanning') + .where("vulnerability_occurrences.location -> 'kubernetes_resource' -> 'agent_id' ?| array[:agent_ids]", agent_ids: agent_ids) + end + + alias_method :declarative_policy_subject, :project + alias_attribute :finding_details, :details + + def self.counted_by_severity + group(:severity).count.transform_keys do |severity| + severities[severity] + end + end + + # sha can be sourced from a joined pipeline or set from the report + def sha + # Some analysers (like Secret Detection) that produce security findings may perform scans across Git history and + # attach specific commit information to the finding. When this is the case, we _must_ use the commit SHA specified + # in the security report to compute the blob URL, otherwise the URL will link to the incorrect revision of the file. + # + # We also need to ensure we _don't_ use the commit SHA from the report if it's the default placeholder value, + # which is defined in the `secrets` analyzer: + # https://gitlab.com/gitlab-org/security-products/analyzers/secrets/-/blob/7e1e03209495a209308f3e9e96c5a4a0d32e1d55/secret.go#L13-13 + commit_sha = location.dig("commit", "sha") + if !commit_sha || commit_sha == SECRET_DETECTION_DEFAULT_COMMIT_SHA + # Two layers of fallbacks. + commit_sha = @sha || pipeline_branch + end + + commit_sha + end + + def state + if vulnerability.nil? || vulnerability.detected? + 'detected' + elsif vulnerability.resolved? + 'resolved' + elsif vulnerability.dismissed? # fail-safe check for cases when dismissal feedback was lost or was not created + 'dismissed' + else + 'confirmed' + end + end + + def token_status + finding_token_status&.status_before_type_cast || ::Security::TokenStatus::UNKNOWN + end + + def source_code? + source_code.present? + end + + def vulnerable_code(lines: vulnerable_lines) + strong_memoize_with(:vulnerable_code, lines) do + source_code.lines[lines]&.join + end + end + + def self.related_dismissal_feedback + Feedback.where('vulnerability_occurrences.uuid = vulnerability_feedback.finding_uuid') + .for_dismissal + end + private_class_method :related_dismissal_feedback + + def self.dismissed + where('EXISTS (?)', related_dismissal_feedback.select(1)) + end + + def self.undismissed + where('NOT EXISTS (?)', related_dismissal_feedback.select(1)) + end + + def feedback(feedback_type:) + load_feedback.find { |f| f.feedback_type == feedback_type } + end + + def load_feedback + BatchLoader.for(uuid).batch do |uuids, loader| + finding_feedbacks = Vulnerabilities::Feedback.all_preloaded.where(finding_uuid: uuids.uniq) + + uuids.each do |finding_uuid| + loader.call( + finding_uuid, + finding_feedbacks.select { |f| finding_uuid == f.finding_uuid } + ) + end + end + end + + def dismissal_feedback + feedback(feedback_type: 'dismissal') + end + + def issue_feedback + related_issues = vulnerability&.related_issues + related_issues.blank? ? feedback(feedback_type: 'issue') : Vulnerabilities::Feedback.find_by(issue: related_issues.first.id) + end + + def merge_request_feedback + feedback(feedback_type: 'merge_request') + end + + def metadata + strong_memoize(:metadata) do + data = Gitlab::Json.parse(raw_metadata) + + data = {} unless data.is_a?(Hash) + + data + rescue JSON::ParserError + {} + end + end + + def description + super.presence || metadata['description'] + end + + def solution + super.presence || metadata['solution'] || remediations&.first&.dig('summary') + end + + def location + super.presence || metadata.fetch('location', {}) + end + + def file + location['file'] + end + + def image + location['image'] + end + + def links + return metadata.fetch('links', []) if finding_links.load.empty? + + finding_links.as_json(only: [:name, :url]) + end + + def remediations + return metadata['remediations'] unless super.present? + + super.as_json(only: [:summary], methods: [:diff]) + end + + def token_type + return unless metadata['identifiers'] + + metadata['identifiers'].find { |hash| hash['type'] == 'gitleaks_rule_id' }&.dig('value') + end + + def token_value + metadata['raw_source_code_extract'] + end + + def cve_enrichment + return unless cve_enrichments.load.any? + + cve_enrichments.first + end + strong_memoize_attr :cve_enrichment + + def build_evidence_request(data) + return if data.nil? + + { + headers: data.fetch('headers', []).map do |request_header| + { + name: request_header['name'], + value: request_header['value'] + } + end, + method: data['method'], + url: data['url'], + body: data['body'] + } + end + + def build_evidence_response(data) + return if data.nil? + + { + headers: data.fetch('headers', []).map do |header_data| + { + name: header_data['name'], + value: header_data['value'] + } + end, + status_code: data['status_code'], + reason_phrase: data['reason_phrase'], + body: data['body'] + } + end + + def build_evidence_supporting_messages(data) + return [] if data.nil? + + data.map do |message| + { + name: message['name'], + request: build_evidence_request(message['request']), + response: build_evidence_response(message['response']) + } + end + end + + def build_evidence_source(data) + return if data.nil? + + { + id: data['id'], + name: data['name'], + url: data['url'] + } + end + + def evidence + evidence_data = finding_evidence.present? ? finding_evidence.data : metadata['evidence'] + + return if evidence_data.nil? + + { + summary: evidence_data&.dig('summary'), + request: build_evidence_request(evidence_data&.dig('request')), + response: build_evidence_response(evidence_data&.dig('response')), + source: build_evidence_source(evidence_data&.dig('source')), + supporting_messages: build_evidence_supporting_messages(evidence_data&.dig('supporting_messages')) + } + end + + def cve_value + identifiers.find(&:cve?)&.name + end + + def cwe_value + identifiers.find(&:cwe?)&.name + end + + def other_identifier_values + identifiers.select(&:other?).map(&:name) + end + + def assets + metadata.fetch('assets', []).map do |asset_data| + { + name: asset_data['name'], + type: asset_data['type'], + url: asset_data['url'] + } + end + end + + def risk_score + finding_risk_score&.risk_score || 0.0 + end + + alias_method :==, :eql? + + def eql?(other) + return false unless other.is_a?(self.class) + + unless other.report_type == report_type && other.primary_identifier_fingerprint == primary_identifier_fingerprint + return false + end + + if project.licensed_feature_available?(:vulnerability_finding_signatures) + matches_signatures(other.signatures, other.uuid) + else + other.location_fingerprint == location_fingerprint + end + end + + # Array.difference (-) method uses hash and eql? methods to do comparison + def hash + # This is causing N+1 queries whenever we are calling findings, ActiveRecord uses #hash method to make sure the + # array with findings is uniq before preloading. This method is used only in Gitlab::Ci::Reports::Security::VulnerabilityReportsComparer + # where we are normalizing security report findings into instances of Vulnerabilities::Finding, this is why we are using original implementation + # when Finding is persisted and identifiers are not preloaded. + return super if persisted? && !identifiers.loaded? + + report_type.hash ^ location_fingerprint.hash ^ primary_identifier_fingerprint.hash + end + + def severity_value + self.class.severities[self.severity] + end + + # We will eventually have only UUIDv5 values for the `uuid` + # attribute of the finding records. + def uuid_v5 + if Gitlab::UUID.v5?(uuid) + uuid + else + ::Security::VulnerabilityUUID.generate( + report_type: report_type, + primary_identifier_fingerprint: primary_identifier.fingerprint, + location_fingerprint: location_fingerprint, + project_id: project_id + ) + end + end + + def self.pluck_uuids + pluck(:uuid) + end + + def self.pluck_vulnerability_ids + pluck(:vulnerability_id) + end + + def pipeline_branch + last_finding_pipeline&.sha || project.default_branch + end + + def false_positive? + vulnerability_flags.any?(&:false_positive?) + end + + def first_finding_pipeline + initial_finding_pipeline + end + + def last_finding_pipeline + latest_finding_pipeline + end + + def vulnerable_lines + # -1 is a magic number here meaning an explicit value + # for start_line or end_line was not provided. If neither + # were provided we return the entire file contents. + return (0..-1) if (start_line < 0) && (end_line < 0) + + range_start = [start_line, 0].max + range_end = [end_line, range_start].max + + (range_start..range_end) + end + + def start_line + location["start_line"].to_i - 1 + end + + def end_line + location["end_line"].to_i - 1 + end + + def source_code + return "" unless file.present? + + blob = project.repository.blob_at(pipeline_branch, file) + blob.present? ? blob.data : "" + end + strong_memoize_attr :source_code + + def identifier_names + identifiers.pluck(:name) + end + + def ai_explanation_available? + AI_ALLOWED_REPORT_TYPES.include?(report_type) + end + + def ai_resolution_available? + AI_ALLOWED_REPORT_TYPES.include?(report_type) + end + + def ai_resolution_enabled? + ai_resolution_available? && ai_resolution_supported_cwe? + end + + def ai_resolution_supported_cwe? + if ::Feature.enabled?(:ignore_supported_cwe_list_check, project) + true + else + HIGH_CONFIDENCE_AI_RESOLUTION_CWES.include?(cwe_value&.upcase) + end + end + + protected + + def primary_identifier_fingerprint + identifiers.first&.fingerprint + end + end end Vulnerabilities::Finding.prepend_mod -- GitLab From 981b07b13be5301873f1e249437f0985c73fa956 Mon Sep 17 00:00:00 2001 From: Gregory Havenga <11164960-ghavenga@users.noreply.gitlab.com> Date: Tue, 21 Oct 2025 15:07:59 +0200 Subject: [PATCH 4/5] Add EnforceVulnerabilityReadDbTriggerFf to MergeRequestLink model --- .../vulnerabilities/merge_request_link.rb | 59 ++++++++++--------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/ee/app/models/vulnerabilities/merge_request_link.rb b/ee/app/models/vulnerabilities/merge_request_link.rb index 0b8692f44713ff..743e4dbca33b25 100644 --- a/ee/app/models/vulnerabilities/merge_request_link.rb +++ b/ee/app/models/vulnerabilities/merge_request_link.rb @@ -1,41 +1,42 @@ # frozen_string_literal: true module Vulnerabilities - class MergeRequestLink < ::SecApplicationRecord - include EachBatch + class MergeRequestLink < ::SecApplicationRecord + extend EnforceVulnerabilityReadDbTriggerFf + include EachBatch - MAX_MERGE_REQUEST_LINKS_PER_VULNERABILITY = 100 + MAX_MERGE_REQUEST_LINKS_PER_VULNERABILITY = 100 - self.table_name = 'vulnerability_merge_request_links' + self.table_name = 'vulnerability_merge_request_links' - belongs_to :vulnerability - belongs_to :merge_request - belongs_to :vulnerability_occurrence, optional: true, class_name: 'Vulnerabilities::Finding' + belongs_to :vulnerability + belongs_to :merge_request + belongs_to :vulnerability_occurrence, optional: true, class_name: 'Vulnerabilities::Finding' - has_one :author, through: :merge_request, class_name: 'User' + has_one :author, through: :merge_request, class_name: 'User' - validates :vulnerability, :merge_request, presence: true - validates :merge_request_id, - uniqueness: { scope: :vulnerability_id, message: N_('is already linked to this vulnerability') } + validates :vulnerability, :merge_request, presence: true + validates :merge_request_id, + uniqueness: { scope: :vulnerability_id, message: N_('is already linked to this vulnerability') } - scope :by_finding_uuids, ->(uuids) do - joins(vulnerability: [:findings]).where(vulnerability: { - vulnerability_occurrences: { uuid: uuids } - }) - end - scope :with_vulnerability_findings, -> { includes(vulnerability: [:findings]) } - scope :with_merge_request, -> { preload(:merge_request) } - scope :by_vulnerability, ->(values) { where(vulnerability_id: values) } + scope :by_finding_uuids, ->(uuids) do + joins(vulnerability: [:findings]).where(vulnerability: { + vulnerability_occurrences: { uuid: uuids } + }) + end + scope :with_vulnerability_findings, -> { includes(vulnerability: [:findings]) } + scope :with_merge_request, -> { preload(:merge_request) } + scope :by_vulnerability, ->(values) { where(vulnerability_id: values) } - def self.count_for_vulnerability(vulnerability) - where(vulnerability: vulnerability).count - end + def self.count_for_vulnerability(vulnerability) + where(vulnerability: vulnerability).count + end - def self.limit_exceeded_for_vulnerability?(vulnerability) - count_for_vulnerability(vulnerability) >= MAX_MERGE_REQUEST_LINKS_PER_VULNERABILITY - end + def self.limit_exceeded_for_vulnerability?(vulnerability) + count_for_vulnerability(vulnerability) >= MAX_MERGE_REQUEST_LINKS_PER_VULNERABILITY + end - def self.find_by_vulnerability_and_merge_request(vulnerability, merge_request) - find_by(vulnerability: vulnerability, merge_request: merge_request) - end - end + def self.find_by_vulnerability_and_merge_request(vulnerability, merge_request) + find_by(vulnerability: vulnerability, merge_request: merge_request) + end + end end -- GitLab From cf1501a00295dc43f27a68ace819b87e2a40e873 Mon Sep 17 00:00:00 2001 From: Gregory Havenga <11164960-ghavenga@users.noreply.gitlab.com> Date: Tue, 21 Oct 2025 15:10:37 +0200 Subject: [PATCH 5/5] Add EnforceVulnerabilityReadDbTriggerFf to Read model and remove uuid uniqueness validation --- ee/app/models/vulnerabilities/read.rb | 633 +++++++++++++------------- 1 file changed, 317 insertions(+), 316 deletions(-) diff --git a/ee/app/models/vulnerabilities/read.rb b/ee/app/models/vulnerabilities/read.rb index 2cac5e118730dd..82b693ed93a343 100644 --- a/ee/app/models/vulnerabilities/read.rb +++ b/ee/app/models/vulnerabilities/read.rb @@ -1,320 +1,321 @@ # frozen_string_literal: true module Vulnerabilities - class Read < ::SecApplicationRecord - extend ::Gitlab::Utils::Override - include ::Namespaces::Traversal::Traversable - include VulnerabilityScopes - include EachBatch - include UnnestedInFilters::Dsl - include FromUnion - include SafelyChangeColumnDefault - include ::Gitlab::SQL::Pattern - include ::Elastic::ApplicationVersionedSearch - - ignore_column :namespace_id, remove_with: '17.7', remove_after: '2024-11-21' - - declarative_enum DismissalReasonEnum - - # Included after scopes and relationships to avoid the warning - include BulkInsertSafe - - SEVERITY_COUNT_LIMIT = 1001 - OWASP_TOP_10_DEFAULT = -1 - - ELASTICSEARCH_TRACKED_FIELDS = ::Search::Elastic::References::Vulnerability::DIRECT_FIELDS + - ::Search::Elastic::References::Vulnerability::DIRECT_TYPECAST_FIELDS + %w[traversal_ids] - - self.table_name = "vulnerability_reads" - self.primary_key = :vulnerability_id - - columns_changing_default :owasp_top_10 - - delegate :group_name, :title, :created_at, :project_name, :finding_scanner_name, :finding_description, :cve_value, :cwe_value, :location, :notes_summary, :full_path, to: :vulnerability, allow_nil: true - delegate :other_identifier_values, :cvss_vectors_with_vendor, to: :vulnerability, allow_nil: true - delegate :dismissed?, to: :vulnerability - - belongs_to :vulnerability, inverse_of: :vulnerability_read - belongs_to :project - belongs_to :scanner, class_name: 'Vulnerabilities::Scanner' - belongs_to :vulnerability_occurrence, optional: true, class_name: 'Vulnerabilities::Finding' - - validates :vulnerability_id, uniqueness: true, presence: true - validates :project_id, presence: true - validates :scanner_id, presence: true - validates :report_type, presence: true - validates :severity, presence: true - validates :state, presence: true - validates :uuid, presence: true - - validates :location_image, length: { maximum: 2048 } - validates :has_issues, inclusion: { in: [true, false], message: N_('must be a boolean value') } - validates :has_merge_request, inclusion: { in: [true, false], message: N_('must be a boolean value') } - validates :resolved_on_default_branch, inclusion: { in: [true, false], message: N_('must be a boolean value') } - validates :has_remediations, inclusion: { in: [true, false], message: N_('must be a boolean value') } - - enum :state, ::Enums::Vulnerability.vulnerability_states - enum :report_type, ::Enums::Vulnerability.report_types - enum :severity, ::Enums::Vulnerability.severity_levels, prefix: :severity - enum :owasp_top_10, ::Enums::Vulnerability.owasp_top_10.merge('undefined' => OWASP_TOP_10_DEFAULT) - - after_initialize :set_default_values, if: :new_record? - - scope :by_uuid, ->(uuids) { where(uuid: uuids) } - scope :by_vulnerabilities, ->(vulnerabilities) { where(vulnerability: vulnerabilities) } - - class << self - alias_method :by_vulnerability, :by_vulnerabilities - end - - scope :order_severity_asc, -> { reorder(severity: :asc, vulnerability_id: :desc) } - scope :order_severity_desc, -> { reorder(severity: :desc, vulnerability_id: :desc) } - scope :order_detected_at_asc, -> { reorder(vulnerability_id: :asc) } - scope :order_detected_at_desc, -> { reorder(vulnerability_id: :desc) } - - scope :order_severity_asc_traversal_ids_asc, -> { reorder(severity: :asc, traversal_ids: :asc, vulnerability_id: :asc) } - scope :order_severity_desc_traversal_ids_desc, -> { reorder(severity: :desc, traversal_ids: :desc, vulnerability_id: :desc) } - - scope :in_parent_group_after_and_including, ->(vulnerability_read) do - where(arel_grouping_by_traversal_ids_and_vulnerability_id.gteq(vulnerability_read.arel_grouping_by_traversal_ids_and_id)) - end - scope :in_parent_group_before_and_including, ->(vulnerability_read) do - where(arel_grouping_by_traversal_ids_and_vulnerability_id.lteq(vulnerability_read.arel_grouping_by_traversal_ids_and_id)) - end - scope :by_group, ->(group) { within(group.traversal_ids) } - scope :unarchived, -> { where(archived: false) } - scope :order_traversal_ids_asc, -> do - reorder(Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'traversal_ids', - order_expression: arel_table[:traversal_ids].asc, - nullable: :not_nullable - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'vulnerability_id', - order_expression: arel_table[:vulnerability_id].asc - ) - ])) - end - scope :by_projects, ->(values) { where(project_id: values) } - scope :by_scanner, ->(scanner) { where(scanner: scanner) } - scope :by_scanner_ids, ->(scanner_ids) { where(scanner_id: scanner_ids) } - scope :grouped_by_severity, -> { reorder(severity: :desc).group(:severity) } - scope :with_report_types, ->(report_types) { where(report_type: report_types) } - scope :with_severities, ->(severities) { where(severity: severities) } - scope :with_states, ->(states) { where(state: states) } - scope :with_owasp_top_10, ->(owasp_top_10) { where(owasp_top_10: owasp_top_10) } - scope :with_identifier_name, ->(name) do - return none if name.nil? - - where("EXISTS ( - SELECT 1 - FROM unnest(vulnerability_reads.identifier_names) AS idt_names - WHERE idt_names ILIKE ? - )", sanitize_sql_like(name)) - end - scope :with_container_image, ->(images) { where(location_image: images) } - scope :with_container_image_starting_with, ->(image) { where(arel_table[:location_image].matches("#{sanitize_sql_like(image)}%")) } - scope :with_cluster_agent_ids, ->(agent_ids) { where(cluster_agent_id: agent_ids) } - scope :with_resolution, ->(has_resolution = true) { where(resolved_on_default_branch: has_resolution) } - scope :with_ai_resolution, ->(resolution = true) { where(has_vulnerability_resolution: resolution) } - scope :with_issues, ->(has_issues = true) { where(has_issues: has_issues) } - scope :with_merge_request, ->(has_merge_request = true) { where(has_merge_request: has_merge_request) } - scope :with_remediations, ->(has_remediations = true) { where(has_remediations: has_remediations) } - scope :with_scanner_external_ids, - ->(scanner_external_ids) { - joins(:scanner).merge(::Vulnerabilities::Scanner.with_external_id(scanner_external_ids)) - } - scope :with_findings_scanner_and_identifiers, - -> { - includes(vulnerability: { findings: [:scanner, :identifiers, { finding_identifiers: :identifier }] }) - } - scope :preload_indexing_data, -> { - preload( - :scanner, - { vulnerability: [ - { findings: [ - { identifiers: [] }, - { finding_identifiers: :identifier } - ] } - ] }, - { project: { namespace: :route } } - ) - } - scope :resolved_on_default_branch, -> { where('resolved_on_default_branch IS TRUE') } - scope :with_dismissal_reason, ->(dismissal_reason) { where(dismissal_reason: dismissal_reason) } - scope :with_export_entities, -> do - preload( - vulnerability: [ - :group, - { project: [:route], - notes: [:updated_by, :author], - findings: [:scanner, :identifiers] } - ] - ) - end - - scope :as_vulnerabilities, -> do - preload(vulnerability: { project: [:route] }).current_scope.tap do |relation| - relation.define_singleton_method(:records) do - super().map(&:vulnerability) - end - end - end - - scope :by_group_using_nested_loop, ->(group) do - where(traversal_ids: all_vulnerable_traversal_ids_for(group)) - end - - scope :with_findings_scanner_identifiers_and_notes, -> { with_findings_scanner_and_identifiers.includes(vulnerability: :notes) } - scope :with_limit, ->(maximum) { limit(maximum) } - scope :order_id_desc, -> { reorder(arel_table[:vulnerability_id].desc) } - - scope :autocomplete_search, ->(query) do - return self if query.blank? - - id_as_text = Arel::Nodes::NamedFunction.new('CAST', [arel_table[:vulnerability_id].as('TEXT')]) - - joins(:vulnerability) - .select('vulnerability_reads.*, vulnerabilities.title') - .fuzzy_search(query, [Vulnerability.arel_table[:title]]) - .or(where(id_as_text.matches("%#{sanitize_sql_like(query.squish)}%"))) - end - - scope :by_ids_desc, ->(ids) do - by_vulnerability(ids).order_id_desc - end - - def self.es_type - ::Search::Elastic::References::Vulnerability::DOC_TYPE - end - - def self.arel_grouping_by_traversal_ids_and_vulnerability_id - arel_table.grouping([arel_table['traversal_ids'], arel_table['vulnerability_id']]) - end - - def self.all_vulnerable_traversal_ids_for(group) - by_group(group).unarchived.loose_index_scan(column: :traversal_ids) - end - - def self.count_by_severity - grouped_by_severity.count - end - - def self.capped_count_by_severity - # Return early when called by `Vulnerabilities::Read.none`. - return {} if current_scope&.null_relation? - - # Handles case when called directly `Vulnerabilities::Read.capped_count_by_severity`. - if current_scope.nil? - severities_to_iterate = severities.keys - local_scope = self - else - severities_to_iterate = Array(current_scope.where_values_hash['severity'].presence || severities.keys) - local_scope = current_scope.unscope(where: :severity) - end - - array_severities_limit = severities_to_iterate.map do |severity| - local_scope.with_severities(severity).select(:id, :severity).limit(SEVERITY_COUNT_LIMIT) - end - - unscoped.from_union(array_severities_limit).count_by_severity - end - - def self.order_by(method) - case method.to_s - when 'severity_desc' then order_severity_desc - when 'severity_asc' then order_severity_asc - when 'detected_desc' then order_detected_at_desc - when 'detected_asc' then order_detected_at_asc - else - order_severity_desc - end - end - - def self.order_by_params_and_traversal_ids(method) - case method.to_s - when 'severity_desc' then order_severity_desc_traversal_ids_desc - when 'severity_asc' then order_severity_asc_traversal_ids_asc - when 'detected_desc' then order_detected_at_desc - when 'detected_asc' then order_detected_at_asc - else - order_severity_desc_traversal_ids_desc - end - end - - def self.container_images - # This method should be used only with pagination. When used without a specific limit, it might try to process an - # unreasonable amount of records leading to a statement timeout. - - # We are enforcing keyset order here to make sure `primary_key` will not be automatically applied when returning - # `ordered_items` from Gitlab::Graphql::Pagination::Keyset::Connection in GraphQL API. `distinct` option must be - # set to true in `Gitlab::Pagination::Keyset::ColumnOrderDefinition` to return the collection in proper order. - - keyset_order = Gitlab::Pagination::Keyset::Order.build( - [ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: :location_image, - column_expression: arel_table[:location_image], - order_expression: arel_table[:location_image].asc - ) - ]) - - where(report_type: [:container_scanning, :cluster_image_scanning]) - .where.not(location_image: nil) - .reorder(keyset_order) - .select(:location_image) - .distinct - end - - def self.fetch_uuids - pluck(:uuid) - end - - def self.generate_es_parent(project) - "group_#{project.namespace.root_ancestor.id}" - end - - def arel_grouping_by_traversal_ids_and_id - self.class.arel_table.grouping([database_serialized_traversal_ids, id]) - end - - def es_parent - self.class.generate_es_parent(project) - end - - def elastic_reference - ::Search::Elastic::References::Vulnerability.serialize(self) - end - - # NOTE: - # 1. For On-premise, post MVC. We may have to honour the setting of skipping indexing for selected projects. Tracked in https://gitlab.com/gitlab-org/gitlab/-/issues/525484 - override :use_elasticsearch? - def use_elasticsearch? - ::Search::Elastic::VulnerabilityIndexingHelper.vulnerability_indexing_allowed? - end - - override :maintain_elasticsearch_update - def maintain_elasticsearch_update(updated_attributes: previous_changes.keys) - super if update_elasticsearch? - end - - private - - def update_elasticsearch? - changed_fields = previous_changes.keys - changed_fields && (changed_fields & ELASTICSEARCH_TRACKED_FIELDS).any? - end - - def database_serialized_traversal_ids - self.class.attribute_types['traversal_ids'] - .serialize(traversal_ids) - .then { |serialized_array| self.class.connection.quote(serialized_array) } - .then { |quoted_array| Arel::Nodes::SqlLiteral.new(quoted_array) } - end - - def set_default_values - self.owasp_top_10 ||= 'undefined' - end - end + class Read < ::SecApplicationRecord + extend EnforceVulnerabilityReadDbTriggerFf + extend ::Gitlab::Utils::Override + include ::Namespaces::Traversal::Traversable + include VulnerabilityScopes + include EachBatch + include UnnestedInFilters::Dsl + include FromUnion + include SafelyChangeColumnDefault + include ::Gitlab::SQL::Pattern + include ::Elastic::ApplicationVersionedSearch + + ignore_column :namespace_id, remove_with: '17.7', remove_after: '2024-11-21' + + declarative_enum DismissalReasonEnum + + # Included after scopes and relationships to avoid the warning + include BulkInsertSafe + + SEVERITY_COUNT_LIMIT = 1001 + OWASP_TOP_10_DEFAULT = -1 + + ELASTICSEARCH_TRACKED_FIELDS = ::Search::Elastic::References::Vulnerability::DIRECT_FIELDS + + ::Search::Elastic::References::Vulnerability::DIRECT_TYPECAST_FIELDS + %w[traversal_ids] + + self.table_name = "vulnerability_reads" + self.primary_key = :vulnerability_id + + columns_changing_default :owasp_top_10 + + delegate :group_name, :title, :created_at, :project_name, :finding_scanner_name, :finding_description, :cve_value, :cwe_value, :location, :notes_summary, :full_path, to: :vulnerability, allow_nil: true + delegate :other_identifier_values, :cvss_vectors_with_vendor, to: :vulnerability, allow_nil: true + delegate :dismissed?, to: :vulnerability + + belongs_to :vulnerability, inverse_of: :vulnerability_read + belongs_to :project + belongs_to :scanner, class_name: 'Vulnerabilities::Scanner' + belongs_to :vulnerability_occurrence, optional: true, class_name: 'Vulnerabilities::Finding' + + validates :vulnerability_id, uniqueness: true, presence: true + validates :project_id, presence: true + validates :scanner_id, presence: true + validates :report_type, presence: true + validates :severity, presence: true + validates :state, presence: true + validates :uuid, presence: true + + validates :location_image, length: { maximum: 2048 } + validates :has_issues, inclusion: { in: [true, false], message: N_('must be a boolean value') } + validates :has_merge_request, inclusion: { in: [true, false], message: N_('must be a boolean value') } + validates :resolved_on_default_branch, inclusion: { in: [true, false], message: N_('must be a boolean value') } + validates :has_remediations, inclusion: { in: [true, false], message: N_('must be a boolean value') } + + enum :state, ::Enums::Vulnerability.vulnerability_states + enum :report_type, ::Enums::Vulnerability.report_types + enum :severity, ::Enums::Vulnerability.severity_levels, prefix: :severity + enum :owasp_top_10, ::Enums::Vulnerability.owasp_top_10.merge('undefined' => OWASP_TOP_10_DEFAULT) + + after_initialize :set_default_values, if: :new_record? + + scope :by_uuid, ->(uuids) { where(uuid: uuids) } + scope :by_vulnerabilities, ->(vulnerabilities) { where(vulnerability: vulnerabilities) } + + class << self + alias_method :by_vulnerability, :by_vulnerabilities + end + + scope :order_severity_asc, -> { reorder(severity: :asc, vulnerability_id: :desc) } + scope :order_severity_desc, -> { reorder(severity: :desc, vulnerability_id: :desc) } + scope :order_detected_at_asc, -> { reorder(vulnerability_id: :asc) } + scope :order_detected_at_desc, -> { reorder(vulnerability_id: :desc) } + + scope :order_severity_asc_traversal_ids_asc, -> { reorder(severity: :asc, traversal_ids: :asc, vulnerability_id: :asc) } + scope :order_severity_desc_traversal_ids_desc, -> { reorder(severity: :desc, traversal_ids: :desc, vulnerability_id: :desc) } + + scope :in_parent_group_after_and_including, ->(vulnerability_read) do + where(arel_grouping_by_traversal_ids_and_vulnerability_id.gteq(vulnerability_read.arel_grouping_by_traversal_ids_and_id)) + end + scope :in_parent_group_before_and_including, ->(vulnerability_read) do + where(arel_grouping_by_traversal_ids_and_vulnerability_id.lteq(vulnerability_read.arel_grouping_by_traversal_ids_and_id)) + end + scope :by_group, ->(group) { within(group.traversal_ids) } + scope :unarchived, -> { where(archived: false) } + scope :order_traversal_ids_asc, -> do + reorder(Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'traversal_ids', + order_expression: arel_table[:traversal_ids].asc, + nullable: :not_nullable + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'vulnerability_id', + order_expression: arel_table[:vulnerability_id].asc + ) + ])) + end + scope :by_projects, ->(values) { where(project_id: values) } + scope :by_scanner, ->(scanner) { where(scanner: scanner) } + scope :by_scanner_ids, ->(scanner_ids) { where(scanner_id: scanner_ids) } + scope :grouped_by_severity, -> { reorder(severity: :desc).group(:severity) } + scope :with_report_types, ->(report_types) { where(report_type: report_types) } + scope :with_severities, ->(severities) { where(severity: severities) } + scope :with_states, ->(states) { where(state: states) } + scope :with_owasp_top_10, ->(owasp_top_10) { where(owasp_top_10: owasp_top_10) } + scope :with_identifier_name, ->(name) do + return none if name.nil? + + where("EXISTS ( + SELECT 1 + FROM unnest(vulnerability_reads.identifier_names) AS idt_names + WHERE idt_names ILIKE ? + )", sanitize_sql_like(name)) + end + scope :with_container_image, ->(images) { where(location_image: images) } + scope :with_container_image_starting_with, ->(image) { where(arel_table[:location_image].matches("#{sanitize_sql_like(image)}%")) } + scope :with_cluster_agent_ids, ->(agent_ids) { where(cluster_agent_id: agent_ids) } + scope :with_resolution, ->(has_resolution = true) { where(resolved_on_default_branch: has_resolution) } + scope :with_ai_resolution, ->(resolution = true) { where(has_vulnerability_resolution: resolution) } + scope :with_issues, ->(has_issues = true) { where(has_issues: has_issues) } + scope :with_merge_request, ->(has_merge_request = true) { where(has_merge_request: has_merge_request) } + scope :with_remediations, ->(has_remediations = true) { where(has_remediations: has_remediations) } + scope :with_scanner_external_ids, + ->(scanner_external_ids) { + joins(:scanner).merge(::Vulnerabilities::Scanner.with_external_id(scanner_external_ids)) + } + scope :with_findings_scanner_and_identifiers, + -> { + includes(vulnerability: { findings: [:scanner, :identifiers, { finding_identifiers: :identifier }] }) + } + scope :preload_indexing_data, -> { + preload( + :scanner, + { vulnerability: [ + { findings: [ + { identifiers: [] }, + { finding_identifiers: :identifier } + ] } + ] }, + { project: { namespace: :route } } + ) + } + scope :resolved_on_default_branch, -> { where('resolved_on_default_branch IS TRUE') } + scope :with_dismissal_reason, ->(dismissal_reason) { where(dismissal_reason: dismissal_reason) } + scope :with_export_entities, -> do + preload( + vulnerability: [ + :group, + { project: [:route], + notes: [:updated_by, :author], + findings: [:scanner, :identifiers] } + ] + ) + end + + scope :as_vulnerabilities, -> do + preload(vulnerability: { project: [:route] }).current_scope.tap do |relation| + relation.define_singleton_method(:records) do + super().map(&:vulnerability) + end + end + end + + scope :by_group_using_nested_loop, ->(group) do + where(traversal_ids: all_vulnerable_traversal_ids_for(group)) + end + + scope :with_findings_scanner_identifiers_and_notes, -> { with_findings_scanner_and_identifiers.includes(vulnerability: :notes) } + scope :with_limit, ->(maximum) { limit(maximum) } + scope :order_id_desc, -> { reorder(arel_table[:vulnerability_id].desc) } + + scope :autocomplete_search, ->(query) do + return self if query.blank? + + id_as_text = Arel::Nodes::NamedFunction.new('CAST', [arel_table[:vulnerability_id].as('TEXT')]) + + joins(:vulnerability) + .select('vulnerability_reads.*, vulnerabilities.title') + .fuzzy_search(query, [Vulnerability.arel_table[:title]]) + .or(where(id_as_text.matches("%#{sanitize_sql_like(query.squish)}%"))) + end + + scope :by_ids_desc, ->(ids) do + by_vulnerability(ids).order_id_desc + end + + def self.es_type + ::Search::Elastic::References::Vulnerability::DOC_TYPE + end + + def self.arel_grouping_by_traversal_ids_and_vulnerability_id + arel_table.grouping([arel_table['traversal_ids'], arel_table['vulnerability_id']]) + end + + def self.all_vulnerable_traversal_ids_for(group) + by_group(group).unarchived.loose_index_scan(column: :traversal_ids) + end + + def self.count_by_severity + grouped_by_severity.count + end + + def self.capped_count_by_severity + # Return early when called by `Vulnerabilities::Read.none`. + return {} if current_scope&.null_relation? + + # Handles case when called directly `Vulnerabilities::Read.capped_count_by_severity`. + if current_scope.nil? + severities_to_iterate = severities.keys + local_scope = self + else + severities_to_iterate = Array(current_scope.where_values_hash['severity'].presence || severities.keys) + local_scope = current_scope.unscope(where: :severity) + end + + array_severities_limit = severities_to_iterate.map do |severity| + local_scope.with_severities(severity).select(:id, :severity).limit(SEVERITY_COUNT_LIMIT) + end + + unscoped.from_union(array_severities_limit).count_by_severity + end + + def self.order_by(method) + case method.to_s + when 'severity_desc' then order_severity_desc + when 'severity_asc' then order_severity_asc + when 'detected_desc' then order_detected_at_desc + when 'detected_asc' then order_detected_at_asc + else + order_severity_desc + end + end + + def self.order_by_params_and_traversal_ids(method) + case method.to_s + when 'severity_desc' then order_severity_desc_traversal_ids_desc + when 'severity_asc' then order_severity_asc_traversal_ids_asc + when 'detected_desc' then order_detected_at_desc + when 'detected_asc' then order_detected_at_asc + else + order_severity_desc_traversal_ids_desc + end + end + + def self.container_images + # This method should be used only with pagination. When used without a specific limit, it might try to process an + # unreasonable amount of records leading to a statement timeout. + + # We are enforcing keyset order here to make sure `primary_key` will not be automatically applied when returning + # `ordered_items` from Gitlab::Graphql::Pagination::Keyset::Connection in GraphQL API. `distinct` option must be + # set to true in `Gitlab::Pagination::Keyset::ColumnOrderDefinition` to return the collection in proper order. + + keyset_order = Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :location_image, + column_expression: arel_table[:location_image], + order_expression: arel_table[:location_image].asc + ) + ]) + + where(report_type: [:container_scanning, :cluster_image_scanning]) + .where.not(location_image: nil) + .reorder(keyset_order) + .select(:location_image) + .distinct + end + + def self.fetch_uuids + pluck(:uuid) + end + + def self.generate_es_parent(project) + "group_#{project.namespace.root_ancestor.id}" + end + + def arel_grouping_by_traversal_ids_and_id + self.class.arel_table.grouping([database_serialized_traversal_ids, id]) + end + + def es_parent + self.class.generate_es_parent(project) + end + + def elastic_reference + ::Search::Elastic::References::Vulnerability.serialize(self) + end + + # NOTE: + # 1. For On-premise, post MVC. We may have to honour the setting of skipping indexing for selected projects. Tracked in https://gitlab.com/gitlab-org/gitlab/-/issues/525484 + override :use_elasticsearch? + def use_elasticsearch? + ::Search::Elastic::VulnerabilityIndexingHelper.vulnerability_indexing_allowed? + end + + override :maintain_elasticsearch_update + def maintain_elasticsearch_update(updated_attributes: previous_changes.keys) + super if update_elasticsearch? + end + + private + + def update_elasticsearch? + changed_fields = previous_changes.keys + changed_fields && (changed_fields & ELASTICSEARCH_TRACKED_FIELDS).any? + end + + def database_serialized_traversal_ids + self.class.attribute_types['traversal_ids'] + .serialize(traversal_ids) + .then { |serialized_array| self.class.connection.quote(serialized_array) } + .then { |quoted_array| Arel::Nodes::SqlLiteral.new(quoted_array) } + end + + def set_default_values + self.owasp_top_10 ||= 'undefined' + end + end end -- GitLab