diff --git a/app/models/sec_application_record.rb b/app/models/sec_application_record.rb index a64990f9ee75f88d66f415c9af5e4570a1fbb6cb..3169270241c8dc400580205aef180ea356ecbdb2 100644 --- a/app/models/sec_application_record.rb +++ b/app/models/sec_application_record.rb @@ -1,13 +1,51 @@ # frozen_string_literal: true class SecApplicationRecord < ApplicationRecord - self.abstract_class = true + self.abstract_class = true - connects_to database: { writing: :sec, reading: :sec } if Gitlab::Database.has_config?(:sec) + connects_to database: { writing: :sec, reading: :sec } if Gitlab::Database.has_config?(:sec) - class << self - def backup_model - Vulnerabilities::Backup.descendants.find { |descendant| self == descendant.original_model } - end - end + UnflaggedVulnReadDatabaseTriggerTransaction = Class.new(StandardError) + + class << self + def backup_model + Vulnerabilities::Backup.descendants.find { |descendant| self == descendant.original_model } + end + + #################### + # This transaction code exists to help identify and prevent instances of code that may need to explicitly pass + # a feature flag setting to the vulnerability reads database trigger. + # + # Once transitioned away from the database trigger, we can remove it. + def feature_flagged_transaction_for(projects) + SecApplicationRecord.transaction do + ::SecApplicationRecord.pass_feature_flag_to_vuln_reads_db_trigger(projects) + + yield + end + end + + def db_trigger_flag_not_set? + result = ::SecApplicationRecord.connection.execute( + "SELECT current_setting('vulnerability_management.dont_execute_db_trigger', true);" + ).first['current_setting'] + + result ? result.empty? : result.nil? + end + + def pass_feature_flag_to_vuln_reads_db_trigger(projects) + feature_enabled = if projects.nil? + Feature.enabled?(:turn_off_vulnerability_read_create_db_trigger_function, :instance) + else + Array(projects).all? do |project| + Feature.enabled?( + :turn_off_vulnerability_read_create_db_trigger_function, project) + end + end + + ::SecApplicationRecord.connection.execute("SELECT set_config( + 'vulnerability_management.dont_execute_db_trigger', '#{feature_enabled}', true);") + end + ################## + end end diff --git a/ee/app/models/concerns/vulnerabilities/enforce_vulnerability_read_db_trigger_ff.rb b/ee/app/models/concerns/vulnerabilities/enforce_vulnerability_read_db_trigger_ff.rb new file mode 100644 index 0000000000000000000000000000000000000000..b9d55128ef16d8eda368056347f84edd7dabd606 --- /dev/null +++ b/ee/app/models/concerns/vulnerabilities/enforce_vulnerability_read_db_trigger_ff.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +# This transaction code exists to help identify and prevent instances of code +# that may need to explicitly pass a feature flag setting to the vulnerability +# reads database trigger. +# +# Once transitioned away from the database trigger, we can remove it. + +module Vulnerabilities + module EnforceVulnerabilityReadDbTriggerFf + UnflaggedVulnReadDatabaseTriggerTransaction = Class.new(StandardError) + + def self.extended(base) + base.define_singleton_method(:transaction) do |*args, **kwargs, &block| + if base.db_trigger_flag_not_set? + if Rails.env.test? + raise UnflaggedVulnReadDatabaseTriggerTransaction, + 'This transaction is not passing the needed feature flag to the vulnerability read db trigger.' + else + Gitlab::AppLogger.warn( + "Sec transaction executed without setting vulnerability read db trigger feature flag!" + ) + end + end + + super(*args, **kwargs, &block) + end + end + end +end diff --git a/ee/app/models/ee/vulnerability.rb b/ee/app/models/ee/vulnerability.rb index d038afd3ddfcc783ecdf49ad3a58389437e34dd9..1fd1c3bd53af2995bcd27df6283147cde4777f43 100644 --- a/ee/app/models/ee/vulnerability.rb +++ b/ee/app/models/ee/vulnerability.rb @@ -21,6 +21,7 @@ module Vulnerability include ::VulnerabilityScopes include ::Gitlab::Utils::StrongMemoize include ::Elastic::ApplicationVersionedSearch + extend ::Vulnerabilities::EnforceVulnerabilityReadDbTriggerFf extend ::Gitlab::Utils::Override diff --git a/ee/app/models/vulnerabilities/finding.rb b/ee/app/models/vulnerabilities/finding.rb index 1fe365e9b608be84b34c30c35fc8d6ee552bf30e..8253ddde73054448462524a6cb454a0b722d2d4a 100644 --- a/ee/app/models/vulnerabilities/finding.rb +++ b/ee/app/models/vulnerabilities/finding.rb @@ -1,603 +1,604 @@ # frozen_string_literal: true module Vulnerabilities - class Finding < ::SecApplicationRecord - include ShaAttribute - include ::Gitlab::Utils::StrongMemoize - include Presentable - include ::VulnerabilityFindingHelpers - include EachBatch - - # https://gitlab.com/groups/gitlab-org/-/epics/3148 - # https://gitlab.com/gitlab-org/gitlab/-/issues/214563#note_370782508 is why the table names are not renamed - self.table_name = 'vulnerability_occurrences' - - FINDINGS_PER_PAGE = 20 - MAX_NUMBER_OF_IDENTIFIERS = 20 - REPORT_TYPES_WITH_LOCATION_IMAGE = %w[container_scanning cluster_image_scanning].freeze - SECRET_DETECTION_DEFAULT_COMMIT_SHA = "0000000" - - AI_ALLOWED_REPORT_TYPES = %w[sast].freeze - - # https://gitlab.com/gitlab-org/gitlab/-/issues/472861 - HIGH_CONFIDENCE_AI_RESOLUTION_CWES = %w[ - CWE-23 - CWE-73 - CWE-78 - CWE-80 - CWE-89 - CWE-116 - CWE-118 - CWE-119 - CWE-120 - CWE-126 - CWE-190 - CWE-200 - CWE-208 - CWE-209 - CWE-272 - CWE-287 - CWE-295 - CWE-297 - CWE-305 - CWE-310 - CWE-311 - CWE-323 - CWE-327 - CWE-328 - CWE-330 - CWE-338 - CWE-345 - CWE-346 - CWE-352 - CWE-362 - CWE-369 - CWE-377 - CWE-378 - CWE-400 - CWE-489 - CWE-521 - CWE-539 - CWE-599 - CWE-611 - CWE-676 - CWE-704 - CWE-754 - CWE-770 - CWE-1004 - CWE-1275 - ].to_set.freeze - - paginates_per FINDINGS_PER_PAGE - - sha_attribute :location_fingerprint - - attr_readonly :initial_pipeline_id - - belongs_to :project, inverse_of: :vulnerability_findings - belongs_to :scanner, class_name: 'Vulnerabilities::Scanner' - belongs_to :primary_identifier, class_name: 'Vulnerabilities::Identifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id' - belongs_to :vulnerability, class_name: 'Vulnerability', inverse_of: :findings, foreign_key: 'vulnerability_id' - has_one :one_vulnerability, class_name: 'Vulnerability', inverse_of: :vulnerability_finding - has_many :state_transitions, through: :vulnerability - has_many :issue_links, through: :vulnerability - has_many :external_issue_links, through: :vulnerability - has_many :merge_request_links, through: :vulnerability - - has_many :finding_identifiers, class_name: 'Vulnerabilities::FindingIdentifier', inverse_of: :finding, foreign_key: 'occurrence_id' - has_many :identifiers, through: :finding_identifiers, class_name: 'Vulnerabilities::Identifier' - - has_many :cve_identifiers, -> { where('LOWER(external_type) = ?', 'cve') }, through: :finding_identifiers, source: :identifier, class_name: 'Vulnerabilities::Identifier' - has_many :cve_enrichments, through: :cve_identifiers, disable_joins: true - - has_one :finding_token_status, class_name: 'Vulnerabilities::FindingTokenStatus', foreign_key: 'vulnerability_occurrence_id', inverse_of: :finding - - has_many :finding_links, class_name: 'Vulnerabilities::FindingLink', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' - - has_many :finding_remediations, class_name: 'Vulnerabilities::FindingRemediation', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' - has_many :remediations, through: :finding_remediations - - has_many :triggered_workflows, class_name: '::Vulnerabilities::TriggeredWorkflow', inverse_of: :vulnerability_occurrence, foreign_key: 'vulnerability_occurrence_id' - - has_one :finding_risk_score, class_name: 'Vulnerabilities::FindingRiskScore', inverse_of: :finding - - # rubocop: disable Rails/InverseOf -- these relations are not present on Ci::Pipeline - belongs_to :initial_finding_pipeline, class_name: '::Ci::Pipeline', foreign_key: 'initial_pipeline_id' - belongs_to :latest_finding_pipeline, class_name: '::Ci::Pipeline', foreign_key: 'latest_pipeline_id' - # rubocop:enable Rails/InverseOf - - has_many :signatures, class_name: 'Vulnerabilities::FindingSignature', inverse_of: :finding - - has_many :vulnerability_flags, class_name: 'Vulnerabilities::Flag', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' - - has_many :feedbacks, class_name: 'Vulnerabilities::Feedback', inverse_of: :finding, primary_key: 'uuid', foreign_key: 'finding_uuid' - - has_one :finding_evidence, class_name: 'Vulnerabilities::Finding::Evidence', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' - - has_many :security_findings, - class_name: 'Security::Finding', - primary_key: :uuid, - foreign_key: :uuid, - inverse_of: :vulnerability_finding - - attribute :config_options, ::Gitlab::Database::Type::IndifferentJsonb.new - - attr_writer :sha - attr_accessor :scan, :found_by_pipeline - - enum :report_type, ::Enums::Vulnerability.report_types - enum :severity, ::Enums::Vulnerability.severity_levels, prefix: :severity - enum :detection_method, ::Enums::Vulnerability.detection_methods - - validates :scanner, presence: true - validates :project, presence: true - validates :uuid, presence: true - - validates :primary_identifier, presence: true - validates :location_fingerprint, presence: true - # Uniqueness validation doesn't work with binary columns, so save this useless query. It is enforce by DB constraint anyway. - # TODO: find out why it fails - # validates :location_fingerprint, presence: true, uniqueness: { scope: [:primary_identifier_id, :scanner_id, :ref, :pipeline_id, :project_id] } - validates :name, presence: true - validates :report_type, presence: true - validates :severity, presence: true - validates :detection_method, presence: true - - validates :metadata_version, presence: true - validates :raw_metadata, presence: true - validates :details, json_schema: { filename: 'vulnerability_finding_details' } - - COLUMN_LENGTH_LIMITS = { - description: 15_000, - solution: 7_000 - }.freeze - - validates :description, length: { maximum: COLUMN_LENGTH_LIMITS[:description] } - validates :solution, length: { maximum: COLUMN_LENGTH_LIMITS[:solution] } - validates :cve, length: { maximum: 48400 } - - delegate :name, :external_id, to: :scanner, prefix: true, allow_nil: true - - scope :report_type, ->(type) { where(report_type: report_types[type]) } - scope :ordered, -> { order(severity: :desc, id: :asc) } - - scope :active, -> { joins(:vulnerability).merge(Vulnerability.active) } - scope :by_vulnerability, ->(vulnerability_id) { where(vulnerability: vulnerability_id) } - scope :ids_by_vulnerability, ->(vulnerability_id) { by_vulnerability(vulnerability_id).pluck(:id) } - scope :by_report_types, ->(values) { where(report_type: values) } - scope :by_projects, ->(values) { where(project_id: values) } - scope :by_scanners, ->(values) { where(scanner_id: values) } - scope :by_severities, ->(values) { where(severity: values) } - scope :by_location_fingerprints, ->(values) { where(location_fingerprint: values) } - scope :by_uuid, ->(uuids) { where(uuid: uuids) } - scope :excluding_uuids, ->(uuids) { where.not(uuid: uuids) } - scope :eager_load_comparison_entities, -> { includes(:scanner, :primary_identifier) } - scope :by_primary_identifiers, ->(identifier_ids) { where(primary_identifier: identifier_ids) } - scope :by_latest_pipeline, ->(pipeline_id) { where(latest_pipeline_id: pipeline_id) } - scope :with_project, -> { includes(:project) } - scope :with_token_status, -> { preload(:finding_token_status) } - scope :with_cve_enrichments, -> { includes(:cve_enrichments) } - scope :with_risk_score, -> { preload(:finding_risk_score) } - - scope :all_preloaded, -> do - preload(:scanner, :identifiers, :feedbacks, project: [:namespace, :project_feature]) - end - - scope :with_false_positive, ->(false_positive) do - flags = ::Vulnerabilities::Flag.arel_table - - where( - false_positive ? 'EXISTS (?)' : 'NOT EXISTS (?)', - ::Vulnerabilities::Flag.select(1).false_positive.where(flags[:vulnerability_occurrence_id].eq(arel_table[:id])) - ) - end - - scope :with_fix_available, ->(fix_available) do - remediation = ::Vulnerabilities::FindingRemediation.arel_table - solution_query = where(fix_available ? 'solution IS NOT NULL' : 'solution IS NULL') - exist_query = where( - fix_available ? 'EXISTS (?)' : 'NOT EXISTS (?)', - ::Vulnerabilities::FindingRemediation.select(1).where(remediation[:vulnerability_occurrence_id].eq(arel_table[:id])) - ) - - fix_available ? solution_query.or(exist_query) : solution_query.and(exist_query) - end - - scope :scoped_project, -> { where('vulnerability_occurrences.project_id = projects.id') } - scope :eager_load_vulnerability_flags, -> { includes(:vulnerability_flags) } - scope :by_location_image, ->(images) do - where(report_type: REPORT_TYPES_WITH_LOCATION_IMAGE) - .where("vulnerability_occurrences.location -> 'image' ?| array[:images]", images: images) - end - scope :by_location_cluster, ->(cluster_ids) do - where(report_type: 'cluster_image_scanning') - .where("vulnerability_occurrences.location -> 'kubernetes_resource' -> 'cluster_id' ?| array[:cluster_ids]", cluster_ids: cluster_ids) - end - scope :by_location_cluster_agent, ->(agent_ids) do - where(report_type: 'cluster_image_scanning') - .where("vulnerability_occurrences.location -> 'kubernetes_resource' -> 'agent_id' ?| array[:agent_ids]", agent_ids: agent_ids) - end - - alias_method :declarative_policy_subject, :project - alias_attribute :finding_details, :details - - def self.counted_by_severity - group(:severity).count.transform_keys do |severity| - severities[severity] - end - end - - # sha can be sourced from a joined pipeline or set from the report - def sha - # Some analysers (like Secret Detection) that produce security findings may perform scans across Git history and - # attach specific commit information to the finding. When this is the case, we _must_ use the commit SHA specified - # in the security report to compute the blob URL, otherwise the URL will link to the incorrect revision of the file. - # - # We also need to ensure we _don't_ use the commit SHA from the report if it's the default placeholder value, - # which is defined in the `secrets` analyzer: - # https://gitlab.com/gitlab-org/security-products/analyzers/secrets/-/blob/7e1e03209495a209308f3e9e96c5a4a0d32e1d55/secret.go#L13-13 - commit_sha = location.dig("commit", "sha") - if !commit_sha || commit_sha == SECRET_DETECTION_DEFAULT_COMMIT_SHA - # Two layers of fallbacks. - commit_sha = @sha || pipeline_branch - end - - commit_sha - end - - def state - if vulnerability.nil? || vulnerability.detected? - 'detected' - elsif vulnerability.resolved? - 'resolved' - elsif vulnerability.dismissed? # fail-safe check for cases when dismissal feedback was lost or was not created - 'dismissed' - else - 'confirmed' - end - end - - def token_status - finding_token_status&.status_before_type_cast || ::Security::TokenStatus::UNKNOWN - end - - def source_code? - source_code.present? - end - - def vulnerable_code(lines: vulnerable_lines) - strong_memoize_with(:vulnerable_code, lines) do - source_code.lines[lines]&.join - end - end - - def self.related_dismissal_feedback - Feedback.where('vulnerability_occurrences.uuid = vulnerability_feedback.finding_uuid') - .for_dismissal - end - private_class_method :related_dismissal_feedback - - def self.dismissed - where('EXISTS (?)', related_dismissal_feedback.select(1)) - end - - def self.undismissed - where('NOT EXISTS (?)', related_dismissal_feedback.select(1)) - end - - def feedback(feedback_type:) - load_feedback.find { |f| f.feedback_type == feedback_type } - end - - def load_feedback - BatchLoader.for(uuid).batch do |uuids, loader| - finding_feedbacks = Vulnerabilities::Feedback.all_preloaded.where(finding_uuid: uuids.uniq) - - uuids.each do |finding_uuid| - loader.call( - finding_uuid, - finding_feedbacks.select { |f| finding_uuid == f.finding_uuid } - ) - end - end - end - - def dismissal_feedback - feedback(feedback_type: 'dismissal') - end - - def issue_feedback - related_issues = vulnerability&.related_issues - related_issues.blank? ? feedback(feedback_type: 'issue') : Vulnerabilities::Feedback.find_by(issue: related_issues.first.id) - end - - def merge_request_feedback - feedback(feedback_type: 'merge_request') - end - - def metadata - strong_memoize(:metadata) do - data = Gitlab::Json.parse(raw_metadata) - - data = {} unless data.is_a?(Hash) - - data - rescue JSON::ParserError - {} - end - end - - def description - super.presence || metadata['description'] - end - - def solution - super.presence || metadata['solution'] || remediations&.first&.dig('summary') - end - - def location - super.presence || metadata.fetch('location', {}) - end - - def file - location['file'] - end - - def image - location['image'] - end - - def links - return metadata.fetch('links', []) if finding_links.load.empty? - - finding_links.as_json(only: [:name, :url]) - end - - def remediations - return metadata['remediations'] unless super.present? - - super.as_json(only: [:summary], methods: [:diff]) - end - - def token_type - return unless metadata['identifiers'] - - metadata['identifiers'].find { |hash| hash['type'] == 'gitleaks_rule_id' }&.dig('value') - end - - def token_value - metadata['raw_source_code_extract'] - end - - def cve_enrichment - return unless cve_enrichments.load.any? - - cve_enrichments.first - end - strong_memoize_attr :cve_enrichment - - def build_evidence_request(data) - return if data.nil? - - { - headers: data.fetch('headers', []).map do |request_header| - { - name: request_header['name'], - value: request_header['value'] - } - end, - method: data['method'], - url: data['url'], - body: data['body'] - } - end - - def build_evidence_response(data) - return if data.nil? - - { - headers: data.fetch('headers', []).map do |header_data| - { - name: header_data['name'], - value: header_data['value'] - } - end, - status_code: data['status_code'], - reason_phrase: data['reason_phrase'], - body: data['body'] - } - end - - def build_evidence_supporting_messages(data) - return [] if data.nil? - - data.map do |message| - { - name: message['name'], - request: build_evidence_request(message['request']), - response: build_evidence_response(message['response']) - } - end - end - - def build_evidence_source(data) - return if data.nil? - - { - id: data['id'], - name: data['name'], - url: data['url'] - } - end - - def evidence - evidence_data = finding_evidence.present? ? finding_evidence.data : metadata['evidence'] - - return if evidence_data.nil? - - { - summary: evidence_data&.dig('summary'), - request: build_evidence_request(evidence_data&.dig('request')), - response: build_evidence_response(evidence_data&.dig('response')), - source: build_evidence_source(evidence_data&.dig('source')), - supporting_messages: build_evidence_supporting_messages(evidence_data&.dig('supporting_messages')) - } - end - - def cve_value - identifiers.find(&:cve?)&.name - end - - def cwe_value - identifiers.find(&:cwe?)&.name - end - - def other_identifier_values - identifiers.select(&:other?).map(&:name) - end - - def assets - metadata.fetch('assets', []).map do |asset_data| - { - name: asset_data['name'], - type: asset_data['type'], - url: asset_data['url'] - } - end - end - - def risk_score - finding_risk_score&.risk_score || 0.0 - end - - alias_method :==, :eql? - - def eql?(other) - return false unless other.is_a?(self.class) - - unless other.report_type == report_type && other.primary_identifier_fingerprint == primary_identifier_fingerprint - return false - end - - if project.licensed_feature_available?(:vulnerability_finding_signatures) - matches_signatures(other.signatures, other.uuid) - else - other.location_fingerprint == location_fingerprint - end - end - - # Array.difference (-) method uses hash and eql? methods to do comparison - def hash - # This is causing N+1 queries whenever we are calling findings, ActiveRecord uses #hash method to make sure the - # array with findings is uniq before preloading. This method is used only in Gitlab::Ci::Reports::Security::VulnerabilityReportsComparer - # where we are normalizing security report findings into instances of Vulnerabilities::Finding, this is why we are using original implementation - # when Finding is persisted and identifiers are not preloaded. - return super if persisted? && !identifiers.loaded? - - report_type.hash ^ location_fingerprint.hash ^ primary_identifier_fingerprint.hash - end - - def severity_value - self.class.severities[self.severity] - end - - # We will eventually have only UUIDv5 values for the `uuid` - # attribute of the finding records. - def uuid_v5 - if Gitlab::UUID.v5?(uuid) - uuid - else - ::Security::VulnerabilityUUID.generate( - report_type: report_type, - primary_identifier_fingerprint: primary_identifier.fingerprint, - location_fingerprint: location_fingerprint, - project_id: project_id - ) - end - end - - def self.pluck_uuids - pluck(:uuid) - end - - def self.pluck_vulnerability_ids - pluck(:vulnerability_id) - end - - def pipeline_branch - last_finding_pipeline&.sha || project.default_branch - end - - def false_positive? - vulnerability_flags.any?(&:false_positive?) - end - - def first_finding_pipeline - initial_finding_pipeline - end - - def last_finding_pipeline - latest_finding_pipeline - end - - def vulnerable_lines - # -1 is a magic number here meaning an explicit value - # for start_line or end_line was not provided. If neither - # were provided we return the entire file contents. - return (0..-1) if (start_line < 0) && (end_line < 0) - - range_start = [start_line, 0].max - range_end = [end_line, range_start].max - - (range_start..range_end) - end - - def start_line - location["start_line"].to_i - 1 - end - - def end_line - location["end_line"].to_i - 1 - end - - def source_code - return "" unless file.present? - - blob = project.repository.blob_at(pipeline_branch, file) - blob.present? ? blob.data : "" - end - strong_memoize_attr :source_code - - def identifier_names - identifiers.pluck(:name) - end - - def ai_explanation_available? - AI_ALLOWED_REPORT_TYPES.include?(report_type) - end - - def ai_resolution_available? - AI_ALLOWED_REPORT_TYPES.include?(report_type) - end - - def ai_resolution_enabled? - ai_resolution_available? && ai_resolution_supported_cwe? - end - - def ai_resolution_supported_cwe? - if ::Feature.enabled?(:ignore_supported_cwe_list_check, project) - true - else - HIGH_CONFIDENCE_AI_RESOLUTION_CWES.include?(cwe_value&.upcase) - end - end - - protected - - def primary_identifier_fingerprint - identifiers.first&.fingerprint - end - end + class Finding < ::SecApplicationRecord + extend EnforceVulnerabilityReadDbTriggerFf + include ShaAttribute + include ::Gitlab::Utils::StrongMemoize + include Presentable + include ::VulnerabilityFindingHelpers + include EachBatch + + # https://gitlab.com/groups/gitlab-org/-/epics/3148 + # https://gitlab.com/gitlab-org/gitlab/-/issues/214563#note_370782508 is why the table names are not renamed + self.table_name = 'vulnerability_occurrences' + + FINDINGS_PER_PAGE = 20 + MAX_NUMBER_OF_IDENTIFIERS = 20 + REPORT_TYPES_WITH_LOCATION_IMAGE = %w[container_scanning cluster_image_scanning].freeze + SECRET_DETECTION_DEFAULT_COMMIT_SHA = "0000000" + + AI_ALLOWED_REPORT_TYPES = %w[sast].freeze + + # https://gitlab.com/gitlab-org/gitlab/-/issues/472861 + HIGH_CONFIDENCE_AI_RESOLUTION_CWES = %w[ + CWE-23 + CWE-73 + CWE-78 + CWE-80 + CWE-89 + CWE-116 + CWE-118 + CWE-119 + CWE-120 + CWE-126 + CWE-190 + CWE-200 + CWE-208 + CWE-209 + CWE-272 + CWE-287 + CWE-295 + CWE-297 + CWE-305 + CWE-310 + CWE-311 + CWE-323 + CWE-327 + CWE-328 + CWE-330 + CWE-338 + CWE-345 + CWE-346 + CWE-352 + CWE-362 + CWE-369 + CWE-377 + CWE-378 + CWE-400 + CWE-489 + CWE-521 + CWE-539 + CWE-599 + CWE-611 + CWE-676 + CWE-704 + CWE-754 + CWE-770 + CWE-1004 + CWE-1275 + ].to_set.freeze + + paginates_per FINDINGS_PER_PAGE + + sha_attribute :location_fingerprint + + attr_readonly :initial_pipeline_id + + belongs_to :project, inverse_of: :vulnerability_findings + belongs_to :scanner, class_name: 'Vulnerabilities::Scanner' + belongs_to :primary_identifier, class_name: 'Vulnerabilities::Identifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id' + belongs_to :vulnerability, class_name: 'Vulnerability', inverse_of: :findings, foreign_key: 'vulnerability_id' + has_one :one_vulnerability, class_name: 'Vulnerability', inverse_of: :vulnerability_finding + has_many :state_transitions, through: :vulnerability + has_many :issue_links, through: :vulnerability + has_many :external_issue_links, through: :vulnerability + has_many :merge_request_links, through: :vulnerability + + has_many :finding_identifiers, class_name: 'Vulnerabilities::FindingIdentifier', inverse_of: :finding, foreign_key: 'occurrence_id' + has_many :identifiers, through: :finding_identifiers, class_name: 'Vulnerabilities::Identifier' + + has_many :cve_identifiers, -> { where('LOWER(external_type) = ?', 'cve') }, through: :finding_identifiers, source: :identifier, class_name: 'Vulnerabilities::Identifier' + has_many :cve_enrichments, through: :cve_identifiers, disable_joins: true + + has_one :finding_token_status, class_name: 'Vulnerabilities::FindingTokenStatus', foreign_key: 'vulnerability_occurrence_id', inverse_of: :finding + + has_many :finding_links, class_name: 'Vulnerabilities::FindingLink', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' + + has_many :finding_remediations, class_name: 'Vulnerabilities::FindingRemediation', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' + has_many :remediations, through: :finding_remediations + + has_many :triggered_workflows, class_name: '::Vulnerabilities::TriggeredWorkflow', inverse_of: :vulnerability_occurrence, foreign_key: 'vulnerability_occurrence_id' + + has_one :finding_risk_score, class_name: 'Vulnerabilities::FindingRiskScore', inverse_of: :finding + + # rubocop: disable Rails/InverseOf -- these relations are not present on Ci::Pipeline + belongs_to :initial_finding_pipeline, class_name: '::Ci::Pipeline', foreign_key: 'initial_pipeline_id' + belongs_to :latest_finding_pipeline, class_name: '::Ci::Pipeline', foreign_key: 'latest_pipeline_id' + # rubocop:enable Rails/InverseOf + + has_many :signatures, class_name: 'Vulnerabilities::FindingSignature', inverse_of: :finding + + has_many :vulnerability_flags, class_name: 'Vulnerabilities::Flag', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' + + has_many :feedbacks, class_name: 'Vulnerabilities::Feedback', inverse_of: :finding, primary_key: 'uuid', foreign_key: 'finding_uuid' + + has_one :finding_evidence, class_name: 'Vulnerabilities::Finding::Evidence', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' + + has_many :security_findings, + class_name: 'Security::Finding', + primary_key: :uuid, + foreign_key: :uuid, + inverse_of: :vulnerability_finding + + attribute :config_options, ::Gitlab::Database::Type::IndifferentJsonb.new + + attr_writer :sha + attr_accessor :scan, :found_by_pipeline + + enum :report_type, ::Enums::Vulnerability.report_types + enum :severity, ::Enums::Vulnerability.severity_levels, prefix: :severity + enum :detection_method, ::Enums::Vulnerability.detection_methods + + validates :scanner, presence: true + validates :project, presence: true + validates :uuid, presence: true + + validates :primary_identifier, presence: true + validates :location_fingerprint, presence: true + # Uniqueness validation doesn't work with binary columns, so save this useless query. It is enforce by DB constraint anyway. + # TODO: find out why it fails + # validates :location_fingerprint, presence: true, uniqueness: { scope: [:primary_identifier_id, :scanner_id, :ref, :pipeline_id, :project_id] } + validates :name, presence: true + validates :report_type, presence: true + validates :severity, presence: true + validates :detection_method, presence: true + + validates :metadata_version, presence: true + validates :raw_metadata, presence: true + validates :details, json_schema: { filename: 'vulnerability_finding_details' } + + COLUMN_LENGTH_LIMITS = { + description: 15_000, + solution: 7_000 + }.freeze + + validates :description, length: { maximum: COLUMN_LENGTH_LIMITS[:description] } + validates :solution, length: { maximum: COLUMN_LENGTH_LIMITS[:solution] } + validates :cve, length: { maximum: 48400 } + + delegate :name, :external_id, to: :scanner, prefix: true, allow_nil: true + + scope :report_type, ->(type) { where(report_type: report_types[type]) } + scope :ordered, -> { order(severity: :desc, id: :asc) } + + scope :active, -> { joins(:vulnerability).merge(Vulnerability.active) } + scope :by_vulnerability, ->(vulnerability_id) { where(vulnerability: vulnerability_id) } + scope :ids_by_vulnerability, ->(vulnerability_id) { by_vulnerability(vulnerability_id).pluck(:id) } + scope :by_report_types, ->(values) { where(report_type: values) } + scope :by_projects, ->(values) { where(project_id: values) } + scope :by_scanners, ->(values) { where(scanner_id: values) } + scope :by_severities, ->(values) { where(severity: values) } + scope :by_location_fingerprints, ->(values) { where(location_fingerprint: values) } + scope :by_uuid, ->(uuids) { where(uuid: uuids) } + scope :excluding_uuids, ->(uuids) { where.not(uuid: uuids) } + scope :eager_load_comparison_entities, -> { includes(:scanner, :primary_identifier) } + scope :by_primary_identifiers, ->(identifier_ids) { where(primary_identifier: identifier_ids) } + scope :by_latest_pipeline, ->(pipeline_id) { where(latest_pipeline_id: pipeline_id) } + scope :with_project, -> { includes(:project) } + scope :with_token_status, -> { preload(:finding_token_status) } + scope :with_cve_enrichments, -> { includes(:cve_enrichments) } + scope :with_risk_score, -> { preload(:finding_risk_score) } + + scope :all_preloaded, -> do + preload(:scanner, :identifiers, :feedbacks, project: [:namespace, :project_feature]) + end + + scope :with_false_positive, ->(false_positive) do + flags = ::Vulnerabilities::Flag.arel_table + + where( + false_positive ? 'EXISTS (?)' : 'NOT EXISTS (?)', + ::Vulnerabilities::Flag.select(1).false_positive.where(flags[:vulnerability_occurrence_id].eq(arel_table[:id])) + ) + end + + scope :with_fix_available, ->(fix_available) do + remediation = ::Vulnerabilities::FindingRemediation.arel_table + solution_query = where(fix_available ? 'solution IS NOT NULL' : 'solution IS NULL') + exist_query = where( + fix_available ? 'EXISTS (?)' : 'NOT EXISTS (?)', + ::Vulnerabilities::FindingRemediation.select(1).where(remediation[:vulnerability_occurrence_id].eq(arel_table[:id])) + ) + + fix_available ? solution_query.or(exist_query) : solution_query.and(exist_query) + end + + scope :scoped_project, -> { where('vulnerability_occurrences.project_id = projects.id') } + scope :eager_load_vulnerability_flags, -> { includes(:vulnerability_flags) } + scope :by_location_image, ->(images) do + where(report_type: REPORT_TYPES_WITH_LOCATION_IMAGE) + .where("vulnerability_occurrences.location -> 'image' ?| array[:images]", images: images) + end + scope :by_location_cluster, ->(cluster_ids) do + where(report_type: 'cluster_image_scanning') + .where("vulnerability_occurrences.location -> 'kubernetes_resource' -> 'cluster_id' ?| array[:cluster_ids]", cluster_ids: cluster_ids) + end + scope :by_location_cluster_agent, ->(agent_ids) do + where(report_type: 'cluster_image_scanning') + .where("vulnerability_occurrences.location -> 'kubernetes_resource' -> 'agent_id' ?| array[:agent_ids]", agent_ids: agent_ids) + end + + alias_method :declarative_policy_subject, :project + alias_attribute :finding_details, :details + + def self.counted_by_severity + group(:severity).count.transform_keys do |severity| + severities[severity] + end + end + + # sha can be sourced from a joined pipeline or set from the report + def sha + # Some analysers (like Secret Detection) that produce security findings may perform scans across Git history and + # attach specific commit information to the finding. When this is the case, we _must_ use the commit SHA specified + # in the security report to compute the blob URL, otherwise the URL will link to the incorrect revision of the file. + # + # We also need to ensure we _don't_ use the commit SHA from the report if it's the default placeholder value, + # which is defined in the `secrets` analyzer: + # https://gitlab.com/gitlab-org/security-products/analyzers/secrets/-/blob/7e1e03209495a209308f3e9e96c5a4a0d32e1d55/secret.go#L13-13 + commit_sha = location.dig("commit", "sha") + if !commit_sha || commit_sha == SECRET_DETECTION_DEFAULT_COMMIT_SHA + # Two layers of fallbacks. + commit_sha = @sha || pipeline_branch + end + + commit_sha + end + + def state + if vulnerability.nil? || vulnerability.detected? + 'detected' + elsif vulnerability.resolved? + 'resolved' + elsif vulnerability.dismissed? # fail-safe check for cases when dismissal feedback was lost or was not created + 'dismissed' + else + 'confirmed' + end + end + + def token_status + finding_token_status&.status_before_type_cast || ::Security::TokenStatus::UNKNOWN + end + + def source_code? + source_code.present? + end + + def vulnerable_code(lines: vulnerable_lines) + strong_memoize_with(:vulnerable_code, lines) do + source_code.lines[lines]&.join + end + end + + def self.related_dismissal_feedback + Feedback.where('vulnerability_occurrences.uuid = vulnerability_feedback.finding_uuid') + .for_dismissal + end + private_class_method :related_dismissal_feedback + + def self.dismissed + where('EXISTS (?)', related_dismissal_feedback.select(1)) + end + + def self.undismissed + where('NOT EXISTS (?)', related_dismissal_feedback.select(1)) + end + + def feedback(feedback_type:) + load_feedback.find { |f| f.feedback_type == feedback_type } + end + + def load_feedback + BatchLoader.for(uuid).batch do |uuids, loader| + finding_feedbacks = Vulnerabilities::Feedback.all_preloaded.where(finding_uuid: uuids.uniq) + + uuids.each do |finding_uuid| + loader.call( + finding_uuid, + finding_feedbacks.select { |f| finding_uuid == f.finding_uuid } + ) + end + end + end + + def dismissal_feedback + feedback(feedback_type: 'dismissal') + end + + def issue_feedback + related_issues = vulnerability&.related_issues + related_issues.blank? ? feedback(feedback_type: 'issue') : Vulnerabilities::Feedback.find_by(issue: related_issues.first.id) + end + + def merge_request_feedback + feedback(feedback_type: 'merge_request') + end + + def metadata + strong_memoize(:metadata) do + data = Gitlab::Json.parse(raw_metadata) + + data = {} unless data.is_a?(Hash) + + data + rescue JSON::ParserError + {} + end + end + + def description + super.presence || metadata['description'] + end + + def solution + super.presence || metadata['solution'] || remediations&.first&.dig('summary') + end + + def location + super.presence || metadata.fetch('location', {}) + end + + def file + location['file'] + end + + def image + location['image'] + end + + def links + return metadata.fetch('links', []) if finding_links.load.empty? + + finding_links.as_json(only: [:name, :url]) + end + + def remediations + return metadata['remediations'] unless super.present? + + super.as_json(only: [:summary], methods: [:diff]) + end + + def token_type + return unless metadata['identifiers'] + + metadata['identifiers'].find { |hash| hash['type'] == 'gitleaks_rule_id' }&.dig('value') + end + + def token_value + metadata['raw_source_code_extract'] + end + + def cve_enrichment + return unless cve_enrichments.load.any? + + cve_enrichments.first + end + strong_memoize_attr :cve_enrichment + + def build_evidence_request(data) + return if data.nil? + + { + headers: data.fetch('headers', []).map do |request_header| + { + name: request_header['name'], + value: request_header['value'] + } + end, + method: data['method'], + url: data['url'], + body: data['body'] + } + end + + def build_evidence_response(data) + return if data.nil? + + { + headers: data.fetch('headers', []).map do |header_data| + { + name: header_data['name'], + value: header_data['value'] + } + end, + status_code: data['status_code'], + reason_phrase: data['reason_phrase'], + body: data['body'] + } + end + + def build_evidence_supporting_messages(data) + return [] if data.nil? + + data.map do |message| + { + name: message['name'], + request: build_evidence_request(message['request']), + response: build_evidence_response(message['response']) + } + end + end + + def build_evidence_source(data) + return if data.nil? + + { + id: data['id'], + name: data['name'], + url: data['url'] + } + end + + def evidence + evidence_data = finding_evidence.present? ? finding_evidence.data : metadata['evidence'] + + return if evidence_data.nil? + + { + summary: evidence_data&.dig('summary'), + request: build_evidence_request(evidence_data&.dig('request')), + response: build_evidence_response(evidence_data&.dig('response')), + source: build_evidence_source(evidence_data&.dig('source')), + supporting_messages: build_evidence_supporting_messages(evidence_data&.dig('supporting_messages')) + } + end + + def cve_value + identifiers.find(&:cve?)&.name + end + + def cwe_value + identifiers.find(&:cwe?)&.name + end + + def other_identifier_values + identifiers.select(&:other?).map(&:name) + end + + def assets + metadata.fetch('assets', []).map do |asset_data| + { + name: asset_data['name'], + type: asset_data['type'], + url: asset_data['url'] + } + end + end + + def risk_score + finding_risk_score&.risk_score || 0.0 + end + + alias_method :==, :eql? + + def eql?(other) + return false unless other.is_a?(self.class) + + unless other.report_type == report_type && other.primary_identifier_fingerprint == primary_identifier_fingerprint + return false + end + + if project.licensed_feature_available?(:vulnerability_finding_signatures) + matches_signatures(other.signatures, other.uuid) + else + other.location_fingerprint == location_fingerprint + end + end + + # Array.difference (-) method uses hash and eql? methods to do comparison + def hash + # This is causing N+1 queries whenever we are calling findings, ActiveRecord uses #hash method to make sure the + # array with findings is uniq before preloading. This method is used only in Gitlab::Ci::Reports::Security::VulnerabilityReportsComparer + # where we are normalizing security report findings into instances of Vulnerabilities::Finding, this is why we are using original implementation + # when Finding is persisted and identifiers are not preloaded. + return super if persisted? && !identifiers.loaded? + + report_type.hash ^ location_fingerprint.hash ^ primary_identifier_fingerprint.hash + end + + def severity_value + self.class.severities[self.severity] + end + + # We will eventually have only UUIDv5 values for the `uuid` + # attribute of the finding records. + def uuid_v5 + if Gitlab::UUID.v5?(uuid) + uuid + else + ::Security::VulnerabilityUUID.generate( + report_type: report_type, + primary_identifier_fingerprint: primary_identifier.fingerprint, + location_fingerprint: location_fingerprint, + project_id: project_id + ) + end + end + + def self.pluck_uuids + pluck(:uuid) + end + + def self.pluck_vulnerability_ids + pluck(:vulnerability_id) + end + + def pipeline_branch + last_finding_pipeline&.sha || project.default_branch + end + + def false_positive? + vulnerability_flags.any?(&:false_positive?) + end + + def first_finding_pipeline + initial_finding_pipeline + end + + def last_finding_pipeline + latest_finding_pipeline + end + + def vulnerable_lines + # -1 is a magic number here meaning an explicit value + # for start_line or end_line was not provided. If neither + # were provided we return the entire file contents. + return (0..-1) if (start_line < 0) && (end_line < 0) + + range_start = [start_line, 0].max + range_end = [end_line, range_start].max + + (range_start..range_end) + end + + def start_line + location["start_line"].to_i - 1 + end + + def end_line + location["end_line"].to_i - 1 + end + + def source_code + return "" unless file.present? + + blob = project.repository.blob_at(pipeline_branch, file) + blob.present? ? blob.data : "" + end + strong_memoize_attr :source_code + + def identifier_names + identifiers.pluck(:name) + end + + def ai_explanation_available? + AI_ALLOWED_REPORT_TYPES.include?(report_type) + end + + def ai_resolution_available? + AI_ALLOWED_REPORT_TYPES.include?(report_type) + end + + def ai_resolution_enabled? + ai_resolution_available? && ai_resolution_supported_cwe? + end + + def ai_resolution_supported_cwe? + if ::Feature.enabled?(:ignore_supported_cwe_list_check, project) + true + else + HIGH_CONFIDENCE_AI_RESOLUTION_CWES.include?(cwe_value&.upcase) + end + end + + protected + + def primary_identifier_fingerprint + identifiers.first&.fingerprint + end + end end Vulnerabilities::Finding.prepend_mod diff --git a/ee/app/models/vulnerabilities/merge_request_link.rb b/ee/app/models/vulnerabilities/merge_request_link.rb index 0b8692f44713ff2ce3d2b2746198c0508288ee2a..743e4dbca33b2573dc83909ddb76ba80a4252a1a 100644 --- a/ee/app/models/vulnerabilities/merge_request_link.rb +++ b/ee/app/models/vulnerabilities/merge_request_link.rb @@ -1,41 +1,42 @@ # frozen_string_literal: true module Vulnerabilities - class MergeRequestLink < ::SecApplicationRecord - include EachBatch + class MergeRequestLink < ::SecApplicationRecord + extend EnforceVulnerabilityReadDbTriggerFf + include EachBatch - MAX_MERGE_REQUEST_LINKS_PER_VULNERABILITY = 100 + MAX_MERGE_REQUEST_LINKS_PER_VULNERABILITY = 100 - self.table_name = 'vulnerability_merge_request_links' + self.table_name = 'vulnerability_merge_request_links' - belongs_to :vulnerability - belongs_to :merge_request - belongs_to :vulnerability_occurrence, optional: true, class_name: 'Vulnerabilities::Finding' + belongs_to :vulnerability + belongs_to :merge_request + belongs_to :vulnerability_occurrence, optional: true, class_name: 'Vulnerabilities::Finding' - has_one :author, through: :merge_request, class_name: 'User' + has_one :author, through: :merge_request, class_name: 'User' - validates :vulnerability, :merge_request, presence: true - validates :merge_request_id, - uniqueness: { scope: :vulnerability_id, message: N_('is already linked to this vulnerability') } + validates :vulnerability, :merge_request, presence: true + validates :merge_request_id, + uniqueness: { scope: :vulnerability_id, message: N_('is already linked to this vulnerability') } - scope :by_finding_uuids, ->(uuids) do - joins(vulnerability: [:findings]).where(vulnerability: { - vulnerability_occurrences: { uuid: uuids } - }) - end - scope :with_vulnerability_findings, -> { includes(vulnerability: [:findings]) } - scope :with_merge_request, -> { preload(:merge_request) } - scope :by_vulnerability, ->(values) { where(vulnerability_id: values) } + scope :by_finding_uuids, ->(uuids) do + joins(vulnerability: [:findings]).where(vulnerability: { + vulnerability_occurrences: { uuid: uuids } + }) + end + scope :with_vulnerability_findings, -> { includes(vulnerability: [:findings]) } + scope :with_merge_request, -> { preload(:merge_request) } + scope :by_vulnerability, ->(values) { where(vulnerability_id: values) } - def self.count_for_vulnerability(vulnerability) - where(vulnerability: vulnerability).count - end + def self.count_for_vulnerability(vulnerability) + where(vulnerability: vulnerability).count + end - def self.limit_exceeded_for_vulnerability?(vulnerability) - count_for_vulnerability(vulnerability) >= MAX_MERGE_REQUEST_LINKS_PER_VULNERABILITY - end + def self.limit_exceeded_for_vulnerability?(vulnerability) + count_for_vulnerability(vulnerability) >= MAX_MERGE_REQUEST_LINKS_PER_VULNERABILITY + end - def self.find_by_vulnerability_and_merge_request(vulnerability, merge_request) - find_by(vulnerability: vulnerability, merge_request: merge_request) - end - end + def self.find_by_vulnerability_and_merge_request(vulnerability, merge_request) + find_by(vulnerability: vulnerability, merge_request: merge_request) + end + end end diff --git a/ee/app/models/vulnerabilities/read.rb b/ee/app/models/vulnerabilities/read.rb index 2cac5e118730dd59e95d9579cb13a30226af854c..82b693ed93a343febd91bd74f8cecc31d2b89630 100644 --- a/ee/app/models/vulnerabilities/read.rb +++ b/ee/app/models/vulnerabilities/read.rb @@ -1,320 +1,321 @@ # frozen_string_literal: true module Vulnerabilities - class Read < ::SecApplicationRecord - extend ::Gitlab::Utils::Override - include ::Namespaces::Traversal::Traversable - include VulnerabilityScopes - include EachBatch - include UnnestedInFilters::Dsl - include FromUnion - include SafelyChangeColumnDefault - include ::Gitlab::SQL::Pattern - include ::Elastic::ApplicationVersionedSearch - - ignore_column :namespace_id, remove_with: '17.7', remove_after: '2024-11-21' - - declarative_enum DismissalReasonEnum - - # Included after scopes and relationships to avoid the warning - include BulkInsertSafe - - SEVERITY_COUNT_LIMIT = 1001 - OWASP_TOP_10_DEFAULT = -1 - - ELASTICSEARCH_TRACKED_FIELDS = ::Search::Elastic::References::Vulnerability::DIRECT_FIELDS + - ::Search::Elastic::References::Vulnerability::DIRECT_TYPECAST_FIELDS + %w[traversal_ids] - - self.table_name = "vulnerability_reads" - self.primary_key = :vulnerability_id - - columns_changing_default :owasp_top_10 - - delegate :group_name, :title, :created_at, :project_name, :finding_scanner_name, :finding_description, :cve_value, :cwe_value, :location, :notes_summary, :full_path, to: :vulnerability, allow_nil: true - delegate :other_identifier_values, :cvss_vectors_with_vendor, to: :vulnerability, allow_nil: true - delegate :dismissed?, to: :vulnerability - - belongs_to :vulnerability, inverse_of: :vulnerability_read - belongs_to :project - belongs_to :scanner, class_name: 'Vulnerabilities::Scanner' - belongs_to :vulnerability_occurrence, optional: true, class_name: 'Vulnerabilities::Finding' - - validates :vulnerability_id, uniqueness: true, presence: true - validates :project_id, presence: true - validates :scanner_id, presence: true - validates :report_type, presence: true - validates :severity, presence: true - validates :state, presence: true - validates :uuid, presence: true - - validates :location_image, length: { maximum: 2048 } - validates :has_issues, inclusion: { in: [true, false], message: N_('must be a boolean value') } - validates :has_merge_request, inclusion: { in: [true, false], message: N_('must be a boolean value') } - validates :resolved_on_default_branch, inclusion: { in: [true, false], message: N_('must be a boolean value') } - validates :has_remediations, inclusion: { in: [true, false], message: N_('must be a boolean value') } - - enum :state, ::Enums::Vulnerability.vulnerability_states - enum :report_type, ::Enums::Vulnerability.report_types - enum :severity, ::Enums::Vulnerability.severity_levels, prefix: :severity - enum :owasp_top_10, ::Enums::Vulnerability.owasp_top_10.merge('undefined' => OWASP_TOP_10_DEFAULT) - - after_initialize :set_default_values, if: :new_record? - - scope :by_uuid, ->(uuids) { where(uuid: uuids) } - scope :by_vulnerabilities, ->(vulnerabilities) { where(vulnerability: vulnerabilities) } - - class << self - alias_method :by_vulnerability, :by_vulnerabilities - end - - scope :order_severity_asc, -> { reorder(severity: :asc, vulnerability_id: :desc) } - scope :order_severity_desc, -> { reorder(severity: :desc, vulnerability_id: :desc) } - scope :order_detected_at_asc, -> { reorder(vulnerability_id: :asc) } - scope :order_detected_at_desc, -> { reorder(vulnerability_id: :desc) } - - scope :order_severity_asc_traversal_ids_asc, -> { reorder(severity: :asc, traversal_ids: :asc, vulnerability_id: :asc) } - scope :order_severity_desc_traversal_ids_desc, -> { reorder(severity: :desc, traversal_ids: :desc, vulnerability_id: :desc) } - - scope :in_parent_group_after_and_including, ->(vulnerability_read) do - where(arel_grouping_by_traversal_ids_and_vulnerability_id.gteq(vulnerability_read.arel_grouping_by_traversal_ids_and_id)) - end - scope :in_parent_group_before_and_including, ->(vulnerability_read) do - where(arel_grouping_by_traversal_ids_and_vulnerability_id.lteq(vulnerability_read.arel_grouping_by_traversal_ids_and_id)) - end - scope :by_group, ->(group) { within(group.traversal_ids) } - scope :unarchived, -> { where(archived: false) } - scope :order_traversal_ids_asc, -> do - reorder(Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'traversal_ids', - order_expression: arel_table[:traversal_ids].asc, - nullable: :not_nullable - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'vulnerability_id', - order_expression: arel_table[:vulnerability_id].asc - ) - ])) - end - scope :by_projects, ->(values) { where(project_id: values) } - scope :by_scanner, ->(scanner) { where(scanner: scanner) } - scope :by_scanner_ids, ->(scanner_ids) { where(scanner_id: scanner_ids) } - scope :grouped_by_severity, -> { reorder(severity: :desc).group(:severity) } - scope :with_report_types, ->(report_types) { where(report_type: report_types) } - scope :with_severities, ->(severities) { where(severity: severities) } - scope :with_states, ->(states) { where(state: states) } - scope :with_owasp_top_10, ->(owasp_top_10) { where(owasp_top_10: owasp_top_10) } - scope :with_identifier_name, ->(name) do - return none if name.nil? - - where("EXISTS ( - SELECT 1 - FROM unnest(vulnerability_reads.identifier_names) AS idt_names - WHERE idt_names ILIKE ? - )", sanitize_sql_like(name)) - end - scope :with_container_image, ->(images) { where(location_image: images) } - scope :with_container_image_starting_with, ->(image) { where(arel_table[:location_image].matches("#{sanitize_sql_like(image)}%")) } - scope :with_cluster_agent_ids, ->(agent_ids) { where(cluster_agent_id: agent_ids) } - scope :with_resolution, ->(has_resolution = true) { where(resolved_on_default_branch: has_resolution) } - scope :with_ai_resolution, ->(resolution = true) { where(has_vulnerability_resolution: resolution) } - scope :with_issues, ->(has_issues = true) { where(has_issues: has_issues) } - scope :with_merge_request, ->(has_merge_request = true) { where(has_merge_request: has_merge_request) } - scope :with_remediations, ->(has_remediations = true) { where(has_remediations: has_remediations) } - scope :with_scanner_external_ids, - ->(scanner_external_ids) { - joins(:scanner).merge(::Vulnerabilities::Scanner.with_external_id(scanner_external_ids)) - } - scope :with_findings_scanner_and_identifiers, - -> { - includes(vulnerability: { findings: [:scanner, :identifiers, { finding_identifiers: :identifier }] }) - } - scope :preload_indexing_data, -> { - preload( - :scanner, - { vulnerability: [ - { findings: [ - { identifiers: [] }, - { finding_identifiers: :identifier } - ] } - ] }, - { project: { namespace: :route } } - ) - } - scope :resolved_on_default_branch, -> { where('resolved_on_default_branch IS TRUE') } - scope :with_dismissal_reason, ->(dismissal_reason) { where(dismissal_reason: dismissal_reason) } - scope :with_export_entities, -> do - preload( - vulnerability: [ - :group, - { project: [:route], - notes: [:updated_by, :author], - findings: [:scanner, :identifiers] } - ] - ) - end - - scope :as_vulnerabilities, -> do - preload(vulnerability: { project: [:route] }).current_scope.tap do |relation| - relation.define_singleton_method(:records) do - super().map(&:vulnerability) - end - end - end - - scope :by_group_using_nested_loop, ->(group) do - where(traversal_ids: all_vulnerable_traversal_ids_for(group)) - end - - scope :with_findings_scanner_identifiers_and_notes, -> { with_findings_scanner_and_identifiers.includes(vulnerability: :notes) } - scope :with_limit, ->(maximum) { limit(maximum) } - scope :order_id_desc, -> { reorder(arel_table[:vulnerability_id].desc) } - - scope :autocomplete_search, ->(query) do - return self if query.blank? - - id_as_text = Arel::Nodes::NamedFunction.new('CAST', [arel_table[:vulnerability_id].as('TEXT')]) - - joins(:vulnerability) - .select('vulnerability_reads.*, vulnerabilities.title') - .fuzzy_search(query, [Vulnerability.arel_table[:title]]) - .or(where(id_as_text.matches("%#{sanitize_sql_like(query.squish)}%"))) - end - - scope :by_ids_desc, ->(ids) do - by_vulnerability(ids).order_id_desc - end - - def self.es_type - ::Search::Elastic::References::Vulnerability::DOC_TYPE - end - - def self.arel_grouping_by_traversal_ids_and_vulnerability_id - arel_table.grouping([arel_table['traversal_ids'], arel_table['vulnerability_id']]) - end - - def self.all_vulnerable_traversal_ids_for(group) - by_group(group).unarchived.loose_index_scan(column: :traversal_ids) - end - - def self.count_by_severity - grouped_by_severity.count - end - - def self.capped_count_by_severity - # Return early when called by `Vulnerabilities::Read.none`. - return {} if current_scope&.null_relation? - - # Handles case when called directly `Vulnerabilities::Read.capped_count_by_severity`. - if current_scope.nil? - severities_to_iterate = severities.keys - local_scope = self - else - severities_to_iterate = Array(current_scope.where_values_hash['severity'].presence || severities.keys) - local_scope = current_scope.unscope(where: :severity) - end - - array_severities_limit = severities_to_iterate.map do |severity| - local_scope.with_severities(severity).select(:id, :severity).limit(SEVERITY_COUNT_LIMIT) - end - - unscoped.from_union(array_severities_limit).count_by_severity - end - - def self.order_by(method) - case method.to_s - when 'severity_desc' then order_severity_desc - when 'severity_asc' then order_severity_asc - when 'detected_desc' then order_detected_at_desc - when 'detected_asc' then order_detected_at_asc - else - order_severity_desc - end - end - - def self.order_by_params_and_traversal_ids(method) - case method.to_s - when 'severity_desc' then order_severity_desc_traversal_ids_desc - when 'severity_asc' then order_severity_asc_traversal_ids_asc - when 'detected_desc' then order_detected_at_desc - when 'detected_asc' then order_detected_at_asc - else - order_severity_desc_traversal_ids_desc - end - end - - def self.container_images - # This method should be used only with pagination. When used without a specific limit, it might try to process an - # unreasonable amount of records leading to a statement timeout. - - # We are enforcing keyset order here to make sure `primary_key` will not be automatically applied when returning - # `ordered_items` from Gitlab::Graphql::Pagination::Keyset::Connection in GraphQL API. `distinct` option must be - # set to true in `Gitlab::Pagination::Keyset::ColumnOrderDefinition` to return the collection in proper order. - - keyset_order = Gitlab::Pagination::Keyset::Order.build( - [ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: :location_image, - column_expression: arel_table[:location_image], - order_expression: arel_table[:location_image].asc - ) - ]) - - where(report_type: [:container_scanning, :cluster_image_scanning]) - .where.not(location_image: nil) - .reorder(keyset_order) - .select(:location_image) - .distinct - end - - def self.fetch_uuids - pluck(:uuid) - end - - def self.generate_es_parent(project) - "group_#{project.namespace.root_ancestor.id}" - end - - def arel_grouping_by_traversal_ids_and_id - self.class.arel_table.grouping([database_serialized_traversal_ids, id]) - end - - def es_parent - self.class.generate_es_parent(project) - end - - def elastic_reference - ::Search::Elastic::References::Vulnerability.serialize(self) - end - - # NOTE: - # 1. For On-premise, post MVC. We may have to honour the setting of skipping indexing for selected projects. Tracked in https://gitlab.com/gitlab-org/gitlab/-/issues/525484 - override :use_elasticsearch? - def use_elasticsearch? - ::Search::Elastic::VulnerabilityIndexingHelper.vulnerability_indexing_allowed? - end - - override :maintain_elasticsearch_update - def maintain_elasticsearch_update(updated_attributes: previous_changes.keys) - super if update_elasticsearch? - end - - private - - def update_elasticsearch? - changed_fields = previous_changes.keys - changed_fields && (changed_fields & ELASTICSEARCH_TRACKED_FIELDS).any? - end - - def database_serialized_traversal_ids - self.class.attribute_types['traversal_ids'] - .serialize(traversal_ids) - .then { |serialized_array| self.class.connection.quote(serialized_array) } - .then { |quoted_array| Arel::Nodes::SqlLiteral.new(quoted_array) } - end - - def set_default_values - self.owasp_top_10 ||= 'undefined' - end - end + class Read < ::SecApplicationRecord + extend EnforceVulnerabilityReadDbTriggerFf + extend ::Gitlab::Utils::Override + include ::Namespaces::Traversal::Traversable + include VulnerabilityScopes + include EachBatch + include UnnestedInFilters::Dsl + include FromUnion + include SafelyChangeColumnDefault + include ::Gitlab::SQL::Pattern + include ::Elastic::ApplicationVersionedSearch + + ignore_column :namespace_id, remove_with: '17.7', remove_after: '2024-11-21' + + declarative_enum DismissalReasonEnum + + # Included after scopes and relationships to avoid the warning + include BulkInsertSafe + + SEVERITY_COUNT_LIMIT = 1001 + OWASP_TOP_10_DEFAULT = -1 + + ELASTICSEARCH_TRACKED_FIELDS = ::Search::Elastic::References::Vulnerability::DIRECT_FIELDS + + ::Search::Elastic::References::Vulnerability::DIRECT_TYPECAST_FIELDS + %w[traversal_ids] + + self.table_name = "vulnerability_reads" + self.primary_key = :vulnerability_id + + columns_changing_default :owasp_top_10 + + delegate :group_name, :title, :created_at, :project_name, :finding_scanner_name, :finding_description, :cve_value, :cwe_value, :location, :notes_summary, :full_path, to: :vulnerability, allow_nil: true + delegate :other_identifier_values, :cvss_vectors_with_vendor, to: :vulnerability, allow_nil: true + delegate :dismissed?, to: :vulnerability + + belongs_to :vulnerability, inverse_of: :vulnerability_read + belongs_to :project + belongs_to :scanner, class_name: 'Vulnerabilities::Scanner' + belongs_to :vulnerability_occurrence, optional: true, class_name: 'Vulnerabilities::Finding' + + validates :vulnerability_id, uniqueness: true, presence: true + validates :project_id, presence: true + validates :scanner_id, presence: true + validates :report_type, presence: true + validates :severity, presence: true + validates :state, presence: true + validates :uuid, presence: true + + validates :location_image, length: { maximum: 2048 } + validates :has_issues, inclusion: { in: [true, false], message: N_('must be a boolean value') } + validates :has_merge_request, inclusion: { in: [true, false], message: N_('must be a boolean value') } + validates :resolved_on_default_branch, inclusion: { in: [true, false], message: N_('must be a boolean value') } + validates :has_remediations, inclusion: { in: [true, false], message: N_('must be a boolean value') } + + enum :state, ::Enums::Vulnerability.vulnerability_states + enum :report_type, ::Enums::Vulnerability.report_types + enum :severity, ::Enums::Vulnerability.severity_levels, prefix: :severity + enum :owasp_top_10, ::Enums::Vulnerability.owasp_top_10.merge('undefined' => OWASP_TOP_10_DEFAULT) + + after_initialize :set_default_values, if: :new_record? + + scope :by_uuid, ->(uuids) { where(uuid: uuids) } + scope :by_vulnerabilities, ->(vulnerabilities) { where(vulnerability: vulnerabilities) } + + class << self + alias_method :by_vulnerability, :by_vulnerabilities + end + + scope :order_severity_asc, -> { reorder(severity: :asc, vulnerability_id: :desc) } + scope :order_severity_desc, -> { reorder(severity: :desc, vulnerability_id: :desc) } + scope :order_detected_at_asc, -> { reorder(vulnerability_id: :asc) } + scope :order_detected_at_desc, -> { reorder(vulnerability_id: :desc) } + + scope :order_severity_asc_traversal_ids_asc, -> { reorder(severity: :asc, traversal_ids: :asc, vulnerability_id: :asc) } + scope :order_severity_desc_traversal_ids_desc, -> { reorder(severity: :desc, traversal_ids: :desc, vulnerability_id: :desc) } + + scope :in_parent_group_after_and_including, ->(vulnerability_read) do + where(arel_grouping_by_traversal_ids_and_vulnerability_id.gteq(vulnerability_read.arel_grouping_by_traversal_ids_and_id)) + end + scope :in_parent_group_before_and_including, ->(vulnerability_read) do + where(arel_grouping_by_traversal_ids_and_vulnerability_id.lteq(vulnerability_read.arel_grouping_by_traversal_ids_and_id)) + end + scope :by_group, ->(group) { within(group.traversal_ids) } + scope :unarchived, -> { where(archived: false) } + scope :order_traversal_ids_asc, -> do + reorder(Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'traversal_ids', + order_expression: arel_table[:traversal_ids].asc, + nullable: :not_nullable + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'vulnerability_id', + order_expression: arel_table[:vulnerability_id].asc + ) + ])) + end + scope :by_projects, ->(values) { where(project_id: values) } + scope :by_scanner, ->(scanner) { where(scanner: scanner) } + scope :by_scanner_ids, ->(scanner_ids) { where(scanner_id: scanner_ids) } + scope :grouped_by_severity, -> { reorder(severity: :desc).group(:severity) } + scope :with_report_types, ->(report_types) { where(report_type: report_types) } + scope :with_severities, ->(severities) { where(severity: severities) } + scope :with_states, ->(states) { where(state: states) } + scope :with_owasp_top_10, ->(owasp_top_10) { where(owasp_top_10: owasp_top_10) } + scope :with_identifier_name, ->(name) do + return none if name.nil? + + where("EXISTS ( + SELECT 1 + FROM unnest(vulnerability_reads.identifier_names) AS idt_names + WHERE idt_names ILIKE ? + )", sanitize_sql_like(name)) + end + scope :with_container_image, ->(images) { where(location_image: images) } + scope :with_container_image_starting_with, ->(image) { where(arel_table[:location_image].matches("#{sanitize_sql_like(image)}%")) } + scope :with_cluster_agent_ids, ->(agent_ids) { where(cluster_agent_id: agent_ids) } + scope :with_resolution, ->(has_resolution = true) { where(resolved_on_default_branch: has_resolution) } + scope :with_ai_resolution, ->(resolution = true) { where(has_vulnerability_resolution: resolution) } + scope :with_issues, ->(has_issues = true) { where(has_issues: has_issues) } + scope :with_merge_request, ->(has_merge_request = true) { where(has_merge_request: has_merge_request) } + scope :with_remediations, ->(has_remediations = true) { where(has_remediations: has_remediations) } + scope :with_scanner_external_ids, + ->(scanner_external_ids) { + joins(:scanner).merge(::Vulnerabilities::Scanner.with_external_id(scanner_external_ids)) + } + scope :with_findings_scanner_and_identifiers, + -> { + includes(vulnerability: { findings: [:scanner, :identifiers, { finding_identifiers: :identifier }] }) + } + scope :preload_indexing_data, -> { + preload( + :scanner, + { vulnerability: [ + { findings: [ + { identifiers: [] }, + { finding_identifiers: :identifier } + ] } + ] }, + { project: { namespace: :route } } + ) + } + scope :resolved_on_default_branch, -> { where('resolved_on_default_branch IS TRUE') } + scope :with_dismissal_reason, ->(dismissal_reason) { where(dismissal_reason: dismissal_reason) } + scope :with_export_entities, -> do + preload( + vulnerability: [ + :group, + { project: [:route], + notes: [:updated_by, :author], + findings: [:scanner, :identifiers] } + ] + ) + end + + scope :as_vulnerabilities, -> do + preload(vulnerability: { project: [:route] }).current_scope.tap do |relation| + relation.define_singleton_method(:records) do + super().map(&:vulnerability) + end + end + end + + scope :by_group_using_nested_loop, ->(group) do + where(traversal_ids: all_vulnerable_traversal_ids_for(group)) + end + + scope :with_findings_scanner_identifiers_and_notes, -> { with_findings_scanner_and_identifiers.includes(vulnerability: :notes) } + scope :with_limit, ->(maximum) { limit(maximum) } + scope :order_id_desc, -> { reorder(arel_table[:vulnerability_id].desc) } + + scope :autocomplete_search, ->(query) do + return self if query.blank? + + id_as_text = Arel::Nodes::NamedFunction.new('CAST', [arel_table[:vulnerability_id].as('TEXT')]) + + joins(:vulnerability) + .select('vulnerability_reads.*, vulnerabilities.title') + .fuzzy_search(query, [Vulnerability.arel_table[:title]]) + .or(where(id_as_text.matches("%#{sanitize_sql_like(query.squish)}%"))) + end + + scope :by_ids_desc, ->(ids) do + by_vulnerability(ids).order_id_desc + end + + def self.es_type + ::Search::Elastic::References::Vulnerability::DOC_TYPE + end + + def self.arel_grouping_by_traversal_ids_and_vulnerability_id + arel_table.grouping([arel_table['traversal_ids'], arel_table['vulnerability_id']]) + end + + def self.all_vulnerable_traversal_ids_for(group) + by_group(group).unarchived.loose_index_scan(column: :traversal_ids) + end + + def self.count_by_severity + grouped_by_severity.count + end + + def self.capped_count_by_severity + # Return early when called by `Vulnerabilities::Read.none`. + return {} if current_scope&.null_relation? + + # Handles case when called directly `Vulnerabilities::Read.capped_count_by_severity`. + if current_scope.nil? + severities_to_iterate = severities.keys + local_scope = self + else + severities_to_iterate = Array(current_scope.where_values_hash['severity'].presence || severities.keys) + local_scope = current_scope.unscope(where: :severity) + end + + array_severities_limit = severities_to_iterate.map do |severity| + local_scope.with_severities(severity).select(:id, :severity).limit(SEVERITY_COUNT_LIMIT) + end + + unscoped.from_union(array_severities_limit).count_by_severity + end + + def self.order_by(method) + case method.to_s + when 'severity_desc' then order_severity_desc + when 'severity_asc' then order_severity_asc + when 'detected_desc' then order_detected_at_desc + when 'detected_asc' then order_detected_at_asc + else + order_severity_desc + end + end + + def self.order_by_params_and_traversal_ids(method) + case method.to_s + when 'severity_desc' then order_severity_desc_traversal_ids_desc + when 'severity_asc' then order_severity_asc_traversal_ids_asc + when 'detected_desc' then order_detected_at_desc + when 'detected_asc' then order_detected_at_asc + else + order_severity_desc_traversal_ids_desc + end + end + + def self.container_images + # This method should be used only with pagination. When used without a specific limit, it might try to process an + # unreasonable amount of records leading to a statement timeout. + + # We are enforcing keyset order here to make sure `primary_key` will not be automatically applied when returning + # `ordered_items` from Gitlab::Graphql::Pagination::Keyset::Connection in GraphQL API. `distinct` option must be + # set to true in `Gitlab::Pagination::Keyset::ColumnOrderDefinition` to return the collection in proper order. + + keyset_order = Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :location_image, + column_expression: arel_table[:location_image], + order_expression: arel_table[:location_image].asc + ) + ]) + + where(report_type: [:container_scanning, :cluster_image_scanning]) + .where.not(location_image: nil) + .reorder(keyset_order) + .select(:location_image) + .distinct + end + + def self.fetch_uuids + pluck(:uuid) + end + + def self.generate_es_parent(project) + "group_#{project.namespace.root_ancestor.id}" + end + + def arel_grouping_by_traversal_ids_and_id + self.class.arel_table.grouping([database_serialized_traversal_ids, id]) + end + + def es_parent + self.class.generate_es_parent(project) + end + + def elastic_reference + ::Search::Elastic::References::Vulnerability.serialize(self) + end + + # NOTE: + # 1. For On-premise, post MVC. We may have to honour the setting of skipping indexing for selected projects. Tracked in https://gitlab.com/gitlab-org/gitlab/-/issues/525484 + override :use_elasticsearch? + def use_elasticsearch? + ::Search::Elastic::VulnerabilityIndexingHelper.vulnerability_indexing_allowed? + end + + override :maintain_elasticsearch_update + def maintain_elasticsearch_update(updated_attributes: previous_changes.keys) + super if update_elasticsearch? + end + + private + + def update_elasticsearch? + changed_fields = previous_changes.keys + changed_fields && (changed_fields & ELASTICSEARCH_TRACKED_FIELDS).any? + end + + def database_serialized_traversal_ids + self.class.attribute_types['traversal_ids'] + .serialize(traversal_ids) + .then { |serialized_array| self.class.connection.quote(serialized_array) } + .then { |quoted_array| Arel::Nodes::SqlLiteral.new(quoted_array) } + end + + def set_default_values + self.owasp_top_10 ||= 'undefined' + end + end end