From 430b00c6d5d79ee5fca7051e501c298547a5919a Mon Sep 17 00:00:00 2001 From: Kai Armstrong Date: Wed, 15 Oct 2025 16:21:05 -0500 Subject: [PATCH] Add SCIP artifact type support for Code Intelligence Adds foundational support for SCIP (SCIP Code Intelligence Protocol) indexes as a new artifact type, enabling native SCIP support for code navigation features. This is Phase 1 of implementing issue https://gitlab.com/gitlab-org/gitlab/-/issues/412981. The artifact type is recognized by Rails, but transformation from SCIP to GitLab's internal JSON format will be implemented in Phase 2 (Workhorse). Changelog: added --- app/models/concerns/enums/ci/job_artifact.rb | 6 ++- .../ci/job_artifacts/create_service.rb | 16 ++++++- .../development/scip_code_intelligence.yml | 7 ++++ ...i_max_artifact_size_scip_to_plan_limits.rb | 9 ++++ db/schema_migrations/20251021091157 | 1 + db/structure.sql | 3 +- doc/api/graphql/reference/_index.md | 1 + lib/gitlab/code_navigation_path.rb | 2 +- spec/lib/gitlab/import_export/all_models.yml | 1 + spec/services/ci/clone_job_service_spec.rb | 2 +- .../ci/job_artifacts/create_service_spec.rb | 42 +++++++++++++++++++ 11 files changed, 85 insertions(+), 5 deletions(-) create mode 100644 config/feature_flags/development/scip_code_intelligence.yml create mode 100644 db/migrate/20251021091157_add_ci_max_artifact_size_scip_to_plan_limits.rb create mode 100644 db/schema_migrations/20251021091157 diff --git a/app/models/concerns/enums/ci/job_artifact.rb b/app/models/concerns/enums/ci/job_artifact.rb index 3b440922ed9505..fc1163c6266f09 100644 --- a/app/models/concerns/enums/ci/job_artifact.rb +++ b/app/models/concerns/enums/ci/job_artifact.rb @@ -36,6 +36,7 @@ module JobArtifact load_performance: 'load-performance.json', metrics: 'metrics.txt', lsif: 'lsif.json', + scip: 'index.scip', dotenv: '.env', cobertura: 'cobertura-coverage.xml', jacoco: 'jacoco-coverage.xml', @@ -66,6 +67,7 @@ module JobArtifact jacoco: :gzip, cluster_applications: :gzip, # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/361094 lsif: :zip, + scip: :zip, cyclonedx: :gzip, annotations: :gzip, repository_xray: :gzip, @@ -109,6 +111,7 @@ module JobArtifact junit license_scanning lsif + scip metrics performance browser_performance @@ -184,7 +187,8 @@ def self.file_type requirements_v2: 29, ## EE-specific annotations: 30, repository_xray: 31, ## EE-specific - jacoco: 32 + jacoco: 32, + scip: 33 # SCIP data for code navigation } end diff --git a/app/services/ci/job_artifacts/create_service.rb b/app/services/ci/job_artifacts/create_service.rb index ae131462e0906d..ae867bf9aae67b 100644 --- a/app/services/ci/job_artifacts/create_service.rb +++ b/app/services/ci/job_artifacts/create_service.rb @@ -6,6 +6,7 @@ class CreateService < ::BaseService include Gitlab::Utils::UsageData LSIF_ARTIFACT_TYPE = 'lsif' + SCIP_ARTIFACT_TYPE = 'scip' OBJECT_STORAGE_ERRORS = [ Errno::EIO, @@ -30,7 +31,7 @@ def authorize(artifact_type:, filesize: nil) final_store_path_config: { root_hash: project.id } ) - if lsif?(artifact_type) + if lsif?(artifact_type) || scip?(artifact_type) headers[:ProcessLsif] = true track_usage_event('i_source_code_code_intelligence', project.id) end @@ -64,6 +65,7 @@ def execute(artifacts_file, params, metadata_file: nil) def validate_requirements(artifact_type:, filesize:) return too_large_error if too_large?(artifact_type, filesize) + return scip_not_enabled_error if scip_not_enabled?(artifact_type) success end @@ -76,14 +78,26 @@ def lsif?(type) type == LSIF_ARTIFACT_TYPE end + def scip?(type) + type == SCIP_ARTIFACT_TYPE + end + def max_size(type) Ci::JobArtifact.max_artifact_size(type: type, project: project) end + def scip_not_enabled?(type) + type == SCIP_ARTIFACT_TYPE && !Feature.enabled?(:scip_code_intelligence, project) + end + def too_large_error error('file size has reached maximum size limit', :payload_too_large) end + def scip_not_enabled_error + error('SCIP artifact type is not enabled', :bad_request) + end + def build_artifact(artifacts_file, params, metadata_file) artifact_attributes = { job: job, diff --git a/config/feature_flags/development/scip_code_intelligence.yml b/config/feature_flags/development/scip_code_intelligence.yml new file mode 100644 index 00000000000000..575eaad410efb4 --- /dev/null +++ b/config/feature_flags/development/scip_code_intelligence.yml @@ -0,0 +1,7 @@ +--- +name: scip_code_intelligence +feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/412981 +milestone: '18.6' +type: development +group: group::code review +default_enabled: false diff --git a/db/migrate/20251021091157_add_ci_max_artifact_size_scip_to_plan_limits.rb b/db/migrate/20251021091157_add_ci_max_artifact_size_scip_to_plan_limits.rb new file mode 100644 index 00000000000000..c01a861d1a23f8 --- /dev/null +++ b/db/migrate/20251021091157_add_ci_max_artifact_size_scip_to_plan_limits.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddCiMaxArtifactSizeScipToPlanLimits < Gitlab::Database::Migration[2.3] + milestone '18.6' + + def change + add_column :plan_limits, :ci_max_artifact_size_scip, :integer, default: 200, null: false + end +end diff --git a/db/schema_migrations/20251021091157 b/db/schema_migrations/20251021091157 new file mode 100644 index 00000000000000..46ba3bf57e0101 --- /dev/null +++ b/db/schema_migrations/20251021091157 @@ -0,0 +1 @@ +c18ccaec16b172077297e9ad6746bfd839aaf6190972b7b98143ae92583e4d35 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 387820a3fa389b..c09d352ffe44b5 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -22802,7 +22802,8 @@ CREATE TABLE plan_limits ( import_placeholder_user_limit_tier_3 integer DEFAULT 0 NOT NULL, import_placeholder_user_limit_tier_4 integer DEFAULT 0 NOT NULL, ci_max_artifact_size_slsa_provenance_statement bigint DEFAULT 0 NOT NULL, - cargo_max_file_size bigint DEFAULT '5368709120'::bigint NOT NULL + cargo_max_file_size bigint DEFAULT '5368709120'::bigint NOT NULL, + ci_max_artifact_size_scip integer DEFAULT 200 NOT NULL ); CREATE SEQUENCE plan_limits_id_seq diff --git a/doc/api/graphql/reference/_index.md b/doc/api/graphql/reference/_index.md index 5577fa57ae3c2b..9f00aa26df9a0d 100644 --- a/doc/api/graphql/reference/_index.md +++ b/doc/api/graphql/reference/_index.md @@ -49848,6 +49848,7 @@ Iteration ID wildcard values. | `REQUIREMENTS` | REQUIREMENTS job artifact file type. | | `REQUIREMENTS_V2` | REQUIREMENTS V2 job artifact file type. | | `SAST` | SAST job artifact file type. | +| `SCIP` | SCIP job artifact file type. | | `SECRET_DETECTION` | SECRET DETECTION job artifact file type. | | `TERRAFORM` | TERRAFORM job artifact file type. | | `TRACE` | TRACE job artifact file type. | diff --git a/lib/gitlab/code_navigation_path.rb b/lib/gitlab/code_navigation_path.rb index a39a25cd392257..acb57d841d55fb 100644 --- a/lib/gitlab/code_navigation_path.rb +++ b/lib/gitlab/code_navigation_path.rb @@ -52,7 +52,7 @@ def build next unless pipeline artifact = Timeout.timeout(ARTIFACT_TIMEOUT) do - pipeline.job_artifacts.with_file_types(['lsif']).last + pipeline.job_artifacts.with_file_types(%w[lsif scip]).last end artifact&.job diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 7aa7b2a7ea4e89..b72535cdb98235 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -481,6 +481,7 @@ builds: - job_artifacts_metrics_referee - job_artifacts_network_referee - job_artifacts_lsif +- job_artifacts_scip - job_artifacts_dotenv - job_artifacts_cobertura - job_artifacts_jacoco diff --git a/spec/services/ci/clone_job_service_spec.rb b/spec/services/ci/clone_job_service_spec.rb index 68816d5554ee1a..4e0f8f40b101ef 100644 --- a/spec/services/ci/clone_job_service_spec.rb +++ b/spec/services/ci/clone_job_service_spec.rb @@ -67,7 +67,7 @@ job_artifacts_container_scanning job_artifacts_cluster_image_scanning job_artifacts_dast job_artifacts_license_scanning job_artifacts_performance job_artifacts_browser_performance job_artifacts_load_performance - job_artifacts_lsif job_artifacts_terraform job_artifacts_cluster_applications + job_artifacts_lsif job_artifacts_scip job_artifacts_terraform job_artifacts_cluster_applications job_artifacts_codequality job_artifacts_metrics scheduled_at job_variables waiting_for_resource_at job_artifacts_metrics_referee job_artifacts_network_referee job_artifacts_dotenv diff --git a/spec/services/ci/job_artifacts/create_service_spec.rb b/spec/services/ci/job_artifacts/create_service_spec.rb index 3912835ffd3912..25c9242261c13d 100644 --- a/spec/services/ci/job_artifacts/create_service_spec.rb +++ b/spec/services/ci/job_artifacts/create_service_spec.rb @@ -39,6 +39,46 @@ end end + shared_examples_for 'handling scip artifact' do + context 'when artifact is scip' do + let(:artifact_type) { 'scip' } + let(:max_artifact_size) { 200.megabytes.to_i } + + before do + allow(Ci::JobArtifact) + .to receive(:max_artifact_size) + .with(type: artifact_type, project: project) + .and_return(max_artifact_size) + end + + context 'when scip_code_intelligence feature flag is enabled' do + before do + stub_feature_flags(scip_code_intelligence: true) + end + + it 'includes ProcessLsif in the headers' do + expect(authorize[:headers][:ProcessLsif]).to eq(true) + end + + it 'returns 200MB in bytes as maximum size' do + expect(authorize[:headers][:MaximumSize]).to eq(200.megabytes.to_i) + end + end + + context 'when scip_code_intelligence feature flag is disabled' do + before do + stub_feature_flags(scip_code_intelligence: false) + end + + it 'returns an error' do + expect(authorize[:status]).to eq(:error) + expect(authorize[:http_status]).to eq(:bad_request) + expect(authorize[:message]).to eq('SCIP artifact type is not enabled') + end + end + end + end + shared_examples_for 'validating requirements' do context 'when filesize is specified' do let(:max_artifact_size) { 10 } @@ -85,6 +125,7 @@ end it_behaves_like 'handling lsif artifact' + it_behaves_like 'handling scip artifact' it_behaves_like 'validating requirements' end @@ -118,6 +159,7 @@ end it_behaves_like 'handling lsif artifact' + it_behaves_like 'handling scip artifact' it_behaves_like 'validating requirements' end -- GitLab