From 5eda149cfba285e162e710d94011067f187bca69 Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Tue, 1 Jul 2025 13:43:13 +0200 Subject: [PATCH 01/19] Add Gitlab::Utils::MimeType#from_filename --- lib/gitlab/utils/mime_type.rb | 11 ++++ spec/lib/gitlab/utils/mime_type_spec.rb | 78 +++++++++++++++++-------- 2 files changed, 66 insertions(+), 23 deletions(-) diff --git a/lib/gitlab/utils/mime_type.rb b/lib/gitlab/utils/mime_type.rb index d82092a2d2c12d..089be3fec7a838 100644 --- a/lib/gitlab/utils/mime_type.rb +++ b/lib/gitlab/utils/mime_type.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true require 'magic' +require 'mime/types' # This wraps calls to a gem which support mime type detection. # We use the `ruby-magic` gem instead of `mimemagic` due to licensing issues @@ -19,6 +20,16 @@ def from_string(string) string.type end + + def from_filename(filename, default: 'application/octet-stream') + return default unless filename.is_a?(String) + + types = ::MIME::Types.type_for(filename) + + return default unless types.present? + + types.first.content_type + end end end end diff --git a/spec/lib/gitlab/utils/mime_type_spec.rb b/spec/lib/gitlab/utils/mime_type_spec.rb index e8b369362243cc..a8473ea46bef94 100644 --- a/spec/lib/gitlab/utils/mime_type_spec.rb +++ b/spec/lib/gitlab/utils/mime_type_spec.rb @@ -1,57 +1,89 @@ # frozen_string_literal: true -require "fast_spec_helper" -require "rspec/parameterized" +require 'fast_spec_helper' +require 'rspec/parameterized' RSpec.describe Gitlab::Utils::MimeType do - describe ".from_io" do + describe '.from_io' do subject { described_class.from_io(io) } - context "input isn't an IO" do - let(:io) { "test" } + context 'input is not an IO' do + let(:io) { 'test' } - it "returns nil" do + it 'returns nil' do expect(subject).to be_nil end end - context "input is a file" do + context 'input is a file' do using RSpec::Parameterized::TableSyntax where(:fixture, :mime_type) do - "banana_sample.gif" | "image/gif" - "rails_sample.jpg" | "image/jpeg" - "rails_sample.png" | "image/png" - "rails_sample.bmp" | "image/bmp" - "rails_sample.tif" | "image/tiff" - "sample.ico" | "image/vnd.microsoft.icon" - "sample_doc.md" | "text/plain" - "csv_empty.csv" | "application/x-empty" + 'banana_sample.gif' | 'image/gif' + 'rails_sample.jpg' | 'image/jpeg' + 'rails_sample.png' | 'image/png' + 'rails_sample.bmp' | 'image/bmp' + 'rails_sample.tif' | 'image/tiff' + 'sample.ico' | 'image/vnd.microsoft.icon' + 'sample_doc.md' | 'text/plain' + 'csv_empty.csv' | 'application/x-empty' end with_them do - let(:io) { File.open(File.join(__dir__, "../../../fixtures", fixture)) } + let(:io) { File.open(File.join(__dir__, '../../../fixtures', fixture)) } it { is_expected.to eq(mime_type) } end end end - describe ".from_string" do + describe '.from_string' do subject { described_class.from_string(str) } - context "input isn't a string" do + context 'input is not a string' do let(:str) { nil } - it "returns nil" do - expect(subject).to be_nil - end + it { is_expected.to be_nil } end - context "input is a string" do - let(:str) { "plain text" } + context 'input is a string' do + let(:str) { 'plain text' } it { is_expected.to eq('text/plain') } end end + + describe '.from_filename' do + using RSpec::Parameterized::TableSyntax + + context 'when default value is not given' do + where(:filename, :mime_type) do + 1 | 'application/octet-stream' + 'test.tf' | 'application/octet-stream' + 'test.css' | 'text/css' + 'test.js' | 'text/javascript' + end + + with_them do + subject { described_class.from_filename(filename) } + + it { is_expected.to eq(mime_type) } + end + end + + context 'when default value is given' do + where(:filename, :default, :mime_type) do + 1 | nil | nil + 'test.tf' | nil | nil + 'test.tf' | 'text/plain' | 'text/plain' + 'test.css' | 'text/plain' | 'text/css' + end + + with_them do + subject { described_class.from_filename(filename, default: default) } + + it { is_expected.to eq(mime_type) } + end + end + end end -- GitLab From 39aa82629b6c4df2bd07dd7e7e14078614b9511d Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Tue, 1 Jul 2025 13:46:14 +0200 Subject: [PATCH 02/19] Use new method in SendFileUpload concern --- app/controllers/concerns/send_file_upload.rb | 12 +----------- app/controllers/projects/jobs_controller.rb | 2 +- app/services/lfs/file_transformer.rb | 2 +- app/uploaders/content_type_whitelist.rb | 2 +- lib/gitlab/sanitizers/exif.rb | 2 +- 5 files changed, 5 insertions(+), 15 deletions(-) diff --git a/app/controllers/concerns/send_file_upload.rb b/app/controllers/concerns/send_file_upload.rb index 5e44679dbfa8ff..0266adcee4f302 100644 --- a/app/controllers/concerns/send_file_upload.rb +++ b/app/controllers/concerns/send_file_upload.rb @@ -44,17 +44,7 @@ def send_upload( def content_type_for(attachment) return '' unless attachment - guess_content_type(attachment) - end - - def guess_content_type(filename) - types = MIME::Types.type_for(filename) - - if types.present? - types.first.content_type - else - "application/octet-stream" - end + ::Gitlab::Utils::MimeType.from_filename(attachment) end private diff --git a/app/controllers/projects/jobs_controller.rb b/app/controllers/projects/jobs_controller.rb index 43a09150b70ea3..75eca566b4e802 100644 --- a/app/controllers/projects/jobs_controller.rb +++ b/app/controllers/projects/jobs_controller.rb @@ -265,7 +265,7 @@ def build_path(build) end def raw_trace_content_disposition(raw_data) - mime_type = Gitlab::Utils::MimeType.from_string(raw_data) + mime_type = ::Gitlab::Utils::MimeType.from_string(raw_data) # if mime_type is nil can also represent 'text/plain' return 'inline' if mime_type.nil? || mime_type == 'text/plain' diff --git a/app/services/lfs/file_transformer.rb b/app/services/lfs/file_transformer.rb index ef227b6e20f732..477c17dcecea0d 100644 --- a/app/services/lfs/file_transformer.rb +++ b/app/services/lfs/file_transformer.rb @@ -72,7 +72,7 @@ def cached_attributes # rubocop: disable CodeReuse/ActiveRecord def create_lfs_object!(lfs_pointer_file, file_content, detect_content_type) LfsObject.find_or_create_by(oid: lfs_pointer_file.sha256, size: lfs_pointer_file.size) do |lfs_object| - lfs_object.file = if detect_content_type && (content_type = Gitlab::Utils::MimeType.from_string(file_content)) + lfs_object.file = if detect_content_type && (content_type = ::Gitlab::Utils::MimeType.from_string(file_content)) CarrierWaveStringFile.new_file( file_content: file_content, filename: '', diff --git a/app/uploaders/content_type_whitelist.rb b/app/uploaders/content_type_whitelist.rb index 82c6b9b3a6114d..4c9d180dab4966 100644 --- a/app/uploaders/content_type_whitelist.rb +++ b/app/uploaders/content_type_whitelist.rb @@ -43,7 +43,7 @@ def whitelisted_content_type?(content_type) def mime_magic_content_type(path) if path File.open(path) do |file| - Gitlab::Utils::MimeType.from_io(file) || 'invalid/invalid' + ::Gitlab::Utils::MimeType.from_io(file) || 'invalid/invalid' end end rescue Errno::ENOENT diff --git a/lib/gitlab/sanitizers/exif.rb b/lib/gitlab/sanitizers/exif.rb index dee9e5f3d0e42f..08fc9f90ac0ed9 100644 --- a/lib/gitlab/sanitizers/exif.rb +++ b/lib/gitlab/sanitizers/exif.rb @@ -169,7 +169,7 @@ def fetch_upload_to_file(uploader, dir) end def check_for_allowed_types(contents, raise_error: true) - mime_type = Gitlab::Utils::MimeType.from_string(contents) + mime_type = ::Gitlab::Utils::MimeType.from_string(contents) allowed = ALLOWED_MIME_TYPES.include?(mime_type) if !allowed && raise_error -- GitLab From 133a76105b094e35a79f78da5ba50ede8b2bba7f Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Tue, 1 Jul 2025 17:40:15 +0200 Subject: [PATCH 03/19] Use AppJsonLogger to log determined_mime_type_from_filename --- lib/gitlab/utils/mime_type.rb | 13 +++++-- spec/lib/gitlab/utils/mime_type_spec.rb | 52 +++++++++++++++++++------ 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/lib/gitlab/utils/mime_type.rb b/lib/gitlab/utils/mime_type.rb index 089be3fec7a838..f5ea1dce50890b 100644 --- a/lib/gitlab/utils/mime_type.rb +++ b/lib/gitlab/utils/mime_type.rb @@ -21,14 +21,21 @@ def from_string(string) string.type end - def from_filename(filename, default: 'application/octet-stream') + def from_filename(filename, called_from:, default: 'application/octet-stream') return default unless filename.is_a?(String) types = ::MIME::Types.type_for(filename) + content_type = types.any? ? types.first.content_type : default - return default unless types.present? + # Temporarily log the detected content types as JSON to build the allowed content type list + # https://gitlab.com/gitlab-org/gitlab/-/issues/444768 + ::Gitlab::AppJsonLogger.info( + message: 'determined_mime_type_from_filename', + determined_content_type: content_type, + called_from: called_from + ) - types.first.content_type + content_type end end end diff --git a/spec/lib/gitlab/utils/mime_type_spec.rb b/spec/lib/gitlab/utils/mime_type_spec.rb index a8473ea46bef94..4e8076330cec1b 100644 --- a/spec/lib/gitlab/utils/mime_type_spec.rb +++ b/spec/lib/gitlab/utils/mime_type_spec.rb @@ -56,33 +56,61 @@ describe '.from_filename' do using RSpec::Parameterized::TableSyntax + before do + allow(Gitlab::AppJsonLogger).to receive(:info) + end + + shared_examples 'log determination' do + it 'logs the determination' do + subject + + expect(Gitlab::AppJsonLogger).to have_received(:info).with( + message: 'determined_mime_type_from_filename', + determined_content_type: mime_type, + called_from: 'SendFileUpload' + ) + end + end + + shared_examples 'not log determination' do + it 'does not log the determination' do + subject + + expect(Gitlab::AppJsonLogger).not_to have_received(:info) + end + end + context 'when default value is not given' do - where(:filename, :mime_type) do - 1 | 'application/octet-stream' - 'test.tf' | 'application/octet-stream' - 'test.css' | 'text/css' - 'test.js' | 'text/javascript' + where(:filename, :mime_type, :log_example) do + 1 | 'application/octet-stream' | 'not log determination' + 'test.tf' | 'application/octet-stream' | 'log determination' + 'test.css' | 'text/css' | 'log determination' + 'test.js' | 'text/javascript' | 'log determination' end with_them do - subject { described_class.from_filename(filename) } + subject { described_class.from_filename(filename, called_from: 'SendFileUpload') } it { is_expected.to eq(mime_type) } + + it_behaves_like params[:log_example] end end context 'when default value is given' do - where(:filename, :default, :mime_type) do - 1 | nil | nil - 'test.tf' | nil | nil - 'test.tf' | 'text/plain' | 'text/plain' - 'test.css' | 'text/plain' | 'text/css' + where(:filename, :default, :mime_type, :log_example) do + 1 | nil | nil | 'not log determination' + 'test.tf' | nil | nil | 'log determination' + 'test.tf' | 'text/plain' | 'text/plain' | 'log determination' + 'test.css' | 'text/plain' | 'text/css' | 'log determination' end with_them do - subject { described_class.from_filename(filename, default: default) } + subject { described_class.from_filename(filename, called_from: 'SendFileUpload', default: default) } it { is_expected.to eq(mime_type) } + + it_behaves_like params[:log_example] end end end -- GitLab From a0b427004b047d0d241eff213461e48e66597ff6 Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Tue, 1 Jul 2025 17:43:32 +0200 Subject: [PATCH 04/19] Send called_from in SendFileUpload --- app/controllers/concerns/send_file_upload.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/concerns/send_file_upload.rb b/app/controllers/concerns/send_file_upload.rb index 0266adcee4f302..1286af9b15b286 100644 --- a/app/controllers/concerns/send_file_upload.rb +++ b/app/controllers/concerns/send_file_upload.rb @@ -44,7 +44,7 @@ def send_upload( def content_type_for(attachment) return '' unless attachment - ::Gitlab::Utils::MimeType.from_filename(attachment) + ::Gitlab::Utils::MimeType.from_filename(attachment, called_from: 'SendFileUpload') end private -- GitLab From ac069ed8f80e97928dd79827a5d89130f83e389c Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Tue, 1 Jul 2025 18:08:29 +0200 Subject: [PATCH 05/19] Start determining content type in API::GenericPackages --- lib/api/generic_packages.rb | 5 +++++ spec/requests/api/generic_packages_spec.rb | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index f77750cdfb1a71..d4b79355240436 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -147,6 +147,7 @@ class GenericPackages < ::API::Base package = ::Packages::Generic::PackageFinder.new(project).execute!(params[:package_name], params[:package_version]) package_file = ::Packages::PackageFileFinder.new(package, encoded_file_name).execute! + determine_content_type(package_file) track_package_event('pull_package', :generic, project: project, namespace: project.namespace) present_package_file!(package_file, content_disposition: :attachment) @@ -177,6 +178,10 @@ def encoded_file_name file_name = [declared_params[:path], declared_params[:file_name]].compact.join('/') declared_params[:path].present? ? URI.encode_uri_component(file_name) : file_name end + + def determine_content_type(package_file) + ::Gitlab::Utils::MimeType.from_filename(package_file.file_name, called_from: 'API::GenericPackages') + end end end end diff --git a/spec/requests/api/generic_packages_spec.rb b/spec/requests/api/generic_packages_spec.rb index d08fdbd4c2825e..360dcece0bce7a 100644 --- a/spec/requests/api/generic_packages_spec.rb +++ b/spec/requests/api/generic_packages_spec.rb @@ -1121,6 +1121,20 @@ def upload_file( end end + shared_examples 'log content type determination' do + it 'logs content type determination' do + allow(Gitlab::AppJsonLogger).to receive(:info) + + subject + + expect(Gitlab::AppJsonLogger).to have_received(:info).with( + message: 'determined_mime_type_from_filename', + determined_content_type: 'application/gzip', + called_from: 'API::GenericPackages' + ) + end + end + context 'when object storage is enabled' do let(:package_file) { create(:package_file, :generic, :object_storage, package: package) } @@ -1139,6 +1153,8 @@ def upload_file( stub_package_file_object_storage end + it_behaves_like 'log content type determination' + it 'includes response-content-disposition and filename in the redirect file URL' do download @@ -1167,6 +1183,8 @@ def upload_file( stub_package_file_object_storage(proxy_download: true) end + it_behaves_like 'log content type determination' + it 'sends a file with response-content-disposition and filename' do expect(::Gitlab::Workhorse).to receive(:send_url) .with(instance_of(String), expected_headers) -- GitLab From fe862e82707d3d8634fe42c3bcef0b1afbbbce23 Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Wed, 9 Jul 2025 12:54:26 +0200 Subject: [PATCH 06/19] Remove unnecessary called_from parameter --- app/controllers/concerns/send_file_upload.rb | 2 +- lib/api/generic_packages.rb | 2 +- lib/gitlab/utils/mime_type.rb | 5 ++--- spec/lib/gitlab/utils/mime_type_spec.rb | 7 +++---- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/app/controllers/concerns/send_file_upload.rb b/app/controllers/concerns/send_file_upload.rb index 1286af9b15b286..0266adcee4f302 100644 --- a/app/controllers/concerns/send_file_upload.rb +++ b/app/controllers/concerns/send_file_upload.rb @@ -44,7 +44,7 @@ def send_upload( def content_type_for(attachment) return '' unless attachment - ::Gitlab::Utils::MimeType.from_filename(attachment, called_from: 'SendFileUpload') + ::Gitlab::Utils::MimeType.from_filename(attachment) end private diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index d4b79355240436..97e252e64a2cb7 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -180,7 +180,7 @@ def encoded_file_name end def determine_content_type(package_file) - ::Gitlab::Utils::MimeType.from_filename(package_file.file_name, called_from: 'API::GenericPackages') + ::Gitlab::Utils::MimeType.from_filename(package_file.file_name) end end end diff --git a/lib/gitlab/utils/mime_type.rb b/lib/gitlab/utils/mime_type.rb index f5ea1dce50890b..534b2f84b1bd92 100644 --- a/lib/gitlab/utils/mime_type.rb +++ b/lib/gitlab/utils/mime_type.rb @@ -21,7 +21,7 @@ def from_string(string) string.type end - def from_filename(filename, called_from:, default: 'application/octet-stream') + def from_filename(filename, default: 'application/octet-stream') return default unless filename.is_a?(String) types = ::MIME::Types.type_for(filename) @@ -31,8 +31,7 @@ def from_filename(filename, called_from:, default: 'application/octet-stream') # https://gitlab.com/gitlab-org/gitlab/-/issues/444768 ::Gitlab::AppJsonLogger.info( message: 'determined_mime_type_from_filename', - determined_content_type: content_type, - called_from: called_from + determined_content_type: content_type ) content_type diff --git a/spec/lib/gitlab/utils/mime_type_spec.rb b/spec/lib/gitlab/utils/mime_type_spec.rb index 4e8076330cec1b..955879d8fd5a7a 100644 --- a/spec/lib/gitlab/utils/mime_type_spec.rb +++ b/spec/lib/gitlab/utils/mime_type_spec.rb @@ -66,8 +66,7 @@ expect(Gitlab::AppJsonLogger).to have_received(:info).with( message: 'determined_mime_type_from_filename', - determined_content_type: mime_type, - called_from: 'SendFileUpload' + determined_content_type: mime_type ) end end @@ -89,7 +88,7 @@ end with_them do - subject { described_class.from_filename(filename, called_from: 'SendFileUpload') } + subject { described_class.from_filename(filename) } it { is_expected.to eq(mime_type) } @@ -106,7 +105,7 @@ end with_them do - subject { described_class.from_filename(filename, called_from: 'SendFileUpload', default: default) } + subject { described_class.from_filename(filename, default: default) } it { is_expected.to eq(mime_type) } -- GitLab From 9d11bda992ed12024be6062ea2a165bc861f11ab Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Wed, 9 Jul 2025 15:17:31 +0200 Subject: [PATCH 07/19] Add log_enabled to only log for generic package --- app/controllers/concerns/send_file_upload.rb | 8 ++--- .../packages/package_files_controller.rb | 4 ++- lib/api/generic_packages.rb | 2 +- lib/gitlab/utils/mime_type.rb | 16 ++++++---- spec/lib/gitlab/utils/mime_type_spec.rb | 32 ++++++++++++------- spec/requests/api/generic_packages_spec.rb | 3 +- .../packages/package_files_controller_spec.rb | 25 +++++++++++++++ 7 files changed, 63 insertions(+), 27 deletions(-) diff --git a/app/controllers/concerns/send_file_upload.rb b/app/controllers/concerns/send_file_upload.rb index 0266adcee4f302..634255dff2757f 100644 --- a/app/controllers/concerns/send_file_upload.rb +++ b/app/controllers/concerns/send_file_upload.rb @@ -3,8 +3,8 @@ module SendFileUpload def send_upload( file_upload, send_params: {}, redirect_params: {}, attachment: nil, proxy: false, - disposition: 'attachment', ssrf_params: {}) - content_type = content_type_for(attachment) + disposition: 'attachment', ssrf_params: {}, log_enabled: false) + content_type = content_type_for(attachment, log_enabled) if attachment response_disposition = ActionDispatch::Http::ContentDisposition.format(disposition: disposition, @@ -41,10 +41,10 @@ def send_upload( end end - def content_type_for(attachment) + def content_type_for(attachment, log_enabled) return '' unless attachment - ::Gitlab::Utils::MimeType.from_filename(attachment) + ::Gitlab::Utils::MimeType.from_filename(attachment, log_enabled: log_enabled) end private diff --git a/app/controllers/projects/packages/package_files_controller.rb b/app/controllers/projects/packages/package_files_controller.rb index 0b3ab1ae1ad999..5e13b482868d22 100644 --- a/app/controllers/projects/packages/package_files_controller.rb +++ b/app/controllers/projects/packages/package_files_controller.rb @@ -13,8 +13,10 @@ def download package_file.package.touch_last_downloaded_at + log_enabled = package_file.package.generic? + send_upload(package_file.file, attachment: package_file.file_name_for_download, - ssrf_params: ::Packages::SsrfProtection.params_for(package_file.package)) + ssrf_params: ::Packages::SsrfProtection.params_for(package_file.package), log_enabled: log_enabled) end end end diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index 97e252e64a2cb7..6194958e75a2a5 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -180,7 +180,7 @@ def encoded_file_name end def determine_content_type(package_file) - ::Gitlab::Utils::MimeType.from_filename(package_file.file_name) + ::Gitlab::Utils::MimeType.from_filename(package_file.file_name, log_enabled: true) end end end diff --git a/lib/gitlab/utils/mime_type.rb b/lib/gitlab/utils/mime_type.rb index 534b2f84b1bd92..07cae32c4ce558 100644 --- a/lib/gitlab/utils/mime_type.rb +++ b/lib/gitlab/utils/mime_type.rb @@ -21,18 +21,20 @@ def from_string(string) string.type end - def from_filename(filename, default: 'application/octet-stream') + def from_filename(filename, default: 'application/octet-stream', log_enabled: false) return default unless filename.is_a?(String) types = ::MIME::Types.type_for(filename) content_type = types.any? ? types.first.content_type : default - # Temporarily log the detected content types as JSON to build the allowed content type list - # https://gitlab.com/gitlab-org/gitlab/-/issues/444768 - ::Gitlab::AppJsonLogger.info( - message: 'determined_mime_type_from_filename', - determined_content_type: content_type - ) + if log_enabled + # Temporarily log the detected content types as JSON to build the allowed content type list + # https://gitlab.com/gitlab-org/gitlab/-/issues/444768 + ::Gitlab::AppJsonLogger.info( + message: 'determined_mime_type_from_filename', + determined_content_type: content_type + ) + end content_type end diff --git a/spec/lib/gitlab/utils/mime_type_spec.rb b/spec/lib/gitlab/utils/mime_type_spec.rb index 955879d8fd5a7a..dc6b4597c6f6bf 100644 --- a/spec/lib/gitlab/utils/mime_type_spec.rb +++ b/spec/lib/gitlab/utils/mime_type_spec.rb @@ -80,15 +80,19 @@ end context 'when default value is not given' do - where(:filename, :mime_type, :log_example) do - 1 | 'application/octet-stream' | 'not log determination' - 'test.tf' | 'application/octet-stream' | 'log determination' - 'test.css' | 'text/css' | 'log determination' - 'test.js' | 'text/javascript' | 'log determination' + where(:filename, :mime_type, :log_enabled, :log_example) do + 1 | 'application/octet-stream' | false | 'not log determination' + 'test.tf' | 'application/octet-stream' | false | 'not log determination' + 'test.css' | 'text/css' | false | 'not log determination' + 'test.js' | 'text/javascript' | false | 'not log determination' + 1 | 'application/octet-stream' | true | 'not log determination' + 'test.tf' | 'application/octet-stream' | true | 'log determination' + 'test.css' | 'text/css' | true | 'log determination' + 'test.js' | 'text/javascript' | true | 'log determination' end with_them do - subject { described_class.from_filename(filename) } + subject { described_class.from_filename(filename, log_enabled: log_enabled) } it { is_expected.to eq(mime_type) } @@ -97,15 +101,19 @@ end context 'when default value is given' do - where(:filename, :default, :mime_type, :log_example) do - 1 | nil | nil | 'not log determination' - 'test.tf' | nil | nil | 'log determination' - 'test.tf' | 'text/plain' | 'text/plain' | 'log determination' - 'test.css' | 'text/plain' | 'text/css' | 'log determination' + where(:filename, :default, :mime_type, :log_enabled, :log_example) do + 1 | nil | nil | false | 'not log determination' + 'test.tf' | nil | nil | false | 'not log determination' + 'test.tf' | 'text/plain' | 'text/plain' | false | 'not log determination' + 'test.css' | 'text/plain' | 'text/css' | false | 'not log determination' + 1 | nil | nil | true | 'not log determination' + 'test.tf' | nil | nil | true | 'log determination' + 'test.tf' | 'text/plain' | 'text/plain' | true | 'log determination' + 'test.css' | 'text/plain' | 'text/css' | true | 'log determination' end with_them do - subject { described_class.from_filename(filename, default: default) } + subject { described_class.from_filename(filename, default: default, log_enabled: log_enabled) } it { is_expected.to eq(mime_type) } diff --git a/spec/requests/api/generic_packages_spec.rb b/spec/requests/api/generic_packages_spec.rb index 360dcece0bce7a..cbf699aca706c5 100644 --- a/spec/requests/api/generic_packages_spec.rb +++ b/spec/requests/api/generic_packages_spec.rb @@ -1129,8 +1129,7 @@ def upload_file( expect(Gitlab::AppJsonLogger).to have_received(:info).with( message: 'determined_mime_type_from_filename', - determined_content_type: 'application/gzip', - called_from: 'API::GenericPackages' + determined_content_type: 'application/gzip' ) end end diff --git a/spec/requests/projects/packages/package_files_controller_spec.rb b/spec/requests/projects/packages/package_files_controller_spec.rb index a68866de8931c5..e65f7a41975253 100644 --- a/spec/requests/projects/packages/package_files_controller_spec.rb +++ b/spec/requests/projects/packages/package_files_controller_spec.rb @@ -18,6 +18,10 @@ ) end + before do + allow(Gitlab::AppJsonLogger).to receive(:info) + end + it 'sends the package file' do subject @@ -25,6 +29,27 @@ .to eq(%(attachment; filename="#{filename}"; filename*=UTF-8''#{filename})) end + it 'logs content type determination' do + subject + + expect(Gitlab::AppJsonLogger).to have_received(:info).with( + message: 'determined_mime_type_from_filename', + determined_content_type: 'application/zip' + ) + end + + context 'when package type is not generic' do + before do + package.update!(package_type: 'npm') + end + + it 'does not enable log' do + subject + + expect(Gitlab::AppJsonLogger).not_to have_received(:info) + end + end + context 'with remote object storage' do let(:package_file) { create(:package_file, :object_storage, package: package, file_name: filename) } -- GitLab From 05a6612dd595391fbf2cf924e2fb32739a22fc7e Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Wed, 9 Jul 2025 15:21:40 +0200 Subject: [PATCH 08/19] Remove unnecessary message --- lib/gitlab/utils/mime_type.rb | 5 +---- spec/lib/gitlab/utils/mime_type_spec.rb | 1 - spec/requests/api/generic_packages_spec.rb | 1 - .../projects/packages/package_files_controller_spec.rb | 1 - 4 files changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/gitlab/utils/mime_type.rb b/lib/gitlab/utils/mime_type.rb index 07cae32c4ce558..f8b1b0083d793d 100644 --- a/lib/gitlab/utils/mime_type.rb +++ b/lib/gitlab/utils/mime_type.rb @@ -30,10 +30,7 @@ def from_filename(filename, default: 'application/octet-stream', log_enabled: fa if log_enabled # Temporarily log the detected content types as JSON to build the allowed content type list # https://gitlab.com/gitlab-org/gitlab/-/issues/444768 - ::Gitlab::AppJsonLogger.info( - message: 'determined_mime_type_from_filename', - determined_content_type: content_type - ) + ::Gitlab::AppJsonLogger.info(determined_content_type: content_type) end content_type diff --git a/spec/lib/gitlab/utils/mime_type_spec.rb b/spec/lib/gitlab/utils/mime_type_spec.rb index dc6b4597c6f6bf..192e985c240e1c 100644 --- a/spec/lib/gitlab/utils/mime_type_spec.rb +++ b/spec/lib/gitlab/utils/mime_type_spec.rb @@ -65,7 +65,6 @@ subject expect(Gitlab::AppJsonLogger).to have_received(:info).with( - message: 'determined_mime_type_from_filename', determined_content_type: mime_type ) end diff --git a/spec/requests/api/generic_packages_spec.rb b/spec/requests/api/generic_packages_spec.rb index cbf699aca706c5..eb2bcc55ef5d57 100644 --- a/spec/requests/api/generic_packages_spec.rb +++ b/spec/requests/api/generic_packages_spec.rb @@ -1128,7 +1128,6 @@ def upload_file( subject expect(Gitlab::AppJsonLogger).to have_received(:info).with( - message: 'determined_mime_type_from_filename', determined_content_type: 'application/gzip' ) end diff --git a/spec/requests/projects/packages/package_files_controller_spec.rb b/spec/requests/projects/packages/package_files_controller_spec.rb index e65f7a41975253..d5e2e5f14be2b4 100644 --- a/spec/requests/projects/packages/package_files_controller_spec.rb +++ b/spec/requests/projects/packages/package_files_controller_spec.rb @@ -33,7 +33,6 @@ subject expect(Gitlab::AppJsonLogger).to have_received(:info).with( - message: 'determined_mime_type_from_filename', determined_content_type: 'application/zip' ) end -- GitLab From 979a62d93b6ae25cdbd2bf0634bffcc52a86a452 Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Thu, 3 Jul 2025 18:07:45 +0200 Subject: [PATCH 09/19] Add feature flag packages_generic_package_content_type --- .../packages_generic_package_content_type.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 config/feature_flags/gitlab_com_derisk/packages_generic_package_content_type.yml diff --git a/config/feature_flags/gitlab_com_derisk/packages_generic_package_content_type.yml b/config/feature_flags/gitlab_com_derisk/packages_generic_package_content_type.yml new file mode 100644 index 00000000000000..739f3aee582b57 --- /dev/null +++ b/config/feature_flags/gitlab_com_derisk/packages_generic_package_content_type.yml @@ -0,0 +1,10 @@ +--- +name: packages_generic_package_content_type +description: Return generic package file with corresponding content type +feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/444768 +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/196427 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/547424 +milestone: '18.3' +group: group::package registry +type: gitlab_com_derisk +default_enabled: false -- GitLab From b67fe0a737c0729b49d05b6b9e06f965f2094630 Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Wed, 9 Jul 2025 15:57:10 +0200 Subject: [PATCH 10/19] Apply feature flag to api and controller --- .../packages/package_files_controller.rb | 2 +- lib/api/generic_packages.rb | 2 ++ spec/requests/api/generic_packages_spec.rb | 14 ++++++++++++++ .../packages/package_files_controller_spec.rb | 18 +++++++++++++++--- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/app/controllers/projects/packages/package_files_controller.rb b/app/controllers/projects/packages/package_files_controller.rb index 5e13b482868d22..e8de61f9b98543 100644 --- a/app/controllers/projects/packages/package_files_controller.rb +++ b/app/controllers/projects/packages/package_files_controller.rb @@ -13,7 +13,7 @@ def download package_file.package.touch_last_downloaded_at - log_enabled = package_file.package.generic? + log_enabled = package_file.package.generic? && Feature.enabled?(:packages_generic_package_content_type, project) send_upload(package_file.file, attachment: package_file.file_name_for_download, ssrf_params: ::Packages::SsrfProtection.params_for(package_file.package), log_enabled: log_enabled) diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index 6194958e75a2a5..d56fa0953fb2ed 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -180,6 +180,8 @@ def encoded_file_name end def determine_content_type(package_file) + return unless Feature.enabled?(:packages_generic_package_content_type, package_file.package.project) + ::Gitlab::Utils::MimeType.from_filename(package_file.file_name, log_enabled: true) end end diff --git a/spec/requests/api/generic_packages_spec.rb b/spec/requests/api/generic_packages_spec.rb index eb2bcc55ef5d57..93d01406053062 100644 --- a/spec/requests/api/generic_packages_spec.rb +++ b/spec/requests/api/generic_packages_spec.rb @@ -1131,6 +1131,20 @@ def upload_file( determined_content_type: 'application/gzip' ) end + + context 'when feature flag is disabled' do + before do + stub_feature_flags(packages_generic_package_content_type: false) + end + + it 'does not log content type determination' do + allow(Gitlab::AppJsonLogger).to receive(:info) + + subject + + expect(Gitlab::AppJsonLogger).not_to have_received(:info) + end + end end context 'when object storage is enabled' do diff --git a/spec/requests/projects/packages/package_files_controller_spec.rb b/spec/requests/projects/packages/package_files_controller_spec.rb index d5e2e5f14be2b4..3e860b69ebf56b 100644 --- a/spec/requests/projects/packages/package_files_controller_spec.rb +++ b/spec/requests/projects/packages/package_files_controller_spec.rb @@ -37,16 +37,28 @@ ) end + shared_examples 'not log' do + it 'does not enable log' do + subject + + expect(Gitlab::AppJsonLogger).not_to have_received(:info) + end + end + context 'when package type is not generic' do before do package.update!(package_type: 'npm') end - it 'does not enable log' do - subject + it_behaves_like 'not log' + end - expect(Gitlab::AppJsonLogger).not_to have_received(:info) + context 'when feature flag is disabled' do + before do + stub_feature_flags(packages_generic_package_content_type: false) end + + it_behaves_like 'not log' end context 'with remote object storage' do -- GitLab From 1aa7b16461db1deb9e621e11039b66379cae3056 Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Thu, 10 Jul 2025 12:07:43 +0200 Subject: [PATCH 11/19] Use delegate #project method --- lib/api/generic_packages.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index d56fa0953fb2ed..309394eb1fe375 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -180,7 +180,7 @@ def encoded_file_name end def determine_content_type(package_file) - return unless Feature.enabled?(:packages_generic_package_content_type, package_file.package.project) + return unless Feature.enabled?(:packages_generic_package_content_type, package_file.project) ::Gitlab::Utils::MimeType.from_filename(package_file.file_name, log_enabled: true) end -- GitLab From a9b92a05687f852170ea66f0d147cf65892dc9fc Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Thu, 10 Jul 2025 12:09:04 +0200 Subject: [PATCH 12/19] Add comment about cleaning up log_enabled parameter --- lib/gitlab/utils/mime_type.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/gitlab/utils/mime_type.rb b/lib/gitlab/utils/mime_type.rb index f8b1b0083d793d..9a2ec61dfa64e4 100644 --- a/lib/gitlab/utils/mime_type.rb +++ b/lib/gitlab/utils/mime_type.rb @@ -29,6 +29,7 @@ def from_filename(filename, default: 'application/octet-stream', log_enabled: fa if log_enabled # Temporarily log the detected content types as JSON to build the allowed content type list + # When we remove this line, `log_enabled` needs to be removed as well # https://gitlab.com/gitlab-org/gitlab/-/issues/444768 ::Gitlab::AppJsonLogger.info(determined_content_type: content_type) end -- GitLab From 3c26c78f78183d178c5df1080cbf6b44b1f1b2c2 Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Thu, 10 Jul 2025 12:14:32 +0200 Subject: [PATCH 13/19] Update feature flag milestone to 18.2 --- .../gitlab_com_derisk/packages_generic_package_content_type.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/feature_flags/gitlab_com_derisk/packages_generic_package_content_type.yml b/config/feature_flags/gitlab_com_derisk/packages_generic_package_content_type.yml index 739f3aee582b57..8ebb9a583fc39d 100644 --- a/config/feature_flags/gitlab_com_derisk/packages_generic_package_content_type.yml +++ b/config/feature_flags/gitlab_com_derisk/packages_generic_package_content_type.yml @@ -4,7 +4,7 @@ description: Return generic package file with corresponding content type feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/444768 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/196427 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/547424 -milestone: '18.3' +milestone: '18.2' group: group::package registry type: gitlab_com_derisk default_enabled: false -- GitLab From baab62bbb935944cd29ddf28ef6027fe813435f5 Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Thu, 10 Jul 2025 12:23:24 +0200 Subject: [PATCH 14/19] Use expect ...to receive instead of have_received --- spec/requests/api/generic_packages_spec.rb | 12 ++++-------- .../packages/package_files_controller_spec.rb | 14 +++++--------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/spec/requests/api/generic_packages_spec.rb b/spec/requests/api/generic_packages_spec.rb index 93d01406053062..ea361bced07844 100644 --- a/spec/requests/api/generic_packages_spec.rb +++ b/spec/requests/api/generic_packages_spec.rb @@ -1123,13 +1123,11 @@ def upload_file( shared_examples 'log content type determination' do it 'logs content type determination' do - allow(Gitlab::AppJsonLogger).to receive(:info) - - subject - - expect(Gitlab::AppJsonLogger).to have_received(:info).with( + expect(Gitlab::AppJsonLogger).to receive(:info).with( determined_content_type: 'application/gzip' ) + + subject end context 'when feature flag is disabled' do @@ -1138,11 +1136,9 @@ def upload_file( end it 'does not log content type determination' do - allow(Gitlab::AppJsonLogger).to receive(:info) + expect(Gitlab::AppJsonLogger).not_to receive(:info) subject - - expect(Gitlab::AppJsonLogger).not_to have_received(:info) end end end diff --git a/spec/requests/projects/packages/package_files_controller_spec.rb b/spec/requests/projects/packages/package_files_controller_spec.rb index 3e860b69ebf56b..2f832c9e987a4f 100644 --- a/spec/requests/projects/packages/package_files_controller_spec.rb +++ b/spec/requests/projects/packages/package_files_controller_spec.rb @@ -18,10 +18,6 @@ ) end - before do - allow(Gitlab::AppJsonLogger).to receive(:info) - end - it 'sends the package file' do subject @@ -30,18 +26,18 @@ end it 'logs content type determination' do - subject - - expect(Gitlab::AppJsonLogger).to have_received(:info).with( + expect(Gitlab::AppJsonLogger).to receive(:info).with( determined_content_type: 'application/zip' ) + + subject end shared_examples 'not log' do it 'does not enable log' do - subject + expect(Gitlab::AppJsonLogger).not_to receive(:info) - expect(Gitlab::AppJsonLogger).not_to have_received(:info) + subject end end -- GitLab From c04ca6fed9b79b9f2d9553b8ea01976cfd612987 Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Fri, 11 Jul 2025 11:52:10 +0200 Subject: [PATCH 15/19] e to receive instead of to have_received --- spec/lib/gitlab/utils/mime_type_spec.rb | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/spec/lib/gitlab/utils/mime_type_spec.rb b/spec/lib/gitlab/utils/mime_type_spec.rb index 192e985c240e1c..c2d0972350f416 100644 --- a/spec/lib/gitlab/utils/mime_type_spec.rb +++ b/spec/lib/gitlab/utils/mime_type_spec.rb @@ -56,25 +56,21 @@ describe '.from_filename' do using RSpec::Parameterized::TableSyntax - before do - allow(Gitlab::AppJsonLogger).to receive(:info) - end - shared_examples 'log determination' do it 'logs the determination' do - subject - - expect(Gitlab::AppJsonLogger).to have_received(:info).with( + expect(Gitlab::AppJsonLogger).to receive(:info).with( determined_content_type: mime_type ) + + subject end end shared_examples 'not log determination' do it 'does not log the determination' do - subject + expect(Gitlab::AppJsonLogger).to receive(:info) - expect(Gitlab::AppJsonLogger).not_to have_received(:info) + subject end end -- GitLab From b21a281ee9e09947b9e86c212095d27968d7e7c9 Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Fri, 11 Jul 2025 15:06:26 +0200 Subject: [PATCH 16/19] Fix spec, use `spec_helper` and add feature_category --- spec/lib/gitlab/utils/mime_type_spec.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/lib/gitlab/utils/mime_type_spec.rb b/spec/lib/gitlab/utils/mime_type_spec.rb index c2d0972350f416..3b7d600acb0953 100644 --- a/spec/lib/gitlab/utils/mime_type_spec.rb +++ b/spec/lib/gitlab/utils/mime_type_spec.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' require 'rspec/parameterized' -RSpec.describe Gitlab::Utils::MimeType do +RSpec.describe Gitlab::Utils::MimeType, feature_category: :shared do describe '.from_io' do subject { described_class.from_io(io) } @@ -68,7 +68,7 @@ shared_examples 'not log determination' do it 'does not log the determination' do - expect(Gitlab::AppJsonLogger).to receive(:info) + expect(Gitlab::AppJsonLogger).not_to receive(:info) subject end -- GitLab From f38c7883e4de2f7d1f7b90df6ad5ffd5d0480a25 Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Fri, 11 Jul 2025 15:11:12 +0200 Subject: [PATCH 17/19] Update feature flag milestone to 18.3 --- .../gitlab_com_derisk/packages_generic_package_content_type.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/feature_flags/gitlab_com_derisk/packages_generic_package_content_type.yml b/config/feature_flags/gitlab_com_derisk/packages_generic_package_content_type.yml index 8ebb9a583fc39d..739f3aee582b57 100644 --- a/config/feature_flags/gitlab_com_derisk/packages_generic_package_content_type.yml +++ b/config/feature_flags/gitlab_com_derisk/packages_generic_package_content_type.yml @@ -4,7 +4,7 @@ description: Return generic package file with corresponding content type feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/444768 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/196427 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/547424 -milestone: '18.2' +milestone: '18.3' group: group::package registry type: gitlab_com_derisk default_enabled: false -- GitLab From 5ff7f37d8f9f72ec0ba3f7617a744708312662f1 Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Fri, 11 Jul 2025 15:22:46 +0200 Subject: [PATCH 18/19] Revert jobs_controller change --- app/controllers/projects/jobs_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/projects/jobs_controller.rb b/app/controllers/projects/jobs_controller.rb index 75eca566b4e802..43a09150b70ea3 100644 --- a/app/controllers/projects/jobs_controller.rb +++ b/app/controllers/projects/jobs_controller.rb @@ -265,7 +265,7 @@ def build_path(build) end def raw_trace_content_disposition(raw_data) - mime_type = ::Gitlab::Utils::MimeType.from_string(raw_data) + mime_type = Gitlab::Utils::MimeType.from_string(raw_data) # if mime_type is nil can also represent 'text/plain' return 'inline' if mime_type.nil? || mime_type == 'text/plain' -- GitLab From ec1097b29a7810a70643cd0d53a0fe3ba26a29aa Mon Sep 17 00:00:00 2001 From: Sylvia Shen Date: Fri, 11 Jul 2025 15:23:09 +0200 Subject: [PATCH 19/19] Change milestone back to 18.2 --- .../gitlab_com_derisk/packages_generic_package_content_type.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/feature_flags/gitlab_com_derisk/packages_generic_package_content_type.yml b/config/feature_flags/gitlab_com_derisk/packages_generic_package_content_type.yml index 739f3aee582b57..8ebb9a583fc39d 100644 --- a/config/feature_flags/gitlab_com_derisk/packages_generic_package_content_type.yml +++ b/config/feature_flags/gitlab_com_derisk/packages_generic_package_content_type.yml @@ -4,7 +4,7 @@ description: Return generic package file with corresponding content type feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/444768 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/196427 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/547424 -milestone: '18.3' +milestone: '18.2' group: group::package registry type: gitlab_com_derisk default_enabled: false -- GitLab