diff --git a/app/controllers/repositories/git_ssh_controller.rb b/app/controllers/repositories/git_ssh_controller.rb new file mode 100644 index 0000000000000000000000000000000000000000..5cc6b002232379e4b7de165c431cd718efe192e1 --- /dev/null +++ b/app/controllers/repositories/git_ssh_controller.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Repositories + class GitSshController < Repositories::GitHttpClientController + GITLAB_SHELL_JWT_ISSUER = "gitlab-shell" + + def upload_pack + set_workhorse_internal_api_content_type + + render json: Gitlab::Workhorse.git_http_ok(repository, repo_type, nil, :git_upload_pack) + end + + private + + def authenticate_user + payload, _ = JSONWebToken::HMACToken.decode(request.headers['Authorization'], ::Gitlab::Shell.secret_token) + render_access_denied! unless payload['iss'] == GITLAB_SHELL_JWT_ISSUER + rescue JWT::DecodeError, JWT::ExpiredSignature, JWT::ImmatureSignature => ex + Gitlab::ErrorTracking.track_exception(ex) + render_access_denied! + end + + def render_access_denied! + render(plain: 'Access is denied', status: :unauthorized) + end + end +end diff --git a/config/feature_flags/experiment/git_via_workhorse.yml b/config/feature_flags/experiment/git_via_workhorse.yml new file mode 100644 index 0000000000000000000000000000000000000000..2a9253d72d9c0c2d5966dba76cf6c193df041378 --- /dev/null +++ b/config/feature_flags/experiment/git_via_workhorse.yml @@ -0,0 +1,9 @@ +--- +name: git_via_workhorse +feature_issue_url: +introduced_by_url: 'https://gitlab.com/gitlab-org/gitlab/-/merge_requests/146227' +rollout_issue_url: +milestone: '16.10' +group: group::source code +type: experiment +default_enabled: false diff --git a/config/routes/git_http.rb b/config/routes/git_http.rb index 6899a89cc7dc0e5b000114969f98f7a0094ed9cf..07eeae08356d7679f7578465af23f448cba1aa2c 100644 --- a/config/routes/git_http.rb +++ b/config/routes/git_http.rb @@ -10,6 +10,10 @@ post '/git-receive-pack', action: :git_receive_pack end + scope(controller: :git_ssh) do + post '/ssh-upload-pack', action: :upload_pack + end + # NOTE: LFS routes are exposed on all repository types, but we still check for # LFS availability on the repository container in LfsRequest#lfs_check_access! diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 87b3838fb85e168b400cde6d8e710a32ea80beb5..e0a8af7cd66faf09488ec67fa155f933906970c5 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -78,7 +78,9 @@ def check_allowed(params) "uploadpack.allowAnySHA1InWant=true"], gitaly: gitaly_payload(params[:action]), gl_console_messages: check_result.console_messages, - need_audit: need_git_audit_event? + need_audit: need_git_audit_event?, + git_rpc_url: ::Gitlab::GitSshRpcUrl.new(project, params[:action]).execute, + git_rpc_auth_header: headers["Gitlab-Shell-Api-Request"] }.merge!(actor.key_details) # Custom option for git-receive-pack command diff --git a/lib/gitlab/git_ssh_rpc_url.rb b/lib/gitlab/git_ssh_rpc_url.rb new file mode 100644 index 0000000000000000000000000000000000000000..50a076c5e152fcb389c9458369ce6b14d988dc39 --- /dev/null +++ b/lib/gitlab/git_ssh_rpc_url.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + class GitSshRpcUrl # rubocop:disable Gitlab/NamespacedClass -- TODO + ACTION_TO_PATH = { + 'git-upload-pack' => '/ssh-upload-pack' + }.freeze + + def initialize(project, action) + @project = project + @action = action + end + + def execute + return unless Feature.enabled?(:git_via_workhorse, type: :experiment) + + path = ACTION_TO_PATH[action.to_s] + + return if path.blank? + + url = ::Gitlab::Routing.url_helpers.project_url(project, format: 'git') + + url + path + end + + private + + attr_reader :action, :project + end +end diff --git a/spec/requests/api/internal/base_spec.rb b/spec/requests/api/internal/base_spec.rb index 0969ef93ef64cb3400fa6ae19c309359326656a7..292eb042253dbb7f8df73a09f86c5f8c4002473d 100644 --- a/spec/requests/api/internal/base_spec.rb +++ b/spec/requests/api/internal/base_spec.rb @@ -620,6 +620,10 @@ def request_response(request:, call:, method:, metadata:) # rubocop:disable Lint context 'git push with personal snippet' do subject { push(key, personal_snippet, env: env.to_json, changes: snippet_changes) } + before do + stub_feature_flags(git_via_workhorse: false) + end + it 'responds with success' do subject @@ -638,6 +642,10 @@ def request_response(request:, call:, method:, metadata:) # rubocop:disable Lint context 'git pull with personal snippet' do subject { pull(key, personal_snippet) } + before do + stub_feature_flags(git_via_workhorse: false) + end + it 'responds with success' do subject diff --git a/workhorse/go.mod b/workhorse/go.mod index b325b9b03d39416445a7404db35bddf10933cf46..72211bf46d8628b2a55ab6a3148fa46c9306ba84 100644 --- a/workhorse/go.mod +++ b/workhorse/go.mod @@ -16,7 +16,7 @@ require ( github.com/johannesboyne/gofakes3 v0.0.0-20240217095638-c55a48f17be6 github.com/jpillora/backoff v1.0.0 github.com/mitchellh/copystructure v1.2.0 - github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/client_golang v1.19.1-0.20240328134234-93cf5d4f5f78 github.com/redis/go-redis/v9 v9.5.1 github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a github.com/sirupsen/logrus v1.9.3 @@ -94,7 +94,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/prometheus/prometheus v0.50.1 // indirect diff --git a/workhorse/go.sum b/workhorse/go.sum index f47f8ea97eb7a9a8bd16bda7eac356933fde2e85..e2a125d1e53a0c8eed2d0f3ef29f673a7d634b25 100644 --- a/workhorse/go.sum +++ b/workhorse/go.sum @@ -373,11 +373,11 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= -github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_golang v1.19.1-0.20240328134234-93cf5d4f5f78 h1:rSOwhjTtzeuOZS3pO9Gzy0vrGMHSR5s7eWiMKBTV8ns= +github.com/prometheus/client_golang v1.19.1-0.20240328134234-93cf5d4f5f78/go.mod h1:kDK4t8GKrX8Q1xkeHV0TTro2F3HIgGRx7X1Kt3GEku8= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= diff --git a/workhorse/internal/git/ssh.go b/workhorse/internal/git/ssh.go new file mode 100644 index 0000000000000000000000000000000000000000..3852d1e52e4b7a0f91609da0f32b17b45b469678 --- /dev/null +++ b/workhorse/internal/git/ssh.go @@ -0,0 +1,58 @@ +package git + +import ( + "fmt" + "net/http" + + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" + + "gitlab.com/gitlab-org/gitaly/v16/client" + + "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/gitaly" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/helper/fail" +) + +type FlushWriter struct { + http.ResponseWriter + controller *http.ResponseController +} + +func (f *FlushWriter) Write(p []byte) (int, error) { + n, err := f.ResponseWriter.Write(p) + if err != nil { + return n, err + } + + return n, f.controller.Flush() +} + +func SshUploadPack(a *api.API) http.Handler { + return repoPreAuthorizeHandler(a, sshUploadPack) +} + +func sshUploadPack(w http.ResponseWriter, r *http.Request, a *api.Response) { + controller := http.NewResponseController(w) + if err := controller.EnableFullDuplex(); err != nil { + fail.Request(w, r, fmt.Errorf("enabling full duplex: %v", err)) + + return + } + + conn, registry, err := gitaly.Connection(a.GitalyServer) + if err != nil { + fail.Request(w, r, fmt.Errorf("look up for gitaly connection: %v", err)) + + return + } + + w.WriteHeader(200) + + request := &gitalypb.SSHUploadPackWithSidechannelRequest{ + Repository: &a.Repository, + GitProtocol: r.Header.Get("Git-Protocol"), + GitConfigOptions: a.GitConfigOptions, + } + out := &FlushWriter{ResponseWriter: w, controller: controller} + client.UploadPackWithSidechannelWithResult(r.Context(), conn, registry, r.Body, out, out, request) +} diff --git a/workhorse/internal/gitaly/gitaly.go b/workhorse/internal/gitaly/gitaly.go index 98f73b40a9d51e60b9de59f5ef3aa8991073f912..01cffccbac197089efd1b011c85d48c70fd61e60 100644 --- a/workhorse/internal/gitaly/gitaly.go +++ b/workhorse/internal/gitaly/gitaly.go @@ -121,6 +121,12 @@ func NewDiffClient(ctx context.Context, server api.GitalyServer) (context.Contex return withOutgoingMetadata(ctx, server), &DiffClient{grpcClient}, nil } +func Connection(server api.GitalyServer) (*grpc.ClientConn, *gitalyclient.SidechannelRegistry, error) { + conn, err := getOrCreateConnection(server) + + return conn, sidechannelRegistry, err +} + func getOrCreateConnection(server api.GitalyServer) (*grpc.ClientConn, error) { key := getCacheKey(server) diff --git a/workhorse/internal/upstream/routes.go b/workhorse/internal/upstream/routes.go index bc0640029bf70557298ee479a2f3ad6f70068615..0edb688388657124964610163532e1a9f85c61ea 100644 --- a/workhorse/internal/upstream/routes.go +++ b/workhorse/internal/upstream/routes.go @@ -242,6 +242,7 @@ func configureRoutes(u *upstream) { u.route("POST", gitProjectPattern+`git-upload-pack\z`, contentEncodingHandler(git.UploadPack(api)), withMatcher(isContentType("application/x-git-upload-pack-request"))), u.route("POST", gitProjectPattern+`git-receive-pack\z`, contentEncodingHandler(git.ReceivePack(api)), withMatcher(isContentType("application/x-git-receive-pack-request"))), u.route("PUT", gitProjectPattern+`gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`, requestBodyUploader, withMatcher(isContentType("application/octet-stream"))), + u.route("POST", gitProjectPattern+`ssh-upload-pack\z`, git.SshUploadPack(api)), // CI Artifacts u.route("POST", apiPattern+`v4/jobs/[0-9]+/artifacts\z`, contentEncodingHandler(upload.Artifacts(api, signingProxy, preparer, &u.Config))),