From ee970a3a0b44d5aaffd8b7dfccd71b3e08d7e557 Mon Sep 17 00:00:00 2001 From: feistel <6742251-feistel@users.noreply.gitlab.com> Date: Fri, 14 Jan 2022 19:58:13 +0100 Subject: [PATCH] Add support for mutual TLS when calling the GitLab API --- app.go | 5 +-- internal/artifact/artifact.go | 11 +++++-- internal/artifact/artifact_test.go | 6 ++-- internal/auth/auth.go | 11 +++++-- internal/auth/auth_test.go | 5 ++- internal/config/config.go | 32 ++++++++++++++++++++ internal/config/flags.go | 3 ++ internal/config/validate.go | 1 + internal/httptransport/transport.go | 1 + internal/source/gitlab/client/client.go | 13 ++++++-- internal/source/gitlab/client/client_test.go | 8 ++--- 11 files changed, 79 insertions(+), 17 deletions(-) diff --git a/app.go b/app.go index 6ce414a4f..276e5ff33 100644 --- a/app.go +++ b/app.go @@ -366,7 +366,7 @@ func runApp(config *cfg.Config) error { return fmt.Errorf("failed to initialize logging: %w", err) } - a.Artifact = artifact.New(config.ArtifactsServer.URL, config.ArtifactsServer.TimeoutSeconds, config.General.Domain) + a.Artifact = artifact.New(config.ArtifactsServer.URL, config.ArtifactsServer.TimeoutSeconds, config.General.Domain, config.GitLab.ClientCertificates) if err := a.setAuth(config); err != nil { return err @@ -409,7 +409,8 @@ func (a *theApp) setAuth(config *cfg.Config) error { var err error a.Auth, err = auth.New(config.General.Domain, config.Authentication.Secret, config.Authentication.ClientID, config.Authentication.ClientSecret, - config.Authentication.RedirectURI, config.GitLab.InternalServer, config.GitLab.PublicServer, config.Authentication.Scope, config.Authentication.Timeout) + config.Authentication.RedirectURI, config.GitLab.InternalServer, config.GitLab.PublicServer, config.Authentication.Scope, config.Authentication.Timeout, + config.GitLab.ClientCertificates) if err != nil { return fmt.Errorf("could not initialize auth package: %w", err) } diff --git a/internal/artifact/artifact.go b/internal/artifact/artifact.go index dab1fb910..b99d4821c 100644 --- a/internal/artifact/artifact.go +++ b/internal/artifact/artifact.go @@ -2,6 +2,7 @@ package artifact import ( "context" + "crypto/tls" "errors" "fmt" "io" @@ -46,13 +47,19 @@ type Artifact struct { // New when provided the arguments defined herein, returns a pointer to an // Artifact that is used to proxy requests. -func New(server string, timeoutSeconds int, pagesDomain string) *Artifact { +func New(server string, timeoutSeconds int, pagesDomain string, clientCerts []tls.Certificate) *Artifact { + httpTransport := httptransport.DefaultTransport.Clone() + + if len(clientCerts) != 0 { + httpTransport.TLSClientConfig.Certificates = clientCerts + } + return &Artifact{ server: strings.TrimRight(server, "/"), suffix: "." + strings.ToLower(pagesDomain), client: &http.Client{ Timeout: time.Second * time.Duration(timeoutSeconds), - Transport: httptransport.DefaultTransport, + Transport: httpTransport, }, } } diff --git a/internal/artifact/artifact_test.go b/internal/artifact/artifact_test.go index b8c2771b5..3f6bfb9cc 100644 --- a/internal/artifact/artifact_test.go +++ b/internal/artifact/artifact_test.go @@ -77,7 +77,7 @@ func TestTryMakeRequest(t *testing.T) { reqURL, err := url.Parse("/-/subgroup/project/-/jobs/1/artifacts" + c.Path) require.NoError(t, err) r := &http.Request{URL: reqURL} - art := artifact.New(testServer.URL, 1, "gitlab-example.io") + art := artifact.New(testServer.URL, 1, "gitlab-example.io", nil) require.True(t, art.TryMakeRequest("group.gitlab-example.io", result, r, c.Token, func(resp *http.Response) bool { return false })) require.Equal(t, c.Status, result.Code) @@ -261,7 +261,7 @@ func TestBuildURL(t *testing.T) { for _, c := range cases { t.Run(c.Description, func(t *testing.T) { - a := artifact.New(c.RawServer, 1, c.PagesDomain) + a := artifact.New(c.RawServer, 1, c.PagesDomain, nil) u, ok := a.BuildURL(c.Host, c.Path) msg := c.Description + " - generated URL: " @@ -291,7 +291,7 @@ func TestContextCanceled(t *testing.T) { r = r.WithContext(ctx) // cancel context explicitly cancel() - art := artifact.New(testServer.URL, 1, "gitlab-example.io") + art := artifact.New(testServer.URL, 1, "gitlab-example.io", nil) require.True(t, art.TryMakeRequest("group.gitlab-example.io", result, r, "", func(resp *http.Response) bool { return false })) require.Equal(t, http.StatusNotFound, result.Code) diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 59dd0749c..35510ecef 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -3,6 +3,7 @@ package auth import ( "context" "crypto/sha256" + "crypto/tls" "encoding/base64" "encoding/json" "errors" @@ -634,13 +635,19 @@ func generateKeys(secret string, count int) ([][]byte, error) { } // New when authentication supported this will be used to create authentication handler -func New(pagesDomain, storeSecret, clientID, clientSecret, redirectURI, internalGitlabServer, publicGitlabServer, authScope string, authTimeout time.Duration) (*Auth, error) { +func New(pagesDomain, storeSecret, clientID, clientSecret, redirectURI, internalGitlabServer, publicGitlabServer, authScope string, authTimeout time.Duration, clientCerts []tls.Certificate) (*Auth, error) { // generate 3 keys, 2 for the cookie store and 1 for JWT signing keys, err := generateKeys(storeSecret, 3) if err != nil { return nil, err } + httpTransport := httptransport.DefaultTransport.Clone() + + if len(clientCerts) != 0 { + httpTransport.TLSClientConfig.Certificates = clientCerts + } + return &Auth{ pagesDomain: pagesDomain, clientID: clientID, @@ -650,7 +657,7 @@ func New(pagesDomain, storeSecret, clientID, clientSecret, redirectURI, internal publicGitlabServer: strings.TrimRight(publicGitlabServer, "/"), apiClient: &http.Client{ Timeout: authTimeout, - Transport: httptransport.DefaultTransport, + Transport: httpTransport, }, store: sessions.NewCookieStore(keys[0], keys[1]), authSecret: storeSecret, diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go index 4236d695f..1450623cb 100644 --- a/internal/auth/auth_test.go +++ b/internal/auth/auth_test.go @@ -29,7 +29,10 @@ func createTestAuth(t *testing.T, internalServer string, publicServer string) *A "http://pages.gitlab-example.com/auth", internalServer, publicServer, - "scope", 5*time.Second) + "scope", + 5*time.Second, + nil, + ) require.NoError(t, err) diff --git a/internal/config/config.go b/internal/config/config.go index 18fe33ebf..b5d59d5f8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -111,6 +111,7 @@ type GitLab struct { JWTTokenExpiration time.Duration Cache Cache EnableDisk bool + ClientCertificates []tls.Certificate } // Log groups settings related to configuring logging @@ -340,6 +341,13 @@ func loadConfig() (*Config, error) { } } + certs, err := loadClientCerts(clientCerts.Split()) + if err != nil { + return nil, err + } + + config.GitLab.ClientCertificates = certs + // Populating remaining GitLab settings config.GitLab.PublicServer = *publicGitLabServer @@ -354,6 +362,30 @@ func loadConfig() (*Config, error) { return config, nil } +func loadClientCerts(certs []string) ([]tls.Certificate, error) { + c := make([]tls.Certificate, 0, len(certs)) + + for i, pair := range certs { + sep := strings.Index(pair, ":") + + if sep == -1 { + return nil, fmt.Errorf("malformed client certs at position %d: %w", i, errMalformedClientCert) + } + + cert := pair[:sep] + key := pair[sep+1:] + + tlsCert, err := tls.LoadX509KeyPair(cert, key) + if err != nil { + return nil, err + } + + c = append(c, tlsCert) + } + + return c, nil +} + func LogConfig(config *Config) { log.WithFields(log.Fields{ "artifacts-server": *artifactsServer, diff --git a/internal/config/flags.go b/internal/config/flags.go index 88ebbb155..fe3a2eb1d 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -96,6 +96,8 @@ var ( listenHTTPSProxyv2 = MultiStringFlag{separator: ","} header = MultiStringFlag{separator: ";;"} + + clientCerts = MultiStringFlag{separator: ","} ) // initFlags will be called from LoadConfig @@ -105,6 +107,7 @@ func initFlags() { flag.Var(&listenProxy, "listen-proxy", "The address(es) or unix socket paths to listen on for proxy requests") flag.Var(&listenHTTPSProxyv2, "listen-https-proxyv2", "The address(es) or unix socket paths to listen on for HTTPS PROXYv2 requests (https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt)") flag.Var(&header, "header", "The additional http header(s) that should be send to the client") + flag.Var(&clientCerts, "client-cert-key-pairs", "File paths to client certificate and key PEM files to use for mutual TLS") // read from -config=/path/to/gitlab-pages-config flag.String(flag.DefaultConfigFlagname, "", "path to config file") diff --git a/internal/config/validate.go b/internal/config/validate.go index bb247287c..d572870c5 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -18,6 +18,7 @@ var ( errArtifactsServerUnsupportedScheme = errors.New("artifacts-server scheme must be either http:// or https://") errArtifactsServerInvalidTimeout = errors.New("artifacts-server-timeout must be greater than or equal to 1") errEmptyListener = errors.New("listener must not be empty") + errMalformedClientCert = errors.New("client certificates must be defined as /path/to/cert:/path/to/key") ) // Validate values populated in Config diff --git a/internal/httptransport/transport.go b/internal/httptransport/transport.go index 374e39d43..5b6f62849 100644 --- a/internal/httptransport/transport.go +++ b/internal/httptransport/transport.go @@ -41,6 +41,7 @@ func NewTransport() *http.Transport { DialTLS: func(network, addr string) (net.Conn, error) { return tls.Dial(network, addr, &tls.Config{RootCAs: pool(), MinVersion: tls.VersionTLS12}) }, + TLSClientConfig: &tls.Config{}, Proxy: http.ProxyFromEnvironment, // overrides the DefaultMaxIdleConnsPerHost = 2 MaxIdleConns: 100, diff --git a/internal/source/gitlab/client/client.go b/internal/source/gitlab/client/client.go index 75868cceb..b61291849 100644 --- a/internal/source/gitlab/client/client.go +++ b/internal/source/gitlab/client/client.go @@ -2,6 +2,7 @@ package client import ( "context" + "crypto/tls" "encoding/json" "errors" "fmt" @@ -45,7 +46,7 @@ type Client struct { // NewClient initializes and returns new Client baseUrl is // appConfig.InternalGitLabServer secretKey is appConfig.GitLabAPISecretKey -func NewClient(baseURL string, secretKey []byte, connectionTimeout, jwtTokenExpiry time.Duration) (*Client, error) { +func NewClient(baseURL string, secretKey []byte, connectionTimeout, jwtTokenExpiry time.Duration, clientCerts []tls.Certificate) (*Client, error) { if len(baseURL) == 0 || len(secretKey) == 0 { return nil, errors.New("GitLab API URL or API secret has not been provided") } @@ -63,6 +64,12 @@ func NewClient(baseURL string, secretKey []byte, connectionTimeout, jwtTokenExpi return nil, errors.New("GitLab JWT token expiry has not been provided") } + httpTransport := httptransport.DefaultTransport.Clone() + + if len(clientCerts) != 0 { + httpTransport.TLSClientConfig.Certificates = clientCerts + } + return &Client{ secretKey: secretKey, baseURL: parsedURL, @@ -70,7 +77,7 @@ func NewClient(baseURL string, secretKey []byte, connectionTimeout, jwtTokenExpi Timeout: connectionTimeout, Transport: httptransport.NewMeteredRoundTripper( correlation.NewInstrumentedRoundTripper( - httptransport.DefaultTransport, + httpTransport, correlation.WithClientName(transportClientName), ), transportClientName, @@ -86,7 +93,7 @@ func NewClient(baseURL string, secretKey []byte, connectionTimeout, jwtTokenExpi // NewFromConfig creates a new client from Config struct func NewFromConfig(cfg *config.GitLab) (*Client, error) { - return NewClient(cfg.InternalServer, cfg.APISecretKey, cfg.ClientHTTPTimeout, cfg.JWTTokenExpiration) + return NewClient(cfg.InternalServer, cfg.APISecretKey, cfg.ClientHTTPTimeout, cfg.JWTTokenExpiration, cfg.ClientCertificates) } // Resolve returns a VirtualDomain configuration wrapped into a Lookup for a diff --git a/internal/source/gitlab/client/client_test.go b/internal/source/gitlab/client/client_test.go index 1a6572db7..bf7dc3e70 100644 --- a/internal/source/gitlab/client/client_test.go +++ b/internal/source/gitlab/client/client_test.go @@ -59,7 +59,7 @@ func TestConnectionReuse(t *testing.T) { } func TestNewValidBaseURL(t *testing.T) { - _, err := NewClient("https://gitlab.com", secretKey(t), defaultClientConnTimeout, defaultJWTTokenExpiry) + _, err := NewClient("https://gitlab.com", secretKey(t), defaultClientConnTimeout, defaultJWTTokenExpiry, nil) require.NoError(t, err) } @@ -129,7 +129,7 @@ func TestNewInvalidConfiguration(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := NewClient(tt.args.baseURL, tt.args.secretKey, tt.args.connectionTimeout, tt.args.jwtTokenExpiry) + got, err := NewClient(tt.args.baseURL, tt.args.secretKey, tt.args.connectionTimeout, tt.args.jwtTokenExpiry, nil) require.Nil(t, got) require.Error(t, err) require.Contains(t, err.Error(), tt.wantErrMsg) @@ -263,7 +263,7 @@ func secretKey(t *testing.T) []byte { func defaultClient(t *testing.T, url string) *Client { t.Helper() - client, err := NewClient(url, secretKey(t), defaultClientConnTimeout, defaultJWTTokenExpiry) + client, err := NewClient(url, secretKey(t), defaultClientConnTimeout, defaultJWTTokenExpiry, nil) require.NoError(t, err) return client @@ -324,7 +324,7 @@ func Test_endpoint(t *testing.T) { for name, tt := range tests { t.Run(name, func(t *testing.T) { - gc, err := NewClient(tt.basePath, []byte("secret"), defaultClientConnTimeout, defaultJWTTokenExpiry) + gc, err := NewClient(tt.basePath, []byte("secret"), defaultClientConnTimeout, defaultJWTTokenExpiry, nil) require.NoError(t, err) got, err := gc.endpoint(tt.urlPath, tt.params) -- GitLab