diff --git a/acceptance_test.go b/acceptance_test.go index 4a1b30286df06c23ad0bc5028af4baad81811cfa..3345ae5bada33111667ff3490cf4da22aab39b73 100644 --- a/acceptance_test.go +++ b/acceptance_test.go @@ -1532,22 +1532,11 @@ func TestGitlabDomainsSource(t *testing.T) { source := NewGitlabDomainsSourceStub(t) defer source.Close() - gitlabSourceConfig := ` -domains: - enabled: - - new-source-test.gitlab.io - broken: pages-broken-poc.gitlab.io -` - gitlabSourceConfigFile, cleanupGitlabSourceConfigFile := CreateGitlabSourceConfigFixtureFile(t, gitlabSourceConfig) - defer cleanupGitlabSourceConfigFile() - - gitlabSourceConfigFile = "GITLAB_SOURCE_CONFIG_FILE=" + gitlabSourceConfigFile - gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t) pagesArgs := []string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey} - teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{gitlabSourceConfigFile}, pagesArgs...) + teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{}, pagesArgs...) defer teardown() t.Run("when a domain exists", func(t *testing.T) { @@ -1568,13 +1557,4 @@ domains: require.Equal(t, http.StatusNotFound, response.StatusCode) }) - - t.Run("broken domain is requested", func(t *testing.T) { - response, err := GetPageFromListener(t, httpListener, "pages-broken-poc.gitlab.io", "index.html") - require.NoError(t, err) - - defer response.Body.Close() - - require.Equal(t, http.StatusBadGateway, response.StatusCode) - }) } diff --git a/app.go b/app.go index 0d6288644e17a55f4507ad9bcf3c8dbf25112538..96117b9be98e4b198d3ee8e0a3bd0e10882de245 100644 --- a/app.go +++ b/app.go @@ -12,7 +12,7 @@ import ( log "github.com/sirupsen/logrus" "gitlab.com/gitlab-org/labkit/errortracking" labmetrics "gitlab.com/gitlab-org/labkit/metrics" - mimedb "gitlab.com/lupine/go-mimedb" + "gitlab.com/lupine/go-mimedb" "gitlab.com/gitlab-org/gitlab-pages/internal/acme" "gitlab.com/gitlab-org/gitlab-pages/internal/artifact" diff --git a/app_config.go b/app_config.go index 5d481bb68f0d3fc3e75c5dea8dd90d7c8151f6d5..3bc9408c323643e46c968ce73586b90752253216 100644 --- a/app_config.go +++ b/app_config.go @@ -27,18 +27,19 @@ type appConfig struct { LogFormat string LogVerbose bool - StoreSecret string - GitLabServer string - InternalGitLabServer string - GitLabAPISecretKey []byte - GitlabClientHTTPTimeout time.Duration - GitlabJWTTokenExpiration time.Duration - ClientID string - ClientSecret string - RedirectURI string - SentryDSN string - SentryEnvironment string - CustomHeaders []string + StoreSecret string + GitLabServer string + InternalGitLabServer string + GitLabAPISecretKey []byte + GitlabClientHTTPTimeout time.Duration + GitlabJWTTokenExpiration time.Duration + GitlabDisableAPIConfigSource bool + ClientID string + ClientSecret string + RedirectURI string + SentryDSN string + SentryEnvironment string + CustomHeaders []string } // InternalGitLabServerURL returns URL to a GitLab instance. @@ -58,3 +59,7 @@ func (config appConfig) GitlabClientConnectionTimeout() time.Duration { func (config appConfig) GitlabJWTTokenExpiry() time.Duration { return config.GitlabJWTTokenExpiration } + +func (config appConfig) GitlabDisableAPIConfigurationSource() bool { + return config.GitlabDisableAPIConfigSource +} diff --git a/go.mod b/go.mod index 908342efd0aa4ca0c6cdd28cde410a7b9d9c85b0..686863071edf0d18e1efd301370a14480502d435 100644 --- a/go.mod +++ b/go.mod @@ -31,5 +31,4 @@ require ( golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b golang.org/x/tools v0.0.0-20191010201905-e5ffc44a6fee gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect - gopkg.in/yaml.v2 v2.2.2 ) diff --git a/helpers_test.go b/helpers_test.go index 195c3cea8709dead30246e095bce98ac1d9d6ba1..1fa5b6bb44186f715dfb85c46ce69f9e20f22f29 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -12,7 +12,6 @@ import ( "net/http/httptest" "os" "os/exec" - "path/filepath" "strings" "testing" "time" @@ -86,27 +85,6 @@ func CreateGitLabAPISecretKeyFixtureFile(t *testing.T) (filepath string) { return secretfile.Name() } -func CreateGitlabSourceConfigFixtureFile(t *testing.T, domains string) (filename string, cleanup func()) { - configfile, err := ioutil.TempFile("shared/pages", "gitlab-source-config-*") - require.NoError(t, err) - configfile.Close() - - cleanup = func() { - os.RemoveAll(configfile.Name()) - } - - require.NoError(t, ioutil.WriteFile(configfile.Name(), []byte(domains), 0644)) - - filename, err = filepath.Abs(configfile.Name()) - require.NoError(t, err) - - if os.Getenv("TEST_DAEMONIZE") != "" { - filename = filepath.Base(filename) - } - - return filename, cleanup -} - // ListenSpec is used to point at a gitlab-pages http server, preserving the // type of port it is (http, https, proxy) type ListenSpec struct { diff --git a/internal/source/config.go b/internal/source/config.go index 9cf87bc65f6a622f932056dfb86be4cd24931622..f262fcbc944963c57f301651859622b9ca36be97 100644 --- a/internal/source/config.go +++ b/internal/source/config.go @@ -3,5 +3,5 @@ package source import "gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab/client" // Config represents an interface that is configuration provider for client -// capable of comunicating with GitLab +// capable of communicating with GitLab type Config client.Config diff --git a/internal/source/disk/disk.go b/internal/source/disk/disk.go index b79d222de9b3030c663f5d5ecfc308646bcbdf09..46db831feb2edbadafeb566fb6c287e9b7d15213 100644 --- a/internal/source/disk/disk.go +++ b/internal/source/disk/disk.go @@ -10,6 +10,7 @@ import ( // Disk struct represents a map of all domains supported by pages that are // stored on a disk with corresponding `config.json`. +// TODO remove disk source https://gitlab.com/gitlab-org/gitlab-pages/-/issues/379 type Disk struct { dm Map lock *sync.RWMutex diff --git a/internal/source/domains.go b/internal/source/domains.go index 8de7c574e1e0b18cd35d5c967225b3f52062d70e..b3b3ade4e11ec3da6b1eb1e8ecf176d10f0647f7 100644 --- a/internal/source/domains.go +++ b/internal/source/domains.go @@ -2,33 +2,21 @@ package source import ( "errors" - "regexp" - "time" + "strings" log "github.com/sirupsen/logrus" "gitlab.com/gitlab-org/gitlab-pages/internal/domain" - "gitlab.com/gitlab-org/gitlab-pages/internal/rollout" "gitlab.com/gitlab-org/gitlab-pages/internal/source/disk" - "gitlab.com/gitlab-org/gitlab-pages/internal/source/domains/gitlabsourceconfig" "gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab" + "gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab/client" ) var ( - gitlabSourceConfig gitlabsourceconfig.GitlabSourceConfig - - // serverlessDomainRegex is a regular expression we use to check if a domain - // is a serverless domain, to short circut gitlab source rollout. It can be - // removed after the rollout is done - serverlessDomainRegex = regexp.MustCompile(`^[^.]+-[[:xdigit:]]{2}a1[[:xdigit:]]{10}f2[[:xdigit:]]{2}[[:xdigit:]]+-?.*`) + // errSourceNotConfigured will be returned when neither disk nor gitlab sources are configured + errSourceNotConfigured = errors.New("domain source configuration not available") ) -func init() { - // Start watching the config file for domains that will use the new `gitlab` source, - // to be removed once we switch completely to using it. - go gitlabsourceconfig.WatchForGitlabSourceConfigChange(&gitlabSourceConfig, 1*time.Minute) -} - // Domains struct represents a map of all domains supported by pages. It is // currently using two sources during the transition to the new GitLab domains // source. @@ -37,22 +25,31 @@ type Domains struct { disk *disk.Disk // legacy disk source } +func useDiskConfigSource(config Config) bool { + return config.GitlabDisableAPIConfigurationSource() || len(config.InternalGitLabServerURL()) == 0 || len(config.GitlabAPISecret()) == 0 +} + // NewDomains is a factory method for domains initializing a mutex. It should // not initialize `dm` as we later check the readiness by comparing it with a // nil value. func NewDomains(config Config) (*Domains, error) { - if len(config.InternalGitLabServerURL()) == 0 || len(config.GitlabAPISecret()) == 0 { + // fallback to disk if these values are empty + if useDiskConfigSource(config) { + log.Warn("disk source will be deprecated soon https://gitlab.com/gitlab-org/gitlab/-/issues/210010") return &Domains{disk: disk.New()}, nil } - gitlab, err := gitlab.New(config) + gl, err := gitlab.New(config) if err != nil { + if strings.Contains(err.Error(), client.ConnectionErrorMsg) { + log.WithError(err).Warn("GitLab API is not configured https://gitlab.com/gitlab-org/gitlab/-/issues/210010") + return &Domains{disk: disk.New()}, nil + } return nil, err } return &Domains{ - gitlab: gitlab, - disk: disk.New(), + gitlab: gl, }, nil } @@ -61,65 +58,35 @@ func NewDomains(config Config) (*Domains, error) { // for some subset of domains, to test / PoC the new GitLab Domains Source that // we plan to use to replace the disk source. func (d *Domains) GetDomain(name string) (*domain.Domain, error) { - if name == gitlabSourceConfig.Domains.Broken { - return nil, errors.New("broken test domain used") + source, err := d.source() + if err != nil { + return nil, err } - - return d.source(name).GetDomain(name) + return source.GetDomain(name) } // Read starts the disk domain source. It is DEPRECATED, because we want to // remove it entirely when disk source gets removed. func (d *Domains) Read(rootDomain string) { - d.disk.Read(rootDomain) + if d.disk != nil { + d.disk.Read(rootDomain) + } } // IsReady checks if the disk domain source managed to traverse entire pages // filesystem and is ready for use. It is DEPRECATED, because we want to remove // it entirely when disk source gets removed. func (d *Domains) IsReady() bool { - return d.disk.IsReady() + // return true if d.disk is nil while we remove all of the disk source code + // TODO https://gitlab.com/gitlab-org/gitlab-pages/-/issues/379 + return d.disk == nil || d.disk.IsReady() } -func (d *Domains) source(domain string) Source { - if d.gitlab == nil { - return d.disk - } - - // This check is only needed until we enable `d.gitlab` source in all - // environments (including on-premises installations) followed by removal of - // `d.disk` source. This can be safely removed afterwards. - if IsServerlessDomain(domain) { - return d.gitlab +func (d *Domains) source() (Source, error) { + if d.gitlab != nil { + return d.gitlab, nil + } else if d.disk != nil { + return d.disk, nil } - - for _, name := range gitlabSourceConfig.Domains.Enabled { - if domain == name { - return d.gitlab - } - } - - r := gitlabSourceConfig.Domains.Rollout - - enabled, err := rollout.Rollout(domain, r.Percentage, r.Stickiness) - if err != nil { - log.WithError(err).Error("Rollout error") - return d.disk - } - - if enabled { - return d.gitlab - } - - return d.disk -} - -// IsServerlessDomain checks if a domain requested is a serverless domain we -// need to handle differently. -// -// Domain is a serverless domain when it matches `serverlessDomainRegex`. The -// regular expression is also defined on the gitlab-rails side, see -// https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/serverless/domain.rb#L7 -func IsServerlessDomain(domain string) bool { - return serverlessDomainRegex.MatchString(domain) + return nil, errSourceNotConfigured } diff --git a/internal/source/domains/gitlabsourceconfig/gitlabsourceconfig.go b/internal/source/domains/gitlabsourceconfig/gitlabsourceconfig.go deleted file mode 100644 index ebc8b485955dc3d6ff7ec3574a4bde7cf3fd03ad..0000000000000000000000000000000000000000 --- a/internal/source/domains/gitlabsourceconfig/gitlabsourceconfig.go +++ /dev/null @@ -1,94 +0,0 @@ -package gitlabsourceconfig - -import ( - "bytes" - "io/ioutil" - "os" - "time" - - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" -) - -// GitlabSourceDomains holds the domains to be used with the gitlab source -type GitlabSourceDomains struct { - Enabled []string - Broken string - Rollout GitlabSourceRollout -} - -// GitlabSourceRollout holds the rollout strategy and percentage -type GitlabSourceRollout struct { - Stickiness string - Percentage int -} - -// GitlabSourceConfig holds the configuration for the gitlab source -type GitlabSourceConfig struct { - Domains GitlabSourceDomains -} - -// UpdateFromYaml updates the config -// We use new variable here (instead of using `config` directly) -// because if `content` is empty `yaml.Unmarshal` does not update -// the fields already set. -func (config *GitlabSourceConfig) UpdateFromYaml(content []byte) error { - updated := GitlabSourceConfig{} - - err := yaml.Unmarshal(content, &updated) - if err != nil { - return err - } - - *config = updated - - log.WithFields(log.Fields{ - "Enabled domains": config.Domains.Enabled, - "Broken domain": config.Domains.Broken, - "Rollout %": config.Domains.Rollout.Percentage, - "Rollout stickiness": config.Domains.Rollout.Stickiness, - }).Info("gitlab source config updated") - - return nil -} - -// WatchForGitlabSourceConfigChange polls the filesystem and updates test domains if needed. -func WatchForGitlabSourceConfigChange(config *GitlabSourceConfig, interval time.Duration) { - var lastContent []byte - - gitlabSourceConfigFile := os.Getenv("GITLAB_SOURCE_CONFIG_FILE") - if gitlabSourceConfigFile == "" { - gitlabSourceConfigFile = ".gitlab-source-config.yml" - } - - for { - content, err := readConfig(gitlabSourceConfigFile) - if err != nil { - log.WithError(err).Warn("Failed to read gitlab source config file") - - time.Sleep(interval) - continue - } - - if !bytes.Equal(lastContent, content) { - lastContent = content - - err = config.UpdateFromYaml(content) - if err != nil { - log.WithError(err).Warn("Failed to update gitlab source config") - } - } - - time.Sleep(interval) - } -} - -func readConfig(configfile string) ([]byte, error) { - content, err := ioutil.ReadFile(configfile) - - if err != nil && !os.IsNotExist(err) { - return nil, err - } - - return content, nil -} diff --git a/internal/source/domains_test.go b/internal/source/domains_test.go index ebafb6fc23f731546d9cd65773e8dcaa2d80f4ac..65fbf31a60fde8a56bcb2e3afdec32304ae3b132 100644 --- a/internal/source/domains_test.go +++ b/internal/source/domains_test.go @@ -1,19 +1,21 @@ package source import ( - "math/rand" + "net/http" + "net/http/httptest" "testing" "time" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitlab-pages/internal/domain" - "gitlab.com/gitlab-org/gitlab-pages/internal/source/disk" ) type sourceConfig struct { - api string - secret string + api string + secret string + disable bool } func (c sourceConfig) InternalGitLabServerURL() string { @@ -30,31 +32,80 @@ func (c sourceConfig) GitlabClientConnectionTimeout() time.Duration { func (c sourceConfig) GitlabJWTTokenExpiry() time.Duration { return 30 * time.Second } +func (c sourceConfig) GitlabDisableAPIConfigurationSource() bool { + return c.disable +} func TestDomainSources(t *testing.T) { - t.Run("when GitLab API URL has been provided", func(t *testing.T) { - domains, err := NewDomains(sourceConfig{api: "https://gitlab.com", secret: "abc"}) - require.NoError(t, err) + // TODO refactor test when disk source is removed https://gitlab.com/gitlab-org/gitlab-pages/-/issues/382 + tests := []struct { + name string + config sourceConfig + mock bool + status int + expectGitlab bool + expectDisk bool + }{ + { + name: "gitlab_source_on_success", + config: sourceConfig{api: "http://localhost", secret: "abc"}, + mock: true, + status: http.StatusNoContent, + expectGitlab: true, + }, + { + name: "disk_source_on_unauthorized", + config: sourceConfig{api: "http://localhost", secret: "abc"}, + mock: true, + status: http.StatusUnauthorized, + expectDisk: true, + }, + { + name: "disk_source_on_api_error", + config: sourceConfig{api: "http://localhost", secret: "abc"}, + mock: true, + status: http.StatusServiceUnavailable, + expectDisk: true, + }, + { + name: "disk_source_on_disabled_api_source", + config: sourceConfig{api: "http://localhost", secret: "abc", disable: true}, + expectDisk: true, + }, + { + name: "disk_source_on_incomplete_config", + config: sourceConfig{api: "", secret: "abc"}, + expectDisk: true, + }, + } - require.NotNil(t, domains.gitlab) - require.NotNil(t, domains.disk) - }) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.mock { + m := http.NewServeMux() + m.HandleFunc("/api/v4/internal/pages/status", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tt.status) + }) - t.Run("when GitLab API has not been provided", func(t *testing.T) { - domains, err := NewDomains(sourceConfig{}) - require.NoError(t, err) + mockServer := httptest.NewServer(m) + defer mockServer.Close() + + tt.config.api = mockServer.URL + } + + domains, err := NewDomains(tt.config) + require.NoError(t, err) + + require.Equal(t, tt.expectGitlab, domains.gitlab != nil) + require.Equal(t, tt.expectDisk, domains.disk != nil) + }) + } - require.Nil(t, domains.gitlab) - require.NotNil(t, domains.disk) - }) } func TestGetDomain(t *testing.T) { - gitlabSourceConfig.Domains.Enabled = []string{"new-source-test.gitlab.io"} - gitlabSourceConfig.Domains.Broken = "pages-broken-poc.gitlab.io" - - t.Run("when requesting a test domain", func(t *testing.T) { - testDomain := gitlabSourceConfig.Domains.Enabled[0] + t.Run("when requesting a domain that exists", func(t *testing.T) { + testDomain := "new-source-test.gitlab.io" newSource := NewMockSource() newSource.On("GetDomain", testDomain). @@ -63,153 +114,26 @@ func TestGetDomain(t *testing.T) { defer newSource.AssertExpectations(t) domains := &Domains{ - disk: disk.New(), gitlab: newSource, } - domains.GetDomain(testDomain) - }) - - t.Run("when requesting a non-test domain", func(t *testing.T) { - newSource := NewMockSource() - defer newSource.AssertExpectations(t) - - domains := &Domains{ - disk: disk.New(), - gitlab: newSource, - } - - domain, err := domains.GetDomain("domain.test.io") - + d, err := domains.GetDomain(testDomain) require.NoError(t, err) - require.Nil(t, domain) + require.NotNil(t, d) + require.Equal(t, d.Name, testDomain) }) - t.Run("when requesting a broken test domain", func(t *testing.T) { + t.Run("when requesting a domain that doesn't exist", func(t *testing.T) { newSource := NewMockSource() + newSource.On("GetDomain", mock.Anything).Return(nil, nil) defer newSource.AssertExpectations(t) domains := &Domains{ - disk: disk.New(), gitlab: newSource, } - domain, err := domains.GetDomain("pages-broken-poc.gitlab.io") - - require.Nil(t, domain) - require.EqualError(t, err, "broken test domain used") - }) - - t.Run("when requesting a test domain in case of the source not being fully configured", func(t *testing.T) { - domains, err := NewDomains(sourceConfig{}) - require.NoError(t, err) - - domain, err := domains.GetDomain("new-source-test.gitlab.io") - - require.Nil(t, domain) + d, err := domains.GetDomain("domain.test.io") require.NoError(t, err) + require.Nil(t, d) }) - - t.Run("when requesting a serverless domain", func(t *testing.T) { - testDomain := "func-aba1aabbccddeef2abaabbcc.serverless.gitlab.io" - - newSource := NewMockSource() - newSource.On("GetDomain", testDomain). - Return(&domain.Domain{Name: testDomain}, nil). - Once() - defer newSource.AssertExpectations(t) - - domains := &Domains{ - disk: disk.New(), - gitlab: newSource, - } - - domains.GetDomain(testDomain) - }) -} - -func TestIsServerlessDomain(t *testing.T) { - t.Run("when a domain is serverless domain", func(t *testing.T) { - require.True(t, IsServerlessDomain("some-function-aba1aabbccddeef2abaabbcc.serverless.gitlab.io")) - }) - - t.Run("when a domain is serverless domain with environment", func(t *testing.T) { - require.True(t, IsServerlessDomain("some-function-aba1aabbccddeef2abaabbcc-testing.serverless.gitlab.io")) - }) - - t.Run("when a domain is not a serverless domain", func(t *testing.T) { - require.False(t, IsServerlessDomain("somedomain.gitlab.io")) - }) -} - -func TestGetDomainWithIncrementalrolloutOfGitLabSource(t *testing.T) { - // This will produce the following pseudo-random sequence: 5, 87, 68 - rand.Seed(42) - - // Generates FNV hash 4091421005, 4091421005 % 100 = 5 - domain05 := "test-domain-a.com" - // Generates FNV 2643293380, 2643293380 % 100 = 80 - domain80 := "test-domain-b.com" - - diskSource := disk.New() - - gitlabSourceConfig.Domains.Rollout.Percentage = 80 - - type testDomain struct { - name string - source string - times int - } - - tests := map[string]struct { - stickiness string - domains []testDomain - }{ - // domain05 should always use gitlab source, - // domain80 should use disk source - "default stickiness": { - stickiness: "", - domains: []testDomain{ - {name: domain05, source: "gitlab"}, - {name: domain80, source: "disk"}, - {name: domain05, source: "gitlab"}, - }, - }, - // Given that randSeed(42) will produce the following pseudo-random sequence: - // {5, 87, 68} the first and third call for domain05 should use gitlab source, - // while the second one should use disk source - "no stickiness": { - stickiness: "random", - domains: []testDomain{ - {name: domain05, source: "gitlab"}, - {name: domain05, source: "disk"}, - {name: domain05, source: "gitlab"}, - }}, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - gitlabSource := NewMockSource() - for _, d := range tc.domains { - if d.source == "gitlab" { - gitlabSource.On("GetDomain", d.name). - Return(&domain.Domain{Name: d.name}, nil). - Once() - } - } - defer gitlabSource.AssertExpectations(t) - - domains := &Domains{ - disk: diskSource, - gitlab: gitlabSource, - } - - gitlabSourceConfig.Domains.Rollout.Stickiness = tc.stickiness - - for _, domain := range tc.domains { - _, err := domains.GetDomain(domain.name) - require.NoError(t, err) - } - }) - } } diff --git a/internal/source/gitlab/client/client.go b/internal/source/gitlab/client/client.go index bfe425b92e60e878d2e544328f4774a916a8859b..e2f07f1e7810e5e260cdd3820fe6752cc996e21f 100644 --- a/internal/source/gitlab/client/client.go +++ b/internal/source/gitlab/client/client.go @@ -18,6 +18,11 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/metrics" ) +// ConnectionErrorMsg to be returned with `gc.Status` if Pages +// cannot contact /api/v4/internal/pages/status either because of a 404 (disabled) +// or a 401 given that the credentials used are wrong +const ConnectionErrorMsg = "failed to connect to internal Pages API" + // Client is a HTTP client to access Pages internal API type Client struct { secretKey []byte @@ -91,6 +96,17 @@ func (gc *Client) GetLookup(ctx context.Context, host string) api.Lookup { return lookup } +// Status checks that Pages can reach the rails internal Pages API +// for source domain configuration. +// Timeout is the same as -gitlab-client-http-timeout +func (gc *Client) Status() error { + _, err := gc.get(context.Background(), "/api/v4/internal/pages/status", url.Values{}) + if err != nil { + return fmt.Errorf("%s: %v", ConnectionErrorMsg, err) + } + return nil +} + func (gc *Client) get(ctx context.Context, path string, params url.Values) (*http.Response, error) { endpoint, err := gc.endpoint(path, params) if err != nil { diff --git a/internal/source/gitlab/client/client_test.go b/internal/source/gitlab/client/client_test.go index 58729153b292a228ff2886de3fc6abbeccfa1780..86d20b32d15817adcfc226e46446a38d59ebd132 100644 --- a/internal/source/gitlab/client/client_test.go +++ b/internal/source/gitlab/client/client_test.go @@ -6,10 +6,11 @@ import ( "fmt" "net/http" "net/http/httptest" + "strconv" "testing" "time" - jwt "github.com/dgrijalva/jwt-go" + "github.com/dgrijalva/jwt-go" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitlab-pages/internal/fixture" @@ -195,6 +196,47 @@ func TestGetVirtualDomainAuthenticatedRequest(t *testing.T) { require.Equal(t, "mygroup/myproject/public/", lookupPath.Source.Path) } +func TestClientStatus(t *testing.T) { + tests := []struct { + name string + status int + wantErr bool + }{ + { + name: "api_enabled", + status: http.StatusNoContent, + wantErr: false, + }, + { + name: "api_unauthorized", + status: http.StatusUnauthorized, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v4/internal/pages/status", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tt.status) + }) + server := httptest.NewServer(mux) + defer server.Close() + + client := defaultClient(t, server.URL) + + err := client.Status() + require.Equal(t, tt.wantErr, err != nil) + + if tt.wantErr { + require.NotNil(t, err) + require.Contains(t, err.Error(), strconv.Itoa(tt.status)) + } + }) + } + +} + func validateToken(t *testing.T, tokenString string) { t.Helper() token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) { diff --git a/internal/source/gitlab/client/config.go b/internal/source/gitlab/client/config.go index 19a87452b66d6a29f23426ab105c98f005e1fb9a..b760d6bed8d1ffe52f035ef485c7fe95a688dce5 100644 --- a/internal/source/gitlab/client/config.go +++ b/internal/source/gitlab/client/config.go @@ -9,4 +9,5 @@ type Config interface { GitlabAPISecret() []byte GitlabClientConnectionTimeout() time.Duration GitlabJWTTokenExpiry() time.Duration + GitlabDisableAPIConfigurationSource() bool } diff --git a/internal/source/gitlab/gitlab.go b/internal/source/gitlab/gitlab.go index 12da9af17f3db754456c56a66b52e02d3b2ac32e..b30e811953bad79b7958e4e2d925a6232ef10bc7 100644 --- a/internal/source/gitlab/gitlab.go +++ b/internal/source/gitlab/gitlab.go @@ -22,14 +22,19 @@ type Gitlab struct { } // New returns a new instance of gitlab domain source. +// It checks that it can reach the API by calling cli.Status. func New(config client.Config) (*Gitlab, error) { - client, err := client.NewFromConfig(config) + cli, err := client.NewFromConfig(config) if err != nil { return nil, err } + if err := cli.Status(); err != nil { + return nil, err + } + // using nil for cache config will use the default values specified in internal/source/gitlab/cache/cache.go#12 - return &Gitlab{client: cache.NewCache(client, nil)}, nil + return &Gitlab{client: cache.NewCache(cli, nil)}, nil } // GetDomain return a representation of a domain that we have fetched from diff --git a/internal/source/source_mock.go b/internal/source/source_mock.go index ee24d804e6702fd17f57c0901cfdbdde238fdfe4..e6d48d687430422bf6884401b51fe38ee2986f50 100644 --- a/internal/source/source_mock.go +++ b/internal/source/source_mock.go @@ -14,8 +14,12 @@ type MockSource struct { // GetDomain is a mocked function func (m *MockSource) GetDomain(name string) (*domain.Domain, error) { args := m.Called(name) + d, ok := args.Get(0).(*domain.Domain) + if !ok { + return nil, args.Error(1) + } - return args.Get(0).(*domain.Domain), args.Error(1) + return d, args.Error(1) } // NewMockSource returns a new Source mock for testing diff --git a/main.go b/main.go index 2614fa0bc82f2c0397814687cfbeecadfb52e63f..008da7d165958b5324210d342291aebbb1820759 100644 --- a/main.go +++ b/main.go @@ -36,42 +36,43 @@ func init() { } var ( - pagesRootCert = flag.String("root-cert", "", "The default path to file certificate to serve static pages") - pagesRootKey = flag.String("root-key", "", "The default path to file certificate to serve static pages") - redirectHTTP = flag.Bool("redirect-http", false, "Redirect pages from HTTP to HTTPS") - useHTTP2 = flag.Bool("use-http2", true, "Enable HTTP2 support") - pagesRoot = flag.String("pages-root", "shared/pages", "The directory where pages are stored") - pagesDomain = flag.String("pages-domain", "gitlab-example.com", "The domain to serve static pages") - artifactsServer = flag.String("artifacts-server", "", "API URL to proxy artifact requests to, e.g.: 'https://gitlab.com/api/v4'") - artifactsServerTimeout = flag.Int("artifacts-server-timeout", 10, "Timeout (in seconds) for a proxied request to the artifacts server") - pagesStatus = flag.String("pages-status", "", "The url path for a status page, e.g., /@status") - metricsAddress = flag.String("metrics-address", "", "The address to listen on for metrics requests") - sentryDSN = flag.String("sentry-dsn", "", "The address for sending sentry crash reporting to") - sentryEnvironment = flag.String("sentry-environment", "", "The environment for sentry crash reporting") - daemonUID = flag.Uint("daemon-uid", 0, "Drop privileges to this user") - daemonGID = flag.Uint("daemon-gid", 0, "Drop privileges to this group") - daemonInplaceChroot = flag.Bool("daemon-inplace-chroot", false, "Fall back to a non-bind-mount chroot of -pages-root when daemonizing") - logFormat = flag.String("log-format", "text", "The log output format: 'text' or 'json'") - logVerbose = flag.Bool("log-verbose", false, "Verbose logging") - _ = flag.String("admin-secret-path", "", "DEPRECATED") - _ = flag.String("admin-unix-listener", "", "DEPRECATED") - _ = flag.String("admin-https-listener", "", "DEPRECATED") - _ = flag.String("admin-https-cert", "", "DEPRECATED") - _ = flag.String("admin-https-key", "", "DEPRECATED") - secret = flag.String("auth-secret", "", "Cookie store hash key, should be at least 32 bytes long") - gitLabAuthServer = flag.String("auth-server", "", "DEPRECATED, use gitlab-server instead. GitLab server, for example https://www.gitlab.com") - gitLabServer = flag.String("gitlab-server", "", "GitLab server, for example https://www.gitlab.com") - internalGitLabServer = flag.String("internal-gitlab-server", "", "Internal GitLab server used for API requests, useful if you want to send that traffic over an internal load balancer, example value https://www.gitlab.com (defaults to value of gitlab-server)") - gitLabAPISecretKey = flag.String("api-secret-key", "", "File with secret key used to authenticate with the GitLab API") - gitlabClientHTTPTimeout = flag.Duration("gitlab-client-http-timeout", 10*time.Second, "GitLab API HTTP client connection timeout in seconds (default: 10s)") - gitlabClientJWTExpiry = flag.Duration("gitlab-client-jwt-expiry", 30*time.Second, "JWT Token expiry time in seconds (default: 30s)") - clientID = flag.String("auth-client-id", "", "GitLab application Client ID") - clientSecret = flag.String("auth-client-secret", "", "GitLab application Client Secret") - redirectURI = flag.String("auth-redirect-uri", "", "GitLab application redirect URI") - maxConns = flag.Uint("max-conns", 5000, "Limit on the number of concurrent connections to the HTTP, HTTPS or proxy listeners") - insecureCiphers = flag.Bool("insecure-ciphers", false, "Use default list of cipher suites, may contain insecure ones like 3DES and RC4") - tlsMinVersion = flag.String("tls-min-version", "tls1.2", tlsconfig.FlagUsage("min")) - tlsMaxVersion = flag.String("tls-max-version", "", tlsconfig.FlagUsage("max")) + pagesRootCert = flag.String("root-cert", "", "The default path to file certificate to serve static pages") + pagesRootKey = flag.String("root-key", "", "The default path to file certificate to serve static pages") + redirectHTTP = flag.Bool("redirect-http", false, "Redirect pages from HTTP to HTTPS") + useHTTP2 = flag.Bool("use-http2", true, "Enable HTTP2 support") + pagesRoot = flag.String("pages-root", "shared/pages", "The directory where pages are stored") + pagesDomain = flag.String("pages-domain", "gitlab-example.com", "The domain to serve static pages") + artifactsServer = flag.String("artifacts-server", "", "API URL to proxy artifact requests to, e.g.: 'https://gitlab.com/api/v4'") + artifactsServerTimeout = flag.Int("artifacts-server-timeout", 10, "Timeout (in seconds) for a proxied request to the artifacts server") + pagesStatus = flag.String("pages-status", "", "The url path for a status page, e.g., /@status") + metricsAddress = flag.String("metrics-address", "", "The address to listen on for metrics requests") + sentryDSN = flag.String("sentry-dsn", "", "The address for sending sentry crash reporting to") + sentryEnvironment = flag.String("sentry-environment", "", "The environment for sentry crash reporting") + daemonUID = flag.Uint("daemon-uid", 0, "Drop privileges to this user") + daemonGID = flag.Uint("daemon-gid", 0, "Drop privileges to this group") + daemonInplaceChroot = flag.Bool("daemon-inplace-chroot", false, "Fall back to a non-bind-mount chroot of -pages-root when daemonizing") + logFormat = flag.String("log-format", "text", "The log output format: 'text' or 'json'") + logVerbose = flag.Bool("log-verbose", false, "Verbose logging") + _ = flag.String("admin-secret-path", "", "DEPRECATED") + _ = flag.String("admin-unix-listener", "", "DEPRECATED") + _ = flag.String("admin-https-listener", "", "DEPRECATED") + _ = flag.String("admin-https-cert", "", "DEPRECATED") + _ = flag.String("admin-https-key", "", "DEPRECATED") + secret = flag.String("auth-secret", "", "Cookie store hash key, should be at least 32 bytes long.") + gitLabAuthServer = flag.String("auth-server", "", "DEPRECATED, use gitlab-server instead. GitLab server, for example https://www.gitlab.com") + gitLabServer = flag.String("gitlab-server", "", "GitLab server, for example https://www.gitlab.com") + internalGitLabServer = flag.String("internal-gitlab-server", "", "Internal GitLab server used for API requests, useful if you want to send that traffic over an internal load balancer, example value https://www.gitlab.com (defaults to value of gitlab-server)") + gitLabAPISecretKey = flag.String("api-secret-key", "", "File with secret key used to authenticate with the GitLab API") + gitlabClientHTTPTimeout = flag.Duration("gitlab-client-http-timeout", 10*time.Second, "GitLab API HTTP client connection timeout in seconds (default: 10s)") + gitlabClientJWTExpiry = flag.Duration("gitlab-client-jwt-expiry", 30*time.Second, "JWT Token expiry time in seconds (default: 30s)") + disableGitlabAPIConfigSource = flag.Bool("disable-gitlab-api-config-source", false, "Disable use of GitLab's API based domain configuration source") + clientID = flag.String("auth-client-id", "", "GitLab application Client ID") + clientSecret = flag.String("auth-client-secret", "", "GitLab application Client Secret") + redirectURI = flag.String("auth-redirect-uri", "", "GitLab application redirect URI") + maxConns = flag.Uint("max-conns", 5000, "Limit on the number of concurrent connections to the HTTP, HTTPS or proxy listeners") + insecureCiphers = flag.Bool("insecure-ciphers", false, "Use default list of cipher suites, may contain insecure ones like 3DES and RC4") + tlsMinVersion = flag.String("tls-min-version", "tls1.2", tlsconfig.FlagUsage("min")) + tlsMaxVersion = flag.String("tls-max-version", "", tlsconfig.FlagUsage("max")) disableCrossOriginRequests = flag.Bool("disable-cross-origin-requests", false, "Disable cross-origin requests") @@ -193,6 +194,7 @@ func configFromFlags() appConfig { config.InternalGitLabServer = internalGitLabServerFromFlags() config.GitlabClientHTTPTimeout = *gitlabClientHTTPTimeout config.GitlabJWTTokenExpiration = *gitlabClientJWTExpiry + config.GitlabDisableAPIConfigSource = *disableGitlabAPIConfigSource config.StoreSecret = *secret config.ClientID = *clientID config.ClientSecret = *clientSecret