From 3c865755ed6bc97f81b8a16fde1280fc19e2fd2d Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Thu, 2 Apr 2020 14:42:17 +1100 Subject: [PATCH 1/9] Add configuration flag -enable-domain-source-api Adds support for `--enable-domain-source-api` boolean default to `true`. Expects `-gitlab-server` and `-api-secret-key` to be set. This will be the default behavior for pages. However, it requires the `/api/v4/internal/pages` endpoint to be enabled on gitlab rails. A `Ping` will be done to verify the API is enabled. The daemon will default to get configuration for domains from disk if the the Ping check fails. --- app_config.go | 5 +++ internal/source/config.go | 2 +- internal/source/domains.go | 41 +++++++++++++++---------- internal/source/domains_test.go | 24 +++++++++------ internal/source/gitlab/client/client.go | 27 +++++++++++++--- internal/source/gitlab/client/config.go | 1 + internal/source/gitlab/gitlab.go | 8 +++-- internal/source/source.go | 2 +- main.go | 2 ++ 9 files changed, 78 insertions(+), 34 deletions(-) diff --git a/app_config.go b/app_config.go index 5d481bb68..03540aaaf 100644 --- a/app_config.go +++ b/app_config.go @@ -33,6 +33,7 @@ type appConfig struct { GitLabAPISecretKey []byte GitlabClientHTTPTimeout time.Duration GitlabJWTTokenExpiration time.Duration + EnableDomainSourceAPI bool ClientID string ClientSecret string RedirectURI string @@ -58,3 +59,7 @@ func (config appConfig) GitlabClientConnectionTimeout() time.Duration { func (config appConfig) GitlabJWTTokenExpiry() time.Duration { return config.GitlabJWTTokenExpiration } + +func (config appConfig) GitlabEnableSourceAPI() bool { + return config.EnableDomainSourceAPI +} diff --git a/internal/source/config.go b/internal/source/config.go index 9cf87bc65..f262fcbc9 100644 --- a/internal/source/config.go +++ b/internal/source/config.go @@ -3,5 +3,5 @@ package source import "gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab/client" // Config represents an interface that is configuration provider for client -// capable of comunicating with GitLab +// capable of communicating with GitLab type Config client.Config diff --git a/internal/source/domains.go b/internal/source/domains.go index 8de7c574e..ea004f67d 100644 --- a/internal/source/domains.go +++ b/internal/source/domains.go @@ -3,6 +3,7 @@ package source import ( "errors" "regexp" + "strings" "time" log "github.com/sirupsen/logrus" @@ -12,54 +13,62 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/internal/source/disk" "gitlab.com/gitlab-org/gitlab-pages/internal/source/domains/gitlabsourceconfig" "gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab" + "gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab/client" ) var ( gitlabSourceConfig gitlabsourceconfig.GitlabSourceConfig // serverlessDomainRegex is a regular expression we use to check if a domain - // is a serverless domain, to short circut gitlab source rollout. It can be + // is a serverless domain, to short circut gitlab Source rollout. It can be // removed after the rollout is done serverlessDomainRegex = regexp.MustCompile(`^[^.]+-[[:xdigit:]]{2}a1[[:xdigit:]]{10}f2[[:xdigit:]]{2}[[:xdigit:]]+-?.*`) ) func init() { - // Start watching the config file for domains that will use the new `gitlab` source, + // Start watching the config file for domains that will use the new `gitlab` Source, // to be removed once we switch completely to using it. go gitlabsourceconfig.WatchForGitlabSourceConfigChange(&gitlabSourceConfig, 1*time.Minute) } // Domains struct represents a map of all domains supported by pages. It is // currently using two sources during the transition to the new GitLab domains -// source. +// Source. type Domains struct { gitlab Source - disk *disk.Disk // legacy disk source + disk *disk.Disk // legacy disk Source } // NewDomains is a factory method for domains initializing a mutex. It should // not initialize `dm` as we later check the readiness by comparing it with a // nil value. func NewDomains(config Config) (*Domains, error) { - if len(config.InternalGitLabServerURL()) == 0 || len(config.GitlabAPISecret()) == 0 { + // fallback to disk if these values are empty + // TODO communicate disk source deprecation https://gitlab.com/gitlab-org/gitlab-pages/-/issues/351 + if !config.GitlabEnableSourceAPI() || len(config.InternalGitLabServerURL()) == 0 || len(config.GitlabAPISecret()) == 0 { + log.Warn("disk source will be deprecated soon, please use -enable-domain-source-api") return &Domains{disk: disk.New()}, nil } - gitlab, err := gitlab.New(config) + gl, err := gitlab.New(config) if err != nil { + if strings.Contains(err.Error(), client.ConnectionErrorMsg) { + log.WithError(err).Warn("cannot -enable-domain-source-api, defaulting to disk source") + return &Domains{disk: disk.New()}, nil + } return nil, err } return &Domains{ - gitlab: gitlab, + gitlab: gl, disk: disk.New(), }, nil } -// GetDomain retrieves a domain information from a source. We are using two -// sources here because it allows us to switch behavior and the domain source +// GetDomain retrieves a domain information from a Source. We are using two +// sources here because it allows us to switch behavior and the domain Source // for some subset of domains, to test / PoC the new GitLab Domains Source that -// we plan to use to replace the disk source. +// we plan to use to replace the disk Source. func (d *Domains) GetDomain(name string) (*domain.Domain, error) { if name == gitlabSourceConfig.Domains.Broken { return nil, errors.New("broken test domain used") @@ -68,15 +77,15 @@ func (d *Domains) GetDomain(name string) (*domain.Domain, error) { return d.source(name).GetDomain(name) } -// Read starts the disk domain source. It is DEPRECATED, because we want to -// remove it entirely when disk source gets removed. +// Read starts the disk domain Source. It is DEPRECATED, because we want to +// remove it entirely when disk Source gets removed. func (d *Domains) Read(rootDomain string) { d.disk.Read(rootDomain) } -// IsReady checks if the disk domain source managed to traverse entire pages +// IsReady checks if the disk domain Source managed to traverse entire pages // filesystem and is ready for use. It is DEPRECATED, because we want to remove -// it entirely when disk source gets removed. +// it entirely when disk Source gets removed. func (d *Domains) IsReady() bool { return d.disk.IsReady() } @@ -86,9 +95,9 @@ func (d *Domains) source(domain string) Source { return d.disk } - // This check is only needed until we enable `d.gitlab` source in all + // This check is only needed until we enable `d.gitlab` Source in all // environments (including on-premises installations) followed by removal of - // `d.disk` source. This can be safely removed afterwards. + // `d.disk` Source. This can be safely removed afterwards. if IsServerlessDomain(domain) { return d.gitlab } diff --git a/internal/source/domains_test.go b/internal/source/domains_test.go index ebafb6fc2..e85dd19c2 100644 --- a/internal/source/domains_test.go +++ b/internal/source/domains_test.go @@ -14,6 +14,7 @@ import ( type sourceConfig struct { api string secret string + enable bool } func (c sourceConfig) InternalGitLabServerURL() string { @@ -30,13 +31,16 @@ func (c sourceConfig) GitlabClientConnectionTimeout() time.Duration { func (c sourceConfig) GitlabJWTTokenExpiry() time.Duration { return 30 * time.Second } +func (c sourceConfig) GitlabEnableSourceAPI() bool { + return c.enable +} func TestDomainSources(t *testing.T) { - t.Run("when GitLab API URL has been provided", func(t *testing.T) { - domains, err := NewDomains(sourceConfig{api: "https://gitlab.com", secret: "abc"}) + t.Run("when GitLab API URL has been provided but cannot authenticate", func(t *testing.T) { + domains, err := NewDomains(sourceConfig{api: "https://gitlab.com", secret: "abc", enable: true}) require.NoError(t, err) - require.NotNil(t, domains.gitlab) + require.Nil(t, domains.gitlab) require.NotNil(t, domains.disk) }) @@ -50,7 +54,7 @@ func TestDomainSources(t *testing.T) { } func TestGetDomain(t *testing.T) { - gitlabSourceConfig.Domains.Enabled = []string{"new-source-test.gitlab.io"} + gitlabSourceConfig.Domains.Enabled = []string{"new-Source-test.gitlab.io"} gitlabSourceConfig.Domains.Broken = "pages-broken-poc.gitlab.io" t.Run("when requesting a test domain", func(t *testing.T) { @@ -100,11 +104,11 @@ func TestGetDomain(t *testing.T) { require.EqualError(t, err, "broken test domain used") }) - t.Run("when requesting a test domain in case of the source not being fully configured", func(t *testing.T) { + t.Run("when requesting a test domain in case of the Source not being fully configured", func(t *testing.T) { domains, err := NewDomains(sourceConfig{}) require.NoError(t, err) - domain, err := domains.GetDomain("new-source-test.gitlab.io") + domain, err := domains.GetDomain("new-Source-test.gitlab.io") require.Nil(t, domain) require.NoError(t, err) @@ -165,8 +169,8 @@ func TestGetDomainWithIncrementalrolloutOfGitLabSource(t *testing.T) { stickiness string domains []testDomain }{ - // domain05 should always use gitlab source, - // domain80 should use disk source + // domain05 should always use gitlab Source, + // domain80 should use disk Source "default stickiness": { stickiness: "", domains: []testDomain{ @@ -176,8 +180,8 @@ func TestGetDomainWithIncrementalrolloutOfGitLabSource(t *testing.T) { }, }, // Given that randSeed(42) will produce the following pseudo-random sequence: - // {5, 87, 68} the first and third call for domain05 should use gitlab source, - // while the second one should use disk source + // {5, 87, 68} the first and third call for domain05 should use gitlab Source, + // while the second one should use disk Source "no stickiness": { stickiness: "random", domains: []testDomain{ diff --git a/internal/source/gitlab/client/client.go b/internal/source/gitlab/client/client.go index bfe425b92..4c1d73388 100644 --- a/internal/source/gitlab/client/client.go +++ b/internal/source/gitlab/client/client.go @@ -18,6 +18,11 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/metrics" ) +// ConnectionErrorMsg to be returned with `gc.preflightCheck` if pages +// cannot contact /api/v4/internal/pages either because of a 404 (disabled) +// or a 401 given that the credentials used are wrong +const ConnectionErrorMsg = "failed to connect to internal pages API" + // Client is a HTTP client to access Pages internal API type Client struct { secretKey []byte @@ -70,13 +75,17 @@ func (gc *Client) Resolve(ctx context.Context, host string) *api.Lookup { return &lookup } -// GetLookup returns a VirtualDomain configuration wrapped into a Lookup for a -// given host -func (gc *Client) GetLookup(ctx context.Context, host string) api.Lookup { +func (gc *Client) getLookupResponse(ctx context.Context, host string) (*http.Response, error) { params := url.Values{} params.Set("host", host) - resp, err := gc.get(ctx, "/api/v4/internal/pages", params) + return gc.get(ctx, "/api/v4/internal/pages", params) +} + +// GetLookup returns a VirtualDomain configuration wrapped into a Lookup for a +// given host +func (gc *Client) GetLookup(ctx context.Context, host string) api.Lookup { + resp, err := gc.getLookupResponse(ctx, host) if err != nil { return api.Lookup{Name: host, Error: err} } @@ -91,6 +100,16 @@ func (gc *Client) GetLookup(ctx context.Context, host string) api.Lookup { return lookup } +// Ping internal/pages API for source domain configuration can be accessed from Pages. +// Timeout is the same as -gitlab-client-http-timeout +func (gc *Client) Ping() error { + _, err := gc.getLookupResponse(context.Background(), "gitlab.com") + if err != nil { + return fmt.Errorf("%s: %v", ConnectionErrorMsg, err) + } + return nil +} + func (gc *Client) get(ctx context.Context, path string, params url.Values) (*http.Response, error) { endpoint, err := gc.endpoint(path, params) if err != nil { diff --git a/internal/source/gitlab/client/config.go b/internal/source/gitlab/client/config.go index 19a87452b..1ce0f94f8 100644 --- a/internal/source/gitlab/client/config.go +++ b/internal/source/gitlab/client/config.go @@ -9,4 +9,5 @@ type Config interface { GitlabAPISecret() []byte GitlabClientConnectionTimeout() time.Duration GitlabJWTTokenExpiry() time.Duration + GitlabEnableSourceAPI() bool } diff --git a/internal/source/gitlab/gitlab.go b/internal/source/gitlab/gitlab.go index 12da9af17..51a9e59bf 100644 --- a/internal/source/gitlab/gitlab.go +++ b/internal/source/gitlab/gitlab.go @@ -23,13 +23,17 @@ type Gitlab struct { // New returns a new instance of gitlab domain source. func New(config client.Config) (*Gitlab, error) { - client, err := client.NewFromConfig(config) + cli, err := client.NewFromConfig(config) if err != nil { return nil, err } + if err := cli.Ping(); err != nil { + return nil, err + } + // using nil for cache config will use the default values specified in internal/source/gitlab/cache/cache.go#12 - return &Gitlab{client: cache.NewCache(client, nil)}, nil + return &Gitlab{client: cache.NewCache(cli, nil)}, nil } // GetDomain return a representation of a domain that we have fetched from diff --git a/internal/source/source.go b/internal/source/source.go index 4b43b8f4b..e33b8dd12 100644 --- a/internal/source/source.go +++ b/internal/source/source.go @@ -2,7 +2,7 @@ package source import "gitlab.com/gitlab-org/gitlab-pages/internal/domain" -// Source represents an abstract interface of a domains configuration source. +// Source represents an abstract interface of a domains configuration Source. type Source interface { GetDomain(string) (*domain.Domain, error) } diff --git a/main.go b/main.go index 2614fa0bc..25cb5ac6a 100644 --- a/main.go +++ b/main.go @@ -65,6 +65,7 @@ var ( gitLabAPISecretKey = flag.String("api-secret-key", "", "File with secret key used to authenticate with the GitLab API") gitlabClientHTTPTimeout = flag.Duration("gitlab-client-http-timeout", 10*time.Second, "GitLab API HTTP client connection timeout in seconds (default: 10s)") gitlabClientJWTExpiry = flag.Duration("gitlab-client-jwt-expiry", 30*time.Second, "JWT Token expiry time in seconds (default: 30s)") + enableDomainSourceAPI = flag.Bool("enable-domain-source-api", true, "Enable use of GitLab's domain source API configuration") clientID = flag.String("auth-client-id", "", "GitLab application Client ID") clientSecret = flag.String("auth-client-secret", "", "GitLab application Client Secret") redirectURI = flag.String("auth-redirect-uri", "", "GitLab application redirect URI") @@ -193,6 +194,7 @@ func configFromFlags() appConfig { config.InternalGitLabServer = internalGitLabServerFromFlags() config.GitlabClientHTTPTimeout = *gitlabClientHTTPTimeout config.GitlabJWTTokenExpiration = *gitlabClientJWTExpiry + config.EnableDomainSourceAPI = *enableDomainSourceAPI config.StoreSecret = *secret config.ClientID = *clientID config.ClientSecret = *clientSecret -- GitLab From 2a7ba6f81529d11e6f6e075995106123dd35ce84 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Fri, 3 Apr 2020 16:56:23 +1100 Subject: [PATCH 2/9] Use internal pages enabled API endpoint --- app_config.go | 6 +++--- internal/source/domains.go | 4 ++-- internal/source/domains_test.go | 2 +- internal/source/gitlab/client/client.go | 14 +++++--------- internal/source/gitlab/client/config.go | 2 +- main.go | 4 ++-- 6 files changed, 14 insertions(+), 18 deletions(-) diff --git a/app_config.go b/app_config.go index 03540aaaf..3c5257d32 100644 --- a/app_config.go +++ b/app_config.go @@ -33,7 +33,7 @@ type appConfig struct { GitLabAPISecretKey []byte GitlabClientHTTPTimeout time.Duration GitlabJWTTokenExpiration time.Duration - EnableDomainSourceAPI bool + DisableDomainSourceAPI bool ClientID string ClientSecret string RedirectURI string @@ -60,6 +60,6 @@ func (config appConfig) GitlabJWTTokenExpiry() time.Duration { return config.GitlabJWTTokenExpiration } -func (config appConfig) GitlabEnableSourceAPI() bool { - return config.EnableDomainSourceAPI +func (config appConfig) GitlabDisableDomainConfiguration() bool { + return config.DisableDomainSourceAPI } diff --git a/internal/source/domains.go b/internal/source/domains.go index ea004f67d..fbad24d20 100644 --- a/internal/source/domains.go +++ b/internal/source/domains.go @@ -45,8 +45,8 @@ type Domains struct { func NewDomains(config Config) (*Domains, error) { // fallback to disk if these values are empty // TODO communicate disk source deprecation https://gitlab.com/gitlab-org/gitlab-pages/-/issues/351 - if !config.GitlabEnableSourceAPI() || len(config.InternalGitLabServerURL()) == 0 || len(config.GitlabAPISecret()) == 0 { - log.Warn("disk source will be deprecated soon, please use -enable-domain-source-api") + if config.GitlabDisableDomainConfiguration() || len(config.InternalGitLabServerURL()) == 0 || len(config.GitlabAPISecret()) == 0 { + log.Warn("disk source will be deprecated soon see TODO") return &Domains{disk: disk.New()}, nil } diff --git a/internal/source/domains_test.go b/internal/source/domains_test.go index e85dd19c2..11c11211c 100644 --- a/internal/source/domains_test.go +++ b/internal/source/domains_test.go @@ -31,7 +31,7 @@ func (c sourceConfig) GitlabClientConnectionTimeout() time.Duration { func (c sourceConfig) GitlabJWTTokenExpiry() time.Duration { return 30 * time.Second } -func (c sourceConfig) GitlabEnableSourceAPI() bool { +func (c sourceConfig) GitlabDisableDomainConfiguration() bool { return c.enable } diff --git a/internal/source/gitlab/client/client.go b/internal/source/gitlab/client/client.go index 4c1d73388..809aae9d4 100644 --- a/internal/source/gitlab/client/client.go +++ b/internal/source/gitlab/client/client.go @@ -75,17 +75,13 @@ func (gc *Client) Resolve(ctx context.Context, host string) *api.Lookup { return &lookup } -func (gc *Client) getLookupResponse(ctx context.Context, host string) (*http.Response, error) { - params := url.Values{} - params.Set("host", host) - - return gc.get(ctx, "/api/v4/internal/pages", params) -} - // GetLookup returns a VirtualDomain configuration wrapped into a Lookup for a // given host func (gc *Client) GetLookup(ctx context.Context, host string) api.Lookup { - resp, err := gc.getLookupResponse(ctx, host) + params := url.Values{} + params.Set("host", host) + + resp, err := gc.get(ctx, "/api/v4/internal/pages", params) if err != nil { return api.Lookup{Name: host, Error: err} } @@ -103,7 +99,7 @@ func (gc *Client) GetLookup(ctx context.Context, host string) api.Lookup { // Ping internal/pages API for source domain configuration can be accessed from Pages. // Timeout is the same as -gitlab-client-http-timeout func (gc *Client) Ping() error { - _, err := gc.getLookupResponse(context.Background(), "gitlab.com") + _, err := gc.get(context.Background(), "/api/v4/internal/pages/enabled", url.Values{}) if err != nil { return fmt.Errorf("%s: %v", ConnectionErrorMsg, err) } diff --git a/internal/source/gitlab/client/config.go b/internal/source/gitlab/client/config.go index 1ce0f94f8..02fe86f30 100644 --- a/internal/source/gitlab/client/config.go +++ b/internal/source/gitlab/client/config.go @@ -9,5 +9,5 @@ type Config interface { GitlabAPISecret() []byte GitlabClientConnectionTimeout() time.Duration GitlabJWTTokenExpiry() time.Duration - GitlabEnableSourceAPI() bool + GitlabDisableDomainConfiguration() bool } diff --git a/main.go b/main.go index 25cb5ac6a..159e7c807 100644 --- a/main.go +++ b/main.go @@ -65,7 +65,7 @@ var ( gitLabAPISecretKey = flag.String("api-secret-key", "", "File with secret key used to authenticate with the GitLab API") gitlabClientHTTPTimeout = flag.Duration("gitlab-client-http-timeout", 10*time.Second, "GitLab API HTTP client connection timeout in seconds (default: 10s)") gitlabClientJWTExpiry = flag.Duration("gitlab-client-jwt-expiry", 30*time.Second, "JWT Token expiry time in seconds (default: 30s)") - enableDomainSourceAPI = flag.Bool("enable-domain-source-api", true, "Enable use of GitLab's domain source API configuration") + disableDomainSourceAPI = flag.Bool("disable-gitlab-config-source", false, "Disable use of GitLab's domain source API configuration") clientID = flag.String("auth-client-id", "", "GitLab application Client ID") clientSecret = flag.String("auth-client-secret", "", "GitLab application Client Secret") redirectURI = flag.String("auth-redirect-uri", "", "GitLab application redirect URI") @@ -194,7 +194,7 @@ func configFromFlags() appConfig { config.InternalGitLabServer = internalGitLabServerFromFlags() config.GitlabClientHTTPTimeout = *gitlabClientHTTPTimeout config.GitlabJWTTokenExpiration = *gitlabClientJWTExpiry - config.EnableDomainSourceAPI = *enableDomainSourceAPI + config.DisableDomainSourceAPI = *disableDomainSourceAPI config.StoreSecret = *secret config.ClientID = *clientID config.ClientSecret = *clientSecret -- GitLab From 746480ebc2c10a3b3233016be158bc8437408872 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Fri, 3 Apr 2020 17:06:34 +1100 Subject: [PATCH 3/9] Undo rename source captialization --- internal/source/domains.go | 30 +++++++++++++++--------------- internal/source/domains_test.go | 12 ++++++------ internal/source/source.go | 2 +- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/internal/source/domains.go b/internal/source/domains.go index fbad24d20..f4a181c46 100644 --- a/internal/source/domains.go +++ b/internal/source/domains.go @@ -20,23 +20,23 @@ var ( gitlabSourceConfig gitlabsourceconfig.GitlabSourceConfig // serverlessDomainRegex is a regular expression we use to check if a domain - // is a serverless domain, to short circut gitlab Source rollout. It can be + // is a serverless domain, to short circuit gitlab source rollout. It can be // removed after the rollout is done serverlessDomainRegex = regexp.MustCompile(`^[^.]+-[[:xdigit:]]{2}a1[[:xdigit:]]{10}f2[[:xdigit:]]{2}[[:xdigit:]]+-?.*`) ) func init() { - // Start watching the config file for domains that will use the new `gitlab` Source, + // Start watching the config file for domains that will use the new `gitlab` source, // to be removed once we switch completely to using it. go gitlabsourceconfig.WatchForGitlabSourceConfigChange(&gitlabSourceConfig, 1*time.Minute) } // Domains struct represents a map of all domains supported by pages. It is // currently using two sources during the transition to the new GitLab domains -// Source. +// source. type Domains struct { gitlab Source - disk *disk.Disk // legacy disk Source + disk *disk.Disk // legacy disk source } // NewDomains is a factory method for domains initializing a mutex. It should @@ -46,14 +46,14 @@ func NewDomains(config Config) (*Domains, error) { // fallback to disk if these values are empty // TODO communicate disk source deprecation https://gitlab.com/gitlab-org/gitlab-pages/-/issues/351 if config.GitlabDisableDomainConfiguration() || len(config.InternalGitLabServerURL()) == 0 || len(config.GitlabAPISecret()) == 0 { - log.Warn("disk source will be deprecated soon see TODO") + log.Warn("disk source will be deprecated soon see https://gitlab.com/gitlab-org/gitlab-pages/-/issues/351") return &Domains{disk: disk.New()}, nil } gl, err := gitlab.New(config) if err != nil { if strings.Contains(err.Error(), client.ConnectionErrorMsg) { - log.WithError(err).Warn("cannot -enable-domain-source-api, defaulting to disk source") + log.WithError(err).Warn("could not reach /api/v4/internal/pages/enabled endpoint, defaulting to disk source") return &Domains{disk: disk.New()}, nil } return nil, err @@ -65,10 +65,10 @@ func NewDomains(config Config) (*Domains, error) { }, nil } -// GetDomain retrieves a domain information from a Source. We are using two -// sources here because it allows us to switch behavior and the domain Source +// GetDomain retrieves a domain information from a source. We are using two +// sources here because it allows us to switch behavior and the domain source // for some subset of domains, to test / PoC the new GitLab Domains Source that -// we plan to use to replace the disk Source. +// we plan to use to replace the disk source. func (d *Domains) GetDomain(name string) (*domain.Domain, error) { if name == gitlabSourceConfig.Domains.Broken { return nil, errors.New("broken test domain used") @@ -77,15 +77,15 @@ func (d *Domains) GetDomain(name string) (*domain.Domain, error) { return d.source(name).GetDomain(name) } -// Read starts the disk domain Source. It is DEPRECATED, because we want to -// remove it entirely when disk Source gets removed. +// Read starts the disk domain source. It is DEPRECATED, because we want to +// remove it entirely when disk source gets removed. func (d *Domains) Read(rootDomain string) { d.disk.Read(rootDomain) } -// IsReady checks if the disk domain Source managed to traverse entire pages +// IsReady checks if the disk domain source managed to traverse entire pages // filesystem and is ready for use. It is DEPRECATED, because we want to remove -// it entirely when disk Source gets removed. +// it entirely when disk source gets removed. func (d *Domains) IsReady() bool { return d.disk.IsReady() } @@ -95,9 +95,9 @@ func (d *Domains) source(domain string) Source { return d.disk } - // This check is only needed until we enable `d.gitlab` Source in all + // This check is only needed until we enable `d.gitlab` source in all // environments (including on-premises installations) followed by removal of - // `d.disk` Source. This can be safely removed afterwards. + // `d.disk` source. This can be safely removed afterwards. if IsServerlessDomain(domain) { return d.gitlab } diff --git a/internal/source/domains_test.go b/internal/source/domains_test.go index 11c11211c..ae6338bc3 100644 --- a/internal/source/domains_test.go +++ b/internal/source/domains_test.go @@ -54,7 +54,7 @@ func TestDomainSources(t *testing.T) { } func TestGetDomain(t *testing.T) { - gitlabSourceConfig.Domains.Enabled = []string{"new-Source-test.gitlab.io"} + gitlabSourceConfig.Domains.Enabled = []string{"new-source-test.gitlab.io"} gitlabSourceConfig.Domains.Broken = "pages-broken-poc.gitlab.io" t.Run("when requesting a test domain", func(t *testing.T) { @@ -108,7 +108,7 @@ func TestGetDomain(t *testing.T) { domains, err := NewDomains(sourceConfig{}) require.NoError(t, err) - domain, err := domains.GetDomain("new-Source-test.gitlab.io") + domain, err := domains.GetDomain("new-source-test.gitlab.io") require.Nil(t, domain) require.NoError(t, err) @@ -169,8 +169,8 @@ func TestGetDomainWithIncrementalrolloutOfGitLabSource(t *testing.T) { stickiness string domains []testDomain }{ - // domain05 should always use gitlab Source, - // domain80 should use disk Source + // domain05 should always use gitlab source, + // domain80 should use disk source "default stickiness": { stickiness: "", domains: []testDomain{ @@ -180,8 +180,8 @@ func TestGetDomainWithIncrementalrolloutOfGitLabSource(t *testing.T) { }, }, // Given that randSeed(42) will produce the following pseudo-random sequence: - // {5, 87, 68} the first and third call for domain05 should use gitlab Source, - // while the second one should use disk Source + // {5, 87, 68} the first and third call for domain05 should use gitlab source, + // while the second one should use disk source "no stickiness": { stickiness: "random", domains: []testDomain{ diff --git a/internal/source/source.go b/internal/source/source.go index e33b8dd12..4b43b8f4b 100644 --- a/internal/source/source.go +++ b/internal/source/source.go @@ -2,7 +2,7 @@ package source import "gitlab.com/gitlab-org/gitlab-pages/internal/domain" -// Source represents an abstract interface of a domains configuration Source. +// Source represents an abstract interface of a domains configuration source. type Source interface { GetDomain(string) (*domain.Domain, error) } -- GitLab From 2d678fb4c27f89e3e08f3e94b2a9387a2f47cf3b Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Tue, 7 Apr 2020 13:47:08 +1000 Subject: [PATCH 4/9] Use internal pages status API --- internal/source/domains.go | 2 +- internal/source/gitlab/client/client.go | 7 ++-- internal/source/gitlab/client/client_test.go | 44 +++++++++++++++++++- internal/source/gitlab/gitlab.go | 2 +- 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/internal/source/domains.go b/internal/source/domains.go index f4a181c46..49fedaf1b 100644 --- a/internal/source/domains.go +++ b/internal/source/domains.go @@ -53,7 +53,7 @@ func NewDomains(config Config) (*Domains, error) { gl, err := gitlab.New(config) if err != nil { if strings.Contains(err.Error(), client.ConnectionErrorMsg) { - log.WithError(err).Warn("could not reach /api/v4/internal/pages/enabled endpoint, defaulting to disk source") + log.WithError(err).Warn("could not reach /api/v4/internal/pages/status, defaulting to disk source") return &Domains{disk: disk.New()}, nil } return nil, err diff --git a/internal/source/gitlab/client/client.go b/internal/source/gitlab/client/client.go index 809aae9d4..e08c3956d 100644 --- a/internal/source/gitlab/client/client.go +++ b/internal/source/gitlab/client/client.go @@ -96,10 +96,11 @@ func (gc *Client) GetLookup(ctx context.Context, host string) api.Lookup { return lookup } -// Ping internal/pages API for source domain configuration can be accessed from Pages. +// Status checks that Pages can reach the rails internal Pages API +// for source domain configuration. // Timeout is the same as -gitlab-client-http-timeout -func (gc *Client) Ping() error { - _, err := gc.get(context.Background(), "/api/v4/internal/pages/enabled", url.Values{}) +func (gc *Client) Status() error { + _, err := gc.get(context.Background(), "/api/v4/internal/pages/status", url.Values{}) if err != nil { return fmt.Errorf("%s: %v", ConnectionErrorMsg, err) } diff --git a/internal/source/gitlab/client/client_test.go b/internal/source/gitlab/client/client_test.go index 58729153b..86d20b32d 100644 --- a/internal/source/gitlab/client/client_test.go +++ b/internal/source/gitlab/client/client_test.go @@ -6,10 +6,11 @@ import ( "fmt" "net/http" "net/http/httptest" + "strconv" "testing" "time" - jwt "github.com/dgrijalva/jwt-go" + "github.com/dgrijalva/jwt-go" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitlab-pages/internal/fixture" @@ -195,6 +196,47 @@ func TestGetVirtualDomainAuthenticatedRequest(t *testing.T) { require.Equal(t, "mygroup/myproject/public/", lookupPath.Source.Path) } +func TestClientStatus(t *testing.T) { + tests := []struct { + name string + status int + wantErr bool + }{ + { + name: "api_enabled", + status: http.StatusNoContent, + wantErr: false, + }, + { + name: "api_unauthorized", + status: http.StatusUnauthorized, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v4/internal/pages/status", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tt.status) + }) + server := httptest.NewServer(mux) + defer server.Close() + + client := defaultClient(t, server.URL) + + err := client.Status() + require.Equal(t, tt.wantErr, err != nil) + + if tt.wantErr { + require.NotNil(t, err) + require.Contains(t, err.Error(), strconv.Itoa(tt.status)) + } + }) + } + +} + func validateToken(t *testing.T, tokenString string) { t.Helper() token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) { diff --git a/internal/source/gitlab/gitlab.go b/internal/source/gitlab/gitlab.go index 51a9e59bf..29d25fc3e 100644 --- a/internal/source/gitlab/gitlab.go +++ b/internal/source/gitlab/gitlab.go @@ -28,7 +28,7 @@ func New(config client.Config) (*Gitlab, error) { return nil, err } - if err := cli.Ping(); err != nil { + if err := cli.Status(); err != nil { return nil, err } -- GitLab From 4f0aa9dc5d7c15b1202508208e8de787c8efbc40 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Wed, 22 Apr 2020 16:51:36 +1000 Subject: [PATCH 5/9] Disable disk source on domain init --- internal/source/domains.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/source/domains.go b/internal/source/domains.go index 49fedaf1b..ddaf369a9 100644 --- a/internal/source/domains.go +++ b/internal/source/domains.go @@ -53,7 +53,7 @@ func NewDomains(config Config) (*Domains, error) { gl, err := gitlab.New(config) if err != nil { if strings.Contains(err.Error(), client.ConnectionErrorMsg) { - log.WithError(err).Warn("could not reach /api/v4/internal/pages/status, defaulting to disk source") + log.Warn("GitLab API is not configured https://gitlab.com/gitlab-org/gitlab-pages/-/issues/351") return &Domains{disk: disk.New()}, nil } return nil, err @@ -61,7 +61,6 @@ func NewDomains(config Config) (*Domains, error) { return &Domains{ gitlab: gl, - disk: disk.New(), }, nil } @@ -80,14 +79,16 @@ func (d *Domains) GetDomain(name string) (*domain.Domain, error) { // Read starts the disk domain source. It is DEPRECATED, because we want to // remove it entirely when disk source gets removed. func (d *Domains) Read(rootDomain string) { - d.disk.Read(rootDomain) + if d.disk != nil { + d.disk.Read(rootDomain) + } } // IsReady checks if the disk domain source managed to traverse entire pages // filesystem and is ready for use. It is DEPRECATED, because we want to remove // it entirely when disk source gets removed. func (d *Domains) IsReady() bool { - return d.disk.IsReady() + return d.disk == nil || d.disk.IsReady() } func (d *Domains) source(domain string) Source { @@ -116,7 +117,7 @@ func (d *Domains) source(domain string) Source { return d.disk } - if enabled { + if enabled || d.disk == nil { return d.gitlab } -- GitLab From 4dd52988ef38ec11311f8d4267ad050858acec0c Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Mon, 4 May 2020 11:24:29 +1000 Subject: [PATCH 6/9] Rename flag to disable-gitlab-api-config-source Make the flag more verbose to explain what it's supposed to do. Update issue https://gitlab.com/gitlab-org/gitlab/-/issues/210010 in the references of the code. Address review comments. --- app_config.go | 30 +++++----- internal/source/domains.go | 11 ++-- internal/source/domains_test.go | 2 +- internal/source/gitlab/client/client.go | 4 +- internal/source/gitlab/client/config.go | 2 +- main.go | 76 ++++++++++++------------- 6 files changed, 64 insertions(+), 61 deletions(-) diff --git a/app_config.go b/app_config.go index 3c5257d32..3bc9408c3 100644 --- a/app_config.go +++ b/app_config.go @@ -27,19 +27,19 @@ type appConfig struct { LogFormat string LogVerbose bool - StoreSecret string - GitLabServer string - InternalGitLabServer string - GitLabAPISecretKey []byte - GitlabClientHTTPTimeout time.Duration - GitlabJWTTokenExpiration time.Duration - DisableDomainSourceAPI bool - ClientID string - ClientSecret string - RedirectURI string - SentryDSN string - SentryEnvironment string - CustomHeaders []string + StoreSecret string + GitLabServer string + InternalGitLabServer string + GitLabAPISecretKey []byte + GitlabClientHTTPTimeout time.Duration + GitlabJWTTokenExpiration time.Duration + GitlabDisableAPIConfigSource bool + ClientID string + ClientSecret string + RedirectURI string + SentryDSN string + SentryEnvironment string + CustomHeaders []string } // InternalGitLabServerURL returns URL to a GitLab instance. @@ -60,6 +60,6 @@ func (config appConfig) GitlabJWTTokenExpiry() time.Duration { return config.GitlabJWTTokenExpiration } -func (config appConfig) GitlabDisableDomainConfiguration() bool { - return config.DisableDomainSourceAPI +func (config appConfig) GitlabDisableAPIConfigurationSource() bool { + return config.GitlabDisableAPIConfigSource } diff --git a/internal/source/domains.go b/internal/source/domains.go index ddaf369a9..6be69df67 100644 --- a/internal/source/domains.go +++ b/internal/source/domains.go @@ -39,21 +39,24 @@ type Domains struct { disk *disk.Disk // legacy disk source } +func useDiskConfigSource(config Config) bool { + return config.GitlabDisableAPIConfigurationSource() || len(config.InternalGitLabServerURL()) == 0 || len(config.GitlabAPISecret()) == 0 +} + // NewDomains is a factory method for domains initializing a mutex. It should // not initialize `dm` as we later check the readiness by comparing it with a // nil value. func NewDomains(config Config) (*Domains, error) { // fallback to disk if these values are empty - // TODO communicate disk source deprecation https://gitlab.com/gitlab-org/gitlab-pages/-/issues/351 - if config.GitlabDisableDomainConfiguration() || len(config.InternalGitLabServerURL()) == 0 || len(config.GitlabAPISecret()) == 0 { - log.Warn("disk source will be deprecated soon see https://gitlab.com/gitlab-org/gitlab-pages/-/issues/351") + if useDiskConfigSource(config) { + log.Warn("disk source will be deprecated soon https://gitlab.com/gitlab-org/gitlab/-/issues/210010") return &Domains{disk: disk.New()}, nil } gl, err := gitlab.New(config) if err != nil { if strings.Contains(err.Error(), client.ConnectionErrorMsg) { - log.Warn("GitLab API is not configured https://gitlab.com/gitlab-org/gitlab-pages/-/issues/351") + log.WithError(err).Warn("GitLab API is not configured https://gitlab.com/gitlab-org/gitlab/-/issues/210010") return &Domains{disk: disk.New()}, nil } return nil, err diff --git a/internal/source/domains_test.go b/internal/source/domains_test.go index ae6338bc3..a9d165cb4 100644 --- a/internal/source/domains_test.go +++ b/internal/source/domains_test.go @@ -31,7 +31,7 @@ func (c sourceConfig) GitlabClientConnectionTimeout() time.Duration { func (c sourceConfig) GitlabJWTTokenExpiry() time.Duration { return 30 * time.Second } -func (c sourceConfig) GitlabDisableDomainConfiguration() bool { +func (c sourceConfig) GitlabDisableAPIConfigurationSource() bool { return c.enable } diff --git a/internal/source/gitlab/client/client.go b/internal/source/gitlab/client/client.go index e08c3956d..b9c24496c 100644 --- a/internal/source/gitlab/client/client.go +++ b/internal/source/gitlab/client/client.go @@ -18,8 +18,8 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/metrics" ) -// ConnectionErrorMsg to be returned with `gc.preflightCheck` if pages -// cannot contact /api/v4/internal/pages either because of a 404 (disabled) +// ConnectionErrorMsg to be returned with `gc.Status` if pages +// cannot contact /api/v4/internal/pages/status either because of a 404 (disabled) // or a 401 given that the credentials used are wrong const ConnectionErrorMsg = "failed to connect to internal pages API" diff --git a/internal/source/gitlab/client/config.go b/internal/source/gitlab/client/config.go index 02fe86f30..b760d6bed 100644 --- a/internal/source/gitlab/client/config.go +++ b/internal/source/gitlab/client/config.go @@ -9,5 +9,5 @@ type Config interface { GitlabAPISecret() []byte GitlabClientConnectionTimeout() time.Duration GitlabJWTTokenExpiry() time.Duration - GitlabDisableDomainConfiguration() bool + GitlabDisableAPIConfigurationSource() bool } diff --git a/main.go b/main.go index 159e7c807..008da7d16 100644 --- a/main.go +++ b/main.go @@ -36,43 +36,43 @@ func init() { } var ( - pagesRootCert = flag.String("root-cert", "", "The default path to file certificate to serve static pages") - pagesRootKey = flag.String("root-key", "", "The default path to file certificate to serve static pages") - redirectHTTP = flag.Bool("redirect-http", false, "Redirect pages from HTTP to HTTPS") - useHTTP2 = flag.Bool("use-http2", true, "Enable HTTP2 support") - pagesRoot = flag.String("pages-root", "shared/pages", "The directory where pages are stored") - pagesDomain = flag.String("pages-domain", "gitlab-example.com", "The domain to serve static pages") - artifactsServer = flag.String("artifacts-server", "", "API URL to proxy artifact requests to, e.g.: 'https://gitlab.com/api/v4'") - artifactsServerTimeout = flag.Int("artifacts-server-timeout", 10, "Timeout (in seconds) for a proxied request to the artifacts server") - pagesStatus = flag.String("pages-status", "", "The url path for a status page, e.g., /@status") - metricsAddress = flag.String("metrics-address", "", "The address to listen on for metrics requests") - sentryDSN = flag.String("sentry-dsn", "", "The address for sending sentry crash reporting to") - sentryEnvironment = flag.String("sentry-environment", "", "The environment for sentry crash reporting") - daemonUID = flag.Uint("daemon-uid", 0, "Drop privileges to this user") - daemonGID = flag.Uint("daemon-gid", 0, "Drop privileges to this group") - daemonInplaceChroot = flag.Bool("daemon-inplace-chroot", false, "Fall back to a non-bind-mount chroot of -pages-root when daemonizing") - logFormat = flag.String("log-format", "text", "The log output format: 'text' or 'json'") - logVerbose = flag.Bool("log-verbose", false, "Verbose logging") - _ = flag.String("admin-secret-path", "", "DEPRECATED") - _ = flag.String("admin-unix-listener", "", "DEPRECATED") - _ = flag.String("admin-https-listener", "", "DEPRECATED") - _ = flag.String("admin-https-cert", "", "DEPRECATED") - _ = flag.String("admin-https-key", "", "DEPRECATED") - secret = flag.String("auth-secret", "", "Cookie store hash key, should be at least 32 bytes long") - gitLabAuthServer = flag.String("auth-server", "", "DEPRECATED, use gitlab-server instead. GitLab server, for example https://www.gitlab.com") - gitLabServer = flag.String("gitlab-server", "", "GitLab server, for example https://www.gitlab.com") - internalGitLabServer = flag.String("internal-gitlab-server", "", "Internal GitLab server used for API requests, useful if you want to send that traffic over an internal load balancer, example value https://www.gitlab.com (defaults to value of gitlab-server)") - gitLabAPISecretKey = flag.String("api-secret-key", "", "File with secret key used to authenticate with the GitLab API") - gitlabClientHTTPTimeout = flag.Duration("gitlab-client-http-timeout", 10*time.Second, "GitLab API HTTP client connection timeout in seconds (default: 10s)") - gitlabClientJWTExpiry = flag.Duration("gitlab-client-jwt-expiry", 30*time.Second, "JWT Token expiry time in seconds (default: 30s)") - disableDomainSourceAPI = flag.Bool("disable-gitlab-config-source", false, "Disable use of GitLab's domain source API configuration") - clientID = flag.String("auth-client-id", "", "GitLab application Client ID") - clientSecret = flag.String("auth-client-secret", "", "GitLab application Client Secret") - redirectURI = flag.String("auth-redirect-uri", "", "GitLab application redirect URI") - maxConns = flag.Uint("max-conns", 5000, "Limit on the number of concurrent connections to the HTTP, HTTPS or proxy listeners") - insecureCiphers = flag.Bool("insecure-ciphers", false, "Use default list of cipher suites, may contain insecure ones like 3DES and RC4") - tlsMinVersion = flag.String("tls-min-version", "tls1.2", tlsconfig.FlagUsage("min")) - tlsMaxVersion = flag.String("tls-max-version", "", tlsconfig.FlagUsage("max")) + pagesRootCert = flag.String("root-cert", "", "The default path to file certificate to serve static pages") + pagesRootKey = flag.String("root-key", "", "The default path to file certificate to serve static pages") + redirectHTTP = flag.Bool("redirect-http", false, "Redirect pages from HTTP to HTTPS") + useHTTP2 = flag.Bool("use-http2", true, "Enable HTTP2 support") + pagesRoot = flag.String("pages-root", "shared/pages", "The directory where pages are stored") + pagesDomain = flag.String("pages-domain", "gitlab-example.com", "The domain to serve static pages") + artifactsServer = flag.String("artifacts-server", "", "API URL to proxy artifact requests to, e.g.: 'https://gitlab.com/api/v4'") + artifactsServerTimeout = flag.Int("artifacts-server-timeout", 10, "Timeout (in seconds) for a proxied request to the artifacts server") + pagesStatus = flag.String("pages-status", "", "The url path for a status page, e.g., /@status") + metricsAddress = flag.String("metrics-address", "", "The address to listen on for metrics requests") + sentryDSN = flag.String("sentry-dsn", "", "The address for sending sentry crash reporting to") + sentryEnvironment = flag.String("sentry-environment", "", "The environment for sentry crash reporting") + daemonUID = flag.Uint("daemon-uid", 0, "Drop privileges to this user") + daemonGID = flag.Uint("daemon-gid", 0, "Drop privileges to this group") + daemonInplaceChroot = flag.Bool("daemon-inplace-chroot", false, "Fall back to a non-bind-mount chroot of -pages-root when daemonizing") + logFormat = flag.String("log-format", "text", "The log output format: 'text' or 'json'") + logVerbose = flag.Bool("log-verbose", false, "Verbose logging") + _ = flag.String("admin-secret-path", "", "DEPRECATED") + _ = flag.String("admin-unix-listener", "", "DEPRECATED") + _ = flag.String("admin-https-listener", "", "DEPRECATED") + _ = flag.String("admin-https-cert", "", "DEPRECATED") + _ = flag.String("admin-https-key", "", "DEPRECATED") + secret = flag.String("auth-secret", "", "Cookie store hash key, should be at least 32 bytes long.") + gitLabAuthServer = flag.String("auth-server", "", "DEPRECATED, use gitlab-server instead. GitLab server, for example https://www.gitlab.com") + gitLabServer = flag.String("gitlab-server", "", "GitLab server, for example https://www.gitlab.com") + internalGitLabServer = flag.String("internal-gitlab-server", "", "Internal GitLab server used for API requests, useful if you want to send that traffic over an internal load balancer, example value https://www.gitlab.com (defaults to value of gitlab-server)") + gitLabAPISecretKey = flag.String("api-secret-key", "", "File with secret key used to authenticate with the GitLab API") + gitlabClientHTTPTimeout = flag.Duration("gitlab-client-http-timeout", 10*time.Second, "GitLab API HTTP client connection timeout in seconds (default: 10s)") + gitlabClientJWTExpiry = flag.Duration("gitlab-client-jwt-expiry", 30*time.Second, "JWT Token expiry time in seconds (default: 30s)") + disableGitlabAPIConfigSource = flag.Bool("disable-gitlab-api-config-source", false, "Disable use of GitLab's API based domain configuration source") + clientID = flag.String("auth-client-id", "", "GitLab application Client ID") + clientSecret = flag.String("auth-client-secret", "", "GitLab application Client Secret") + redirectURI = flag.String("auth-redirect-uri", "", "GitLab application redirect URI") + maxConns = flag.Uint("max-conns", 5000, "Limit on the number of concurrent connections to the HTTP, HTTPS or proxy listeners") + insecureCiphers = flag.Bool("insecure-ciphers", false, "Use default list of cipher suites, may contain insecure ones like 3DES and RC4") + tlsMinVersion = flag.String("tls-min-version", "tls1.2", tlsconfig.FlagUsage("min")) + tlsMaxVersion = flag.String("tls-max-version", "", tlsconfig.FlagUsage("max")) disableCrossOriginRequests = flag.Bool("disable-cross-origin-requests", false, "Disable cross-origin requests") @@ -194,7 +194,7 @@ func configFromFlags() appConfig { config.InternalGitLabServer = internalGitLabServerFromFlags() config.GitlabClientHTTPTimeout = *gitlabClientHTTPTimeout config.GitlabJWTTokenExpiration = *gitlabClientJWTExpiry - config.DisableDomainSourceAPI = *disableDomainSourceAPI + config.GitlabDisableAPIConfigSource = *disableGitlabAPIConfigSource config.StoreSecret = *secret config.ClientID = *clientID config.ClientSecret = *clientSecret -- GitLab From 25dba4e19d4352ac08f35c991c6709a9b9f73a72 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Tue, 5 May 2020 15:37:16 +1000 Subject: [PATCH 7/9] Remove gitlabsourceconfig pkg Removes need of gitlabsourceconfig package as we default to use gitlab API-based configuration source instead of disk. --- acceptance_test.go | 22 +-- app.go | 2 +- helpers_test.go | 22 --- internal/source/disk/disk.go | 1 + internal/source/domains.go | 74 ++-------- .../gitlabsourceconfig/gitlabsourceconfig.go | 94 ------------- internal/source/domains_test.go | 128 +----------------- internal/source/gitlab/gitlab.go | 1 + internal/source/source_mock.go | 6 +- 9 files changed, 26 insertions(+), 324 deletions(-) delete mode 100644 internal/source/domains/gitlabsourceconfig/gitlabsourceconfig.go diff --git a/acceptance_test.go b/acceptance_test.go index 4a1b30286..3345ae5ba 100644 --- a/acceptance_test.go +++ b/acceptance_test.go @@ -1532,22 +1532,11 @@ func TestGitlabDomainsSource(t *testing.T) { source := NewGitlabDomainsSourceStub(t) defer source.Close() - gitlabSourceConfig := ` -domains: - enabled: - - new-source-test.gitlab.io - broken: pages-broken-poc.gitlab.io -` - gitlabSourceConfigFile, cleanupGitlabSourceConfigFile := CreateGitlabSourceConfigFixtureFile(t, gitlabSourceConfig) - defer cleanupGitlabSourceConfigFile() - - gitlabSourceConfigFile = "GITLAB_SOURCE_CONFIG_FILE=" + gitlabSourceConfigFile - gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t) pagesArgs := []string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey} - teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{gitlabSourceConfigFile}, pagesArgs...) + teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{}, pagesArgs...) defer teardown() t.Run("when a domain exists", func(t *testing.T) { @@ -1568,13 +1557,4 @@ domains: require.Equal(t, http.StatusNotFound, response.StatusCode) }) - - t.Run("broken domain is requested", func(t *testing.T) { - response, err := GetPageFromListener(t, httpListener, "pages-broken-poc.gitlab.io", "index.html") - require.NoError(t, err) - - defer response.Body.Close() - - require.Equal(t, http.StatusBadGateway, response.StatusCode) - }) } diff --git a/app.go b/app.go index 0d6288644..96117b9be 100644 --- a/app.go +++ b/app.go @@ -12,7 +12,7 @@ import ( log "github.com/sirupsen/logrus" "gitlab.com/gitlab-org/labkit/errortracking" labmetrics "gitlab.com/gitlab-org/labkit/metrics" - mimedb "gitlab.com/lupine/go-mimedb" + "gitlab.com/lupine/go-mimedb" "gitlab.com/gitlab-org/gitlab-pages/internal/acme" "gitlab.com/gitlab-org/gitlab-pages/internal/artifact" diff --git a/helpers_test.go b/helpers_test.go index 195c3cea8..1fa5b6bb4 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -12,7 +12,6 @@ import ( "net/http/httptest" "os" "os/exec" - "path/filepath" "strings" "testing" "time" @@ -86,27 +85,6 @@ func CreateGitLabAPISecretKeyFixtureFile(t *testing.T) (filepath string) { return secretfile.Name() } -func CreateGitlabSourceConfigFixtureFile(t *testing.T, domains string) (filename string, cleanup func()) { - configfile, err := ioutil.TempFile("shared/pages", "gitlab-source-config-*") - require.NoError(t, err) - configfile.Close() - - cleanup = func() { - os.RemoveAll(configfile.Name()) - } - - require.NoError(t, ioutil.WriteFile(configfile.Name(), []byte(domains), 0644)) - - filename, err = filepath.Abs(configfile.Name()) - require.NoError(t, err) - - if os.Getenv("TEST_DAEMONIZE") != "" { - filename = filepath.Base(filename) - } - - return filename, cleanup -} - // ListenSpec is used to point at a gitlab-pages http server, preserving the // type of port it is (http, https, proxy) type ListenSpec struct { diff --git a/internal/source/disk/disk.go b/internal/source/disk/disk.go index b79d222de..46db831fe 100644 --- a/internal/source/disk/disk.go +++ b/internal/source/disk/disk.go @@ -10,6 +10,7 @@ import ( // Disk struct represents a map of all domains supported by pages that are // stored on a disk with corresponding `config.json`. +// TODO remove disk source https://gitlab.com/gitlab-org/gitlab-pages/-/issues/379 type Disk struct { dm Map lock *sync.RWMutex diff --git a/internal/source/domains.go b/internal/source/domains.go index 6be69df67..2363c3c7e 100644 --- a/internal/source/domains.go +++ b/internal/source/domains.go @@ -2,35 +2,21 @@ package source import ( "errors" - "regexp" "strings" - "time" log "github.com/sirupsen/logrus" "gitlab.com/gitlab-org/gitlab-pages/internal/domain" - "gitlab.com/gitlab-org/gitlab-pages/internal/rollout" "gitlab.com/gitlab-org/gitlab-pages/internal/source/disk" - "gitlab.com/gitlab-org/gitlab-pages/internal/source/domains/gitlabsourceconfig" "gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab" "gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab/client" ) var ( - gitlabSourceConfig gitlabsourceconfig.GitlabSourceConfig - - // serverlessDomainRegex is a regular expression we use to check if a domain - // is a serverless domain, to short circuit gitlab source rollout. It can be - // removed after the rollout is done - serverlessDomainRegex = regexp.MustCompile(`^[^.]+-[[:xdigit:]]{2}a1[[:xdigit:]]{10}f2[[:xdigit:]]{2}[[:xdigit:]]+-?.*`) + // errSourceNotConfigured will be returned when neither disk nor gitlab sources are configured + errSourceNotConfigured = errors.New("source not configured") ) -func init() { - // Start watching the config file for domains that will use the new `gitlab` source, - // to be removed once we switch completely to using it. - go gitlabsourceconfig.WatchForGitlabSourceConfigChange(&gitlabSourceConfig, 1*time.Minute) -} - // Domains struct represents a map of all domains supported by pages. It is // currently using two sources during the transition to the new GitLab domains // source. @@ -72,11 +58,11 @@ func NewDomains(config Config) (*Domains, error) { // for some subset of domains, to test / PoC the new GitLab Domains Source that // we plan to use to replace the disk source. func (d *Domains) GetDomain(name string) (*domain.Domain, error) { - if name == gitlabSourceConfig.Domains.Broken { - return nil, errors.New("broken test domain used") + source, err := d.source() + if err != nil { + return nil, err } - - return d.source(name).GetDomain(name) + return source.GetDomain(name) } // Read starts the disk domain source. It is DEPRECATED, because we want to @@ -91,48 +77,16 @@ func (d *Domains) Read(rootDomain string) { // filesystem and is ready for use. It is DEPRECATED, because we want to remove // it entirely when disk source gets removed. func (d *Domains) IsReady() bool { + // return true if d.disk is nil while we remove all of the disk source code + // TODO https://gitlab.com/gitlab-org/gitlab-pages/-/issues/379 return d.disk == nil || d.disk.IsReady() } -func (d *Domains) source(domain string) Source { - if d.gitlab == nil { - return d.disk - } - - // This check is only needed until we enable `d.gitlab` source in all - // environments (including on-premises installations) followed by removal of - // `d.disk` source. This can be safely removed afterwards. - if IsServerlessDomain(domain) { - return d.gitlab - } - - for _, name := range gitlabSourceConfig.Domains.Enabled { - if domain == name { - return d.gitlab - } - } - - r := gitlabSourceConfig.Domains.Rollout - - enabled, err := rollout.Rollout(domain, r.Percentage, r.Stickiness) - if err != nil { - log.WithError(err).Error("Rollout error") - return d.disk +func (d *Domains) source() (Source, error) { + if d.gitlab != nil { + return d.gitlab, nil + } else if d.disk != nil { + return d.disk, nil } - - if enabled || d.disk == nil { - return d.gitlab - } - - return d.disk -} - -// IsServerlessDomain checks if a domain requested is a serverless domain we -// need to handle differently. -// -// Domain is a serverless domain when it matches `serverlessDomainRegex`. The -// regular expression is also defined on the gitlab-rails side, see -// https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/serverless/domain.rb#L7 -func IsServerlessDomain(domain string) bool { - return serverlessDomainRegex.MatchString(domain) + return nil, errSourceNotConfigured } diff --git a/internal/source/domains/gitlabsourceconfig/gitlabsourceconfig.go b/internal/source/domains/gitlabsourceconfig/gitlabsourceconfig.go deleted file mode 100644 index ebc8b4859..000000000 --- a/internal/source/domains/gitlabsourceconfig/gitlabsourceconfig.go +++ /dev/null @@ -1,94 +0,0 @@ -package gitlabsourceconfig - -import ( - "bytes" - "io/ioutil" - "os" - "time" - - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" -) - -// GitlabSourceDomains holds the domains to be used with the gitlab source -type GitlabSourceDomains struct { - Enabled []string - Broken string - Rollout GitlabSourceRollout -} - -// GitlabSourceRollout holds the rollout strategy and percentage -type GitlabSourceRollout struct { - Stickiness string - Percentage int -} - -// GitlabSourceConfig holds the configuration for the gitlab source -type GitlabSourceConfig struct { - Domains GitlabSourceDomains -} - -// UpdateFromYaml updates the config -// We use new variable here (instead of using `config` directly) -// because if `content` is empty `yaml.Unmarshal` does not update -// the fields already set. -func (config *GitlabSourceConfig) UpdateFromYaml(content []byte) error { - updated := GitlabSourceConfig{} - - err := yaml.Unmarshal(content, &updated) - if err != nil { - return err - } - - *config = updated - - log.WithFields(log.Fields{ - "Enabled domains": config.Domains.Enabled, - "Broken domain": config.Domains.Broken, - "Rollout %": config.Domains.Rollout.Percentage, - "Rollout stickiness": config.Domains.Rollout.Stickiness, - }).Info("gitlab source config updated") - - return nil -} - -// WatchForGitlabSourceConfigChange polls the filesystem and updates test domains if needed. -func WatchForGitlabSourceConfigChange(config *GitlabSourceConfig, interval time.Duration) { - var lastContent []byte - - gitlabSourceConfigFile := os.Getenv("GITLAB_SOURCE_CONFIG_FILE") - if gitlabSourceConfigFile == "" { - gitlabSourceConfigFile = ".gitlab-source-config.yml" - } - - for { - content, err := readConfig(gitlabSourceConfigFile) - if err != nil { - log.WithError(err).Warn("Failed to read gitlab source config file") - - time.Sleep(interval) - continue - } - - if !bytes.Equal(lastContent, content) { - lastContent = content - - err = config.UpdateFromYaml(content) - if err != nil { - log.WithError(err).Warn("Failed to update gitlab source config") - } - } - - time.Sleep(interval) - } -} - -func readConfig(configfile string) ([]byte, error) { - content, err := ioutil.ReadFile(configfile) - - if err != nil && !os.IsNotExist(err) { - return nil, err - } - - return content, nil -} diff --git a/internal/source/domains_test.go b/internal/source/domains_test.go index a9d165cb4..0c0147c37 100644 --- a/internal/source/domains_test.go +++ b/internal/source/domains_test.go @@ -1,14 +1,13 @@ package source import ( - "math/rand" "testing" "time" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitlab-pages/internal/domain" - "gitlab.com/gitlab-org/gitlab-pages/internal/source/disk" ) type sourceConfig struct { @@ -54,11 +53,8 @@ func TestDomainSources(t *testing.T) { } func TestGetDomain(t *testing.T) { - gitlabSourceConfig.Domains.Enabled = []string{"new-source-test.gitlab.io"} - gitlabSourceConfig.Domains.Broken = "pages-broken-poc.gitlab.io" - t.Run("when requesting a test domain", func(t *testing.T) { - testDomain := gitlabSourceConfig.Domains.Enabled[0] + testDomain := "new-source-test.gitlab.io" newSource := NewMockSource() newSource.On("GetDomain", testDomain). @@ -67,7 +63,6 @@ func TestGetDomain(t *testing.T) { defer newSource.AssertExpectations(t) domains := &Domains{ - disk: disk.New(), gitlab: newSource, } @@ -76,10 +71,10 @@ func TestGetDomain(t *testing.T) { t.Run("when requesting a non-test domain", func(t *testing.T) { newSource := NewMockSource() + newSource.On("GetDomain", mock.Anything).Return(nil, nil) defer newSource.AssertExpectations(t) domains := &Domains{ - disk: disk.New(), gitlab: newSource, } @@ -89,21 +84,6 @@ func TestGetDomain(t *testing.T) { require.Nil(t, domain) }) - t.Run("when requesting a broken test domain", func(t *testing.T) { - newSource := NewMockSource() - defer newSource.AssertExpectations(t) - - domains := &Domains{ - disk: disk.New(), - gitlab: newSource, - } - - domain, err := domains.GetDomain("pages-broken-poc.gitlab.io") - - require.Nil(t, domain) - require.EqualError(t, err, "broken test domain used") - }) - t.Run("when requesting a test domain in case of the Source not being fully configured", func(t *testing.T) { domains, err := NewDomains(sourceConfig{}) require.NoError(t, err) @@ -114,106 +94,4 @@ func TestGetDomain(t *testing.T) { require.NoError(t, err) }) - t.Run("when requesting a serverless domain", func(t *testing.T) { - testDomain := "func-aba1aabbccddeef2abaabbcc.serverless.gitlab.io" - - newSource := NewMockSource() - newSource.On("GetDomain", testDomain). - Return(&domain.Domain{Name: testDomain}, nil). - Once() - defer newSource.AssertExpectations(t) - - domains := &Domains{ - disk: disk.New(), - gitlab: newSource, - } - - domains.GetDomain(testDomain) - }) -} - -func TestIsServerlessDomain(t *testing.T) { - t.Run("when a domain is serverless domain", func(t *testing.T) { - require.True(t, IsServerlessDomain("some-function-aba1aabbccddeef2abaabbcc.serverless.gitlab.io")) - }) - - t.Run("when a domain is serverless domain with environment", func(t *testing.T) { - require.True(t, IsServerlessDomain("some-function-aba1aabbccddeef2abaabbcc-testing.serverless.gitlab.io")) - }) - - t.Run("when a domain is not a serverless domain", func(t *testing.T) { - require.False(t, IsServerlessDomain("somedomain.gitlab.io")) - }) -} - -func TestGetDomainWithIncrementalrolloutOfGitLabSource(t *testing.T) { - // This will produce the following pseudo-random sequence: 5, 87, 68 - rand.Seed(42) - - // Generates FNV hash 4091421005, 4091421005 % 100 = 5 - domain05 := "test-domain-a.com" - // Generates FNV 2643293380, 2643293380 % 100 = 80 - domain80 := "test-domain-b.com" - - diskSource := disk.New() - - gitlabSourceConfig.Domains.Rollout.Percentage = 80 - - type testDomain struct { - name string - source string - times int - } - - tests := map[string]struct { - stickiness string - domains []testDomain - }{ - // domain05 should always use gitlab source, - // domain80 should use disk source - "default stickiness": { - stickiness: "", - domains: []testDomain{ - {name: domain05, source: "gitlab"}, - {name: domain80, source: "disk"}, - {name: domain05, source: "gitlab"}, - }, - }, - // Given that randSeed(42) will produce the following pseudo-random sequence: - // {5, 87, 68} the first and third call for domain05 should use gitlab source, - // while the second one should use disk source - "no stickiness": { - stickiness: "random", - domains: []testDomain{ - {name: domain05, source: "gitlab"}, - {name: domain05, source: "disk"}, - {name: domain05, source: "gitlab"}, - }}, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - gitlabSource := NewMockSource() - for _, d := range tc.domains { - if d.source == "gitlab" { - gitlabSource.On("GetDomain", d.name). - Return(&domain.Domain{Name: d.name}, nil). - Once() - } - } - defer gitlabSource.AssertExpectations(t) - - domains := &Domains{ - disk: diskSource, - gitlab: gitlabSource, - } - - gitlabSourceConfig.Domains.Rollout.Stickiness = tc.stickiness - - for _, domain := range tc.domains { - _, err := domains.GetDomain(domain.name) - require.NoError(t, err) - } - }) - } } diff --git a/internal/source/gitlab/gitlab.go b/internal/source/gitlab/gitlab.go index 29d25fc3e..b30e81195 100644 --- a/internal/source/gitlab/gitlab.go +++ b/internal/source/gitlab/gitlab.go @@ -22,6 +22,7 @@ type Gitlab struct { } // New returns a new instance of gitlab domain source. +// It checks that it can reach the API by calling cli.Status. func New(config client.Config) (*Gitlab, error) { cli, err := client.NewFromConfig(config) if err != nil { diff --git a/internal/source/source_mock.go b/internal/source/source_mock.go index ee24d804e..e6d48d687 100644 --- a/internal/source/source_mock.go +++ b/internal/source/source_mock.go @@ -14,8 +14,12 @@ type MockSource struct { // GetDomain is a mocked function func (m *MockSource) GetDomain(name string) (*domain.Domain, error) { args := m.Called(name) + d, ok := args.Get(0).(*domain.Domain) + if !ok { + return nil, args.Error(1) + } - return args.Get(0).(*domain.Domain), args.Error(1) + return d, args.Error(1) } // NewMockSource returns a new Source mock for testing -- GitLab From 7601ea6b5deb6d3fd4809f8f2cb5482ed86741ee Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Tue, 5 May 2020 16:12:34 +1000 Subject: [PATCH 8/9] Remove yaml pkg dependency --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index 908342efd..686863071 100644 --- a/go.mod +++ b/go.mod @@ -31,5 +31,4 @@ require ( golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b golang.org/x/tools v0.0.0-20191010201905-e5ffc44a6fee gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect - gopkg.in/yaml.v2 v2.2.2 ) -- GitLab From 76f972ebc7becd09606b64d5fc6180c65f8fb6c1 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Wed, 6 May 2020 17:15:08 +1000 Subject: [PATCH 9/9] Update test domain sources Implement feedback from review. --- internal/source/domains.go | 2 +- internal/source/domains_test.go | 112 ++++++++++++++++-------- internal/source/gitlab/client/client.go | 4 +- 3 files changed, 80 insertions(+), 38 deletions(-) diff --git a/internal/source/domains.go b/internal/source/domains.go index 2363c3c7e..b3b3ade4e 100644 --- a/internal/source/domains.go +++ b/internal/source/domains.go @@ -14,7 +14,7 @@ import ( var ( // errSourceNotConfigured will be returned when neither disk nor gitlab sources are configured - errSourceNotConfigured = errors.New("source not configured") + errSourceNotConfigured = errors.New("domain source configuration not available") ) // Domains struct represents a map of all domains supported by pages. It is diff --git a/internal/source/domains_test.go b/internal/source/domains_test.go index 0c0147c37..65fbf31a6 100644 --- a/internal/source/domains_test.go +++ b/internal/source/domains_test.go @@ -1,6 +1,8 @@ package source import ( + "net/http" + "net/http/httptest" "testing" "time" @@ -11,9 +13,9 @@ import ( ) type sourceConfig struct { - api string - secret string - enable bool + api string + secret string + disable bool } func (c sourceConfig) InternalGitLabServerURL() string { @@ -31,29 +33,78 @@ func (c sourceConfig) GitlabJWTTokenExpiry() time.Duration { return 30 * time.Second } func (c sourceConfig) GitlabDisableAPIConfigurationSource() bool { - return c.enable + return c.disable } func TestDomainSources(t *testing.T) { - t.Run("when GitLab API URL has been provided but cannot authenticate", func(t *testing.T) { - domains, err := NewDomains(sourceConfig{api: "https://gitlab.com", secret: "abc", enable: true}) - require.NoError(t, err) - - require.Nil(t, domains.gitlab) - require.NotNil(t, domains.disk) - }) - - t.Run("when GitLab API has not been provided", func(t *testing.T) { - domains, err := NewDomains(sourceConfig{}) - require.NoError(t, err) + // TODO refactor test when disk source is removed https://gitlab.com/gitlab-org/gitlab-pages/-/issues/382 + tests := []struct { + name string + config sourceConfig + mock bool + status int + expectGitlab bool + expectDisk bool + }{ + { + name: "gitlab_source_on_success", + config: sourceConfig{api: "http://localhost", secret: "abc"}, + mock: true, + status: http.StatusNoContent, + expectGitlab: true, + }, + { + name: "disk_source_on_unauthorized", + config: sourceConfig{api: "http://localhost", secret: "abc"}, + mock: true, + status: http.StatusUnauthorized, + expectDisk: true, + }, + { + name: "disk_source_on_api_error", + config: sourceConfig{api: "http://localhost", secret: "abc"}, + mock: true, + status: http.StatusServiceUnavailable, + expectDisk: true, + }, + { + name: "disk_source_on_disabled_api_source", + config: sourceConfig{api: "http://localhost", secret: "abc", disable: true}, + expectDisk: true, + }, + { + name: "disk_source_on_incomplete_config", + config: sourceConfig{api: "", secret: "abc"}, + expectDisk: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.mock { + m := http.NewServeMux() + m.HandleFunc("/api/v4/internal/pages/status", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tt.status) + }) + + mockServer := httptest.NewServer(m) + defer mockServer.Close() + + tt.config.api = mockServer.URL + } + + domains, err := NewDomains(tt.config) + require.NoError(t, err) + + require.Equal(t, tt.expectGitlab, domains.gitlab != nil) + require.Equal(t, tt.expectDisk, domains.disk != nil) + }) + } - require.Nil(t, domains.gitlab) - require.NotNil(t, domains.disk) - }) } func TestGetDomain(t *testing.T) { - t.Run("when requesting a test domain", func(t *testing.T) { + t.Run("when requesting a domain that exists", func(t *testing.T) { testDomain := "new-source-test.gitlab.io" newSource := NewMockSource() @@ -66,10 +117,13 @@ func TestGetDomain(t *testing.T) { gitlab: newSource, } - domains.GetDomain(testDomain) + d, err := domains.GetDomain(testDomain) + require.NoError(t, err) + require.NotNil(t, d) + require.Equal(t, d.Name, testDomain) }) - t.Run("when requesting a non-test domain", func(t *testing.T) { + t.Run("when requesting a domain that doesn't exist", func(t *testing.T) { newSource := NewMockSource() newSource.On("GetDomain", mock.Anything).Return(nil, nil) defer newSource.AssertExpectations(t) @@ -78,20 +132,8 @@ func TestGetDomain(t *testing.T) { gitlab: newSource, } - domain, err := domains.GetDomain("domain.test.io") - + d, err := domains.GetDomain("domain.test.io") require.NoError(t, err) - require.Nil(t, domain) + require.Nil(t, d) }) - - t.Run("when requesting a test domain in case of the Source not being fully configured", func(t *testing.T) { - domains, err := NewDomains(sourceConfig{}) - require.NoError(t, err) - - domain, err := domains.GetDomain("new-source-test.gitlab.io") - - require.Nil(t, domain) - require.NoError(t, err) - }) - } diff --git a/internal/source/gitlab/client/client.go b/internal/source/gitlab/client/client.go index b9c24496c..e2f07f1e7 100644 --- a/internal/source/gitlab/client/client.go +++ b/internal/source/gitlab/client/client.go @@ -18,10 +18,10 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/metrics" ) -// ConnectionErrorMsg to be returned with `gc.Status` if pages +// ConnectionErrorMsg to be returned with `gc.Status` if Pages // cannot contact /api/v4/internal/pages/status either because of a 404 (disabled) // or a 401 given that the credentials used are wrong -const ConnectionErrorMsg = "failed to connect to internal pages API" +const ConnectionErrorMsg = "failed to connect to internal Pages API" // Client is a HTTP client to access Pages internal API type Client struct { -- GitLab