diff --git a/app.go b/app.go index 9be4409c27ee786440b65e69080d92e7dee4aea3..67b94f2ad3050eb440ce7605afc49a073eb965b7 100644 --- a/app.go +++ b/app.go @@ -7,9 +7,7 @@ import ( "net" "net/http" "os" - "strings" "sync" - "time" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/cors" @@ -28,6 +26,7 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/internal/logging" "gitlab.com/gitlab-org/gitlab-pages/internal/netutil" "gitlab.com/gitlab-org/gitlab-pages/internal/request" + "gitlab.com/gitlab-org/gitlab-pages/internal/source" ) const ( @@ -47,8 +46,7 @@ var ( type theApp struct { appConfig - dm domain.Map - lock sync.RWMutex + domains *source.Domains Artifact *artifact.Artifact Auth *auth.Auth AcmeMiddleware *acme.Middleware @@ -56,15 +54,7 @@ type theApp struct { } func (a *theApp) isReady() bool { - return a.dm != nil -} - -func (a *theApp) domain(host string) *domain.Domain { - host = strings.ToLower(host) - a.lock.RLock() - defer a.lock.RUnlock() - domain, _ := a.dm[host] - return domain + return a.domains.Ready() } func (a *theApp) ServeTLS(ch *tls.ClientHelloInfo) (*tls.Certificate, error) { @@ -106,6 +96,10 @@ func (a *theApp) getHostAndDomain(r *http.Request) (host string, domain *domain. return host, a.domain(host) } +func (a *theApp) domain(host string) *domain.Domain { + return a.domains.GetDomain(host) +} + func (a *theApp) checkAuthenticationIfNotExists(domain *domain.Domain, w http.ResponseWriter, r *http.Request) bool { if domain == nil || !domain.HasProject(r) { @@ -205,7 +199,7 @@ func (a *theApp) acmeMiddleware(handler http.Handler) http.Handler { // authMiddleware handles authentication requests func (a *theApp) authMiddleware(handler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if a.Auth.TryAuthenticate(w, r, a.dm, &a.lock) { + if a.Auth.TryAuthenticate(w, r, a.domains) { return } @@ -322,12 +316,6 @@ func (a *theApp) buildHandlerPipeline() (http.Handler, error) { return handler, nil } -func (a *theApp) UpdateDomains(dm domain.Map) { - a.lock.Lock() - defer a.lock.Unlock() - a.dm = dm -} - func (a *theApp) Run() { var wg sync.WaitGroup @@ -366,7 +354,7 @@ func (a *theApp) Run() { a.listenAdminUnix(&wg) a.listenAdminHTTPS(&wg) - go domain.Watch(a.Domain, a.UpdateDomains, time.Second) + a.domains.Watch(a.Domain) wg.Wait() } @@ -473,7 +461,8 @@ func (a *theApp) listenAdminHTTPS(wg *sync.WaitGroup) { } func runApp(config appConfig) { - a := theApp{appConfig: config} + a := theApp{appConfig: config, domains: source.NewDomains()} + err := logging.ConfigureLogging(a.LogFormat, a.LogVerbose) if err != nil { log.WithError(err).Fatal("Failed to initialize logging") diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 77bc7d8e978be73bd5ce6e7975d4cde900ad7d94..95a26250f3581ef992f6defa2f25ba2191fc7125 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -11,7 +11,6 @@ import ( "net/http" "net/url" "strings" - "sync" "time" "github.com/gorilla/securecookie" @@ -19,10 +18,10 @@ import ( log "github.com/sirupsen/logrus" "gitlab.com/gitlab-org/labkit/errortracking" - "gitlab.com/gitlab-org/gitlab-pages/internal/domain" "gitlab.com/gitlab-org/gitlab-pages/internal/httperrors" "gitlab.com/gitlab-org/gitlab-pages/internal/httptransport" "gitlab.com/gitlab-org/gitlab-pages/internal/request" + "gitlab.com/gitlab-org/gitlab-pages/internal/source" "golang.org/x/crypto/hkdf" ) @@ -108,7 +107,7 @@ func (a *Auth) checkSession(w http.ResponseWriter, r *http.Request) (*sessions.S } // TryAuthenticate tries to authenticate user and fetch access token if request is a callback to auth -func (a *Auth) TryAuthenticate(w http.ResponseWriter, r *http.Request, dm domain.Map, lock *sync.RWMutex) bool { +func (a *Auth) TryAuthenticate(w http.ResponseWriter, r *http.Request, domains *source.Domains) bool { if a == nil { return false @@ -126,7 +125,7 @@ func (a *Auth) TryAuthenticate(w http.ResponseWriter, r *http.Request, dm domain logRequest(r).Info("Receive OAuth authentication callback") - if a.handleProxyingAuth(session, w, r, dm, lock) { + if a.handleProxyingAuth(session, w, r, domains) { return true } @@ -200,16 +199,17 @@ func (a *Auth) checkAuthenticationResponse(session *sessions.Session, w http.Res http.Redirect(w, r, redirectURI, 302) } -func (a *Auth) domainAllowed(domain string, dm domain.Map, lock *sync.RWMutex) bool { - lock.RLock() - defer lock.RUnlock() +func (a *Auth) domainAllowed(domain string, domains *source.Domains) bool { + domainConfigured := (domain == a.pagesDomain) || strings.HasSuffix("."+domain, a.pagesDomain) - domain = strings.ToLower(domain) - _, present := dm[domain] - return domain == a.pagesDomain || strings.HasSuffix("."+domain, a.pagesDomain) || present + if domainConfigured { + return true + } + + return domains.HasDomain(domain) } -func (a *Auth) handleProxyingAuth(session *sessions.Session, w http.ResponseWriter, r *http.Request, dm domain.Map, lock *sync.RWMutex) bool { +func (a *Auth) handleProxyingAuth(session *sessions.Session, w http.ResponseWriter, r *http.Request, domains *source.Domains) bool { // If request is for authenticating via custom domain if shouldProxyAuth(r) { domain := r.URL.Query().Get("domain") @@ -228,7 +228,7 @@ func (a *Auth) handleProxyingAuth(session *sessions.Session, w http.ResponseWrit host = proxyurl.Host } - if !a.domainAllowed(host, dm, lock) { + if !a.domainAllowed(host, domains) { logRequest(r).WithField("domain", host).Warn("Domain is not configured") httperrors.Serve401(w) return true diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go index 8be5e8354e27881d81f3e04b217f3c30f9271ed5..ad6550ac7a6826bf92e911c2d3af370c08fe62bf 100644 --- a/internal/auth/auth_test.go +++ b/internal/auth/auth_test.go @@ -5,14 +5,13 @@ import ( "net/http" "net/http/httptest" "net/url" - "sync" "testing" "github.com/gorilla/sessions" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitlab-pages/internal/domain" "gitlab.com/gitlab-org/gitlab-pages/internal/request" + "gitlab.com/gitlab-org/gitlab-pages/internal/source" ) func createAuth(t *testing.T) *Auth { @@ -55,7 +54,7 @@ func TestTryAuthenticate(t *testing.T) { require.NoError(t, err) r := request.WithHTTPSFlag(&http.Request{URL: reqURL}, true) - require.Equal(t, false, auth.TryAuthenticate(result, r, make(domain.Map), &sync.RWMutex{})) + require.Equal(t, false, auth.TryAuthenticate(result, r, source.NewDomains())) } func TestTryAuthenticateWithError(t *testing.T) { @@ -66,7 +65,7 @@ func TestTryAuthenticateWithError(t *testing.T) { require.NoError(t, err) r := request.WithHTTPSFlag(&http.Request{URL: reqURL}, true) - require.Equal(t, true, auth.TryAuthenticate(result, r, make(domain.Map), &sync.RWMutex{})) + require.Equal(t, true, auth.TryAuthenticate(result, r, source.NewDomains())) require.Equal(t, 401, result.Code) } @@ -83,7 +82,7 @@ func TestTryAuthenticateWithCodeButInvalidState(t *testing.T) { session.Values["state"] = "state" session.Save(r, result) - require.Equal(t, true, auth.TryAuthenticate(result, r, make(domain.Map), &sync.RWMutex{})) + require.Equal(t, true, auth.TryAuthenticate(result, r, source.NewDomains())) require.Equal(t, 401, result.Code) } @@ -123,7 +122,7 @@ func testTryAuthenticateWithCodeAndState(t *testing.T, https bool) { }) result := httptest.NewRecorder() - require.Equal(t, true, auth.TryAuthenticate(result, r, make(domain.Map), &sync.RWMutex{})) + require.Equal(t, true, auth.TryAuthenticate(result, r, source.NewDomains())) require.Equal(t, 302, result.Code) require.Equal(t, "https://pages.gitlab-example.com/project/", result.Header().Get("Location")) require.Equal(t, 600, result.Result().Cookies()[0].MaxAge) diff --git a/internal/domain/config.go b/internal/domain/config.go new file mode 100644 index 0000000000000000000000000000000000000000..040b2279af2a6646d8d95f190d5ff3533af87e26 --- /dev/null +++ b/internal/domain/config.go @@ -0,0 +1,11 @@ +package domain + +// ProjectConfig holds a custom project domain configuration +type ProjectConfig struct { + DomainName string + Certificate string + Key string + HTTPSOnly bool + ProjectID uint64 + AccessControl bool +} diff --git a/internal/domain/domain.go b/internal/domain/domain.go index 2f499ff971b58095cd3fe0a6a4db07c10ab19f30..784f5d41a75fc9281f5604c0999417357c7824ce 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -16,19 +16,10 @@ import ( "golang.org/x/sys/unix" - "gitlab.com/gitlab-org/gitlab-pages/internal/host" "gitlab.com/gitlab-org/gitlab-pages/internal/httperrors" "gitlab.com/gitlab-org/gitlab-pages/internal/httputil" ) -const ( - subgroupScanLimit int = 21 - // maxProjectDepth is set to the maximum nested project depth in gitlab (21) plus 3. - // One for the project, one for the first empty element of the split (URL.Path starts with /), - // and one for the real file path - maxProjectDepth int = subgroupScanLimit + 3 -) - type locationDirectoryError struct { FullPath string RelativePath string @@ -38,20 +29,23 @@ type locationFileNoExtensionError struct { FullPath string } -type project struct { - NamespaceProject bool - HTTPSOnly bool - AccessControl bool - ID uint64 +// GroupConfig represents a per-request config for a group domain +type GroupConfig interface { + IsHTTPSOnly(*http.Request) bool + HasAccessControl(*http.Request) bool + IsNamespaceProject(*http.Request) bool + ProjectID(*http.Request) uint64 + ProjectExists(*http.Request) bool + ProjectWithSubpath(*http.Request) (string, string, error) } // Domain is a domain that gitlab-pages can serve. type Domain struct { - group + Group string + Project string - // custom domains: - projectName string - config *domainConfig + ProjectConfig *ProjectConfig + GroupConfig GroupConfig // handles group domain config certificate *tls.Certificate certificateError error @@ -60,15 +54,31 @@ type Domain struct { // String implements Stringer. func (d *Domain) String() string { - if d.group.name != "" && d.projectName != "" { - return d.group.name + "/" + d.projectName + if d.Group != "" && d.Project != "" { + return d.Group + "/" + d.Project } - if d.group.name != "" { - return d.group.name + if d.Group != "" { + return d.Group } - return d.projectName + return d.Project +} + +func (d *Domain) isCustomDomain() bool { + if d.isUnconfigured() { + panic("project config and group config should not be nil at the same time") + } + + return d.ProjectConfig != nil && d.GroupConfig == nil +} + +func (d *Domain) isUnconfigured() bool { + if d == nil { + return true + } + + return d.ProjectConfig == nil && d.GroupConfig == nil } func (l *locationDirectoryError) Error() string { @@ -106,87 +116,51 @@ func handleGZip(w http.ResponseWriter, r *http.Request, fullPath string) string return gzipPath } -// Look up a project inside the domain based on the host and path. Returns the -// project and its name (if applicable) -func (d *Domain) getProjectWithSubpath(r *http.Request) (*project, string, string) { - // Check for a project specified in the URL: http://group.gitlab.io/projectA - // If present, these projects shadow the group domain. - split := strings.SplitN(r.URL.Path, "/", maxProjectDepth) - if len(split) >= 2 { - project, projectPath, urlPath := d.digProjectWithSubpath("", split[1:]) - if project != nil { - return project, projectPath, urlPath - } - } - - // Since the URL doesn't specify a project (e.g. http://mydomain.gitlab.io), - // return the group project if it exists. - if host := host.FromRequest(r); host != "" { - if groupProject := d.projects[host]; groupProject != nil { - return groupProject, host, strings.Join(split[1:], "/") - } - } - - return nil, "", "" -} - // IsHTTPSOnly figures out if the request should be handled with HTTPS // only by looking at group and project level config. func (d *Domain) IsHTTPSOnly(r *http.Request) bool { - if d == nil { + if d.isUnconfigured() { return false } // Check custom domain config (e.g. http://example.com) - if d.config != nil { - return d.config.HTTPSOnly + if d.isCustomDomain() { + return d.ProjectConfig.HTTPSOnly } // Check projects served under the group domain, including the default one - if project, _, _ := d.getProjectWithSubpath(r); project != nil { - return project.HTTPSOnly - } - - return false + return d.GroupConfig.IsHTTPSOnly(r) } // IsAccessControlEnabled figures out if the request is to a project that has access control enabled func (d *Domain) IsAccessControlEnabled(r *http.Request) bool { - if d == nil { + if d.isUnconfigured() { return false } // Check custom domain config (e.g. http://example.com) - if d.config != nil { - return d.config.AccessControl + if d.isCustomDomain() { + return d.ProjectConfig.AccessControl } // Check projects served under the group domain, including the default one - if project, _, _ := d.getProjectWithSubpath(r); project != nil { - return project.AccessControl - } - - return false + return d.GroupConfig.HasAccessControl(r) } // HasAcmeChallenge checks domain directory contains particular acme challenge func (d *Domain) HasAcmeChallenge(token string) bool { - if d == nil { - return false - } - - if d.config == nil { + if d.isUnconfigured() || !d.isCustomDomain() { return false } - _, err := d.resolvePath(d.projectName, ".well-known/acme-challenge", token) + _, err := d.resolvePath(d.Project, ".well-known/acme-challenge", token) // there is an acme challenge on disk if err == nil { return true } - _, err = d.resolvePath(d.projectName, ".well-known/acme-challenge", token, "index.html") + _, err = d.resolvePath(d.Project, ".well-known/acme-challenge", token, "index.html") if err == nil { return true @@ -197,56 +171,44 @@ func (d *Domain) HasAcmeChallenge(token string) bool { // IsNamespaceProject figures out if the request is to a namespace project func (d *Domain) IsNamespaceProject(r *http.Request) bool { - if d == nil { + if d.isUnconfigured() { return false } // If request is to a custom domain, we do not handle it as a namespace project // as there can't be multiple projects under the same custom domain - if d.config != nil { + if d.isCustomDomain() { return false } // Check projects served under the group domain, including the default one - if project, _, _ := d.getProjectWithSubpath(r); project != nil { - return project.NamespaceProject - } - - return false + return d.GroupConfig.IsNamespaceProject(r) } // GetID figures out what is the ID of the project user tries to access func (d *Domain) GetID(r *http.Request) uint64 { - if d == nil { + if d.isUnconfigured() { return 0 } - if d.config != nil { - return d.config.ID + if d.isCustomDomain() { + return d.ProjectConfig.ProjectID } - if project, _, _ := d.getProjectWithSubpath(r); project != nil { - return project.ID - } - - return 0 + return d.GroupConfig.ProjectID(r) } // HasProject figures out if the project exists that the user tries to access func (d *Domain) HasProject(r *http.Request) bool { - if d == nil { + if d.isUnconfigured() { return false } - if d.config != nil { - return true - } - - if project, _, _ := d.getProjectWithSubpath(r); project != nil { + if d.isCustomDomain() { return true } - return false + return d.GroupConfig.ProjectExists(r) } // Detect file's content-type either by extension or mime-sniffing. @@ -341,7 +303,7 @@ func (d *Domain) serveCustomFile(w http.ResponseWriter, r *http.Request, code in // Resolve the HTTP request to a path on disk, converting requests for // directories to requests for index.html inside the directory if appropriate. func (d *Domain) resolvePath(projectName string, subPath ...string) (string, error) { - publicPath := filepath.Join(d.group.name, projectName, "public") + publicPath := filepath.Join(d.Group, projectName, "public") // Don't use filepath.Join as cleans the path, // where we want to traverse full path as supplied by user @@ -428,8 +390,8 @@ func (d *Domain) tryFile(w http.ResponseWriter, r *http.Request, projectName str } func (d *Domain) serveFileFromGroup(w http.ResponseWriter, r *http.Request) bool { - project, projectName, subPath := d.getProjectWithSubpath(r) - if project == nil { + projectName, subPath, err := d.GroupConfig.ProjectWithSubpath(r) + if err != nil { httperrors.Serve404(w) return true } @@ -442,8 +404,9 @@ func (d *Domain) serveFileFromGroup(w http.ResponseWriter, r *http.Request) bool } func (d *Domain) serveNotFoundFromGroup(w http.ResponseWriter, r *http.Request) { - project, projectName, _ := d.getProjectWithSubpath(r) - if project == nil { + projectName, _, err := d.GroupConfig.ProjectWithSubpath(r) + + if err != nil { httperrors.Serve404(w) return } @@ -459,7 +422,7 @@ func (d *Domain) serveNotFoundFromGroup(w http.ResponseWriter, r *http.Request) func (d *Domain) serveFileFromConfig(w http.ResponseWriter, r *http.Request) bool { // Try to serve file for http://host/... => /group/project/... - if d.tryFile(w, r, d.projectName, r.URL.Path) == nil { + if d.tryFile(w, r, d.Project, r.URL.Path) == nil { return true } @@ -468,7 +431,7 @@ func (d *Domain) serveFileFromConfig(w http.ResponseWriter, r *http.Request) boo func (d *Domain) serveNotFoundFromConfig(w http.ResponseWriter, r *http.Request) { // Try serving not found page for http://host/ => /group/project/404.html - if d.tryNotFound(w, r, d.projectName) == nil { + if d.tryNotFound(w, r, d.Project) == nil { return } @@ -478,13 +441,16 @@ func (d *Domain) serveNotFoundFromConfig(w http.ResponseWriter, r *http.Request) // EnsureCertificate parses the PEM-encoded certificate for the domain func (d *Domain) EnsureCertificate() (*tls.Certificate, error) { - if d.config == nil { + if d.isUnconfigured() || !d.isCustomDomain() { return nil, errors.New("tls certificates can be loaded only for pages with configuration") } d.certificateOnce.Do(func() { var cert tls.Certificate - cert, d.certificateError = tls.X509KeyPair([]byte(d.config.Certificate), []byte(d.config.Key)) + cert, d.certificateError = tls.X509KeyPair( + []byte(d.ProjectConfig.Certificate), + []byte(d.ProjectConfig.Key), + ) if d.certificateError == nil { d.certificate = &cert } @@ -495,12 +461,12 @@ func (d *Domain) EnsureCertificate() (*tls.Certificate, error) { // ServeFileHTTP implements http.Handler. Returns true if something was served, false if not. func (d *Domain) ServeFileHTTP(w http.ResponseWriter, r *http.Request) bool { - if d == nil { + if d.isUnconfigured() { httperrors.Serve404(w) return true } - if d.config != nil { + if d.isCustomDomain() { return d.serveFileFromConfig(w, r) } @@ -509,12 +475,12 @@ func (d *Domain) ServeFileHTTP(w http.ResponseWriter, r *http.Request) bool { // ServeNotFoundHTTP implements http.Handler. Serves the not found pages from the projects. func (d *Domain) ServeNotFoundHTTP(w http.ResponseWriter, r *http.Request) { - if d == nil { + if d.isUnconfigured() { httperrors.Serve404(w) return } - if d.config != nil { + if d.isCustomDomain() { d.serveNotFoundFromConfig(w, r) } else { d.serveNotFoundFromGroup(w, r) diff --git a/internal/domain/domain_test.go b/internal/domain/domain_test.go index d5db33c96feecad577b63d0b90e93a962454771f..d59b9ce39914fb6c063eb2f118a666476f735104 100644 --- a/internal/domain/domain_test.go +++ b/internal/domain/domain_test.go @@ -8,7 +8,6 @@ import ( "net/url" "os" "testing" - "time" "github.com/stretchr/testify/require" @@ -24,67 +23,14 @@ func serveFileOrNotFound(domain *Domain) http.HandlerFunc { } } -func testGroupServeHTTPHost(t *testing.T, host string) { - testGroup := &Domain{ - projectName: "", - group: group{ - name: "group", - projects: map[string]*project{ - "group.test.io": &project{}, - "group.gitlab-example.com": &project{}, - "project": &project{}, - "project2": &project{}, - }, - }, - } - - makeURL := func(path string) string { - return "http://" + host + path - } - - serve := serveFileOrNotFound(testGroup) - - require.HTTPBodyContains(t, serve, "GET", makeURL("/"), nil, "main-dir") - require.HTTPBodyContains(t, serve, "GET", makeURL("/index"), nil, "main-dir") - require.HTTPBodyContains(t, serve, "GET", makeURL("/index.html"), nil, "main-dir") - testhelpers.AssertRedirectTo(t, serve, "GET", makeURL("/project"), nil, "//"+host+"/project/") - require.HTTPBodyContains(t, serve, "GET", makeURL("/project/"), nil, "project-subdir") - require.HTTPBodyContains(t, serve, "GET", makeURL("/project/index"), nil, "project-subdir") - require.HTTPBodyContains(t, serve, "GET", makeURL("/project/index/"), nil, "project-subdir") - require.HTTPBodyContains(t, serve, "GET", makeURL("/project/index.html"), nil, "project-subdir") - testhelpers.AssertRedirectTo(t, serve, "GET", makeURL("/project/subdir"), nil, "//"+host+"/project/subdir/") - require.HTTPBodyContains(t, serve, "GET", makeURL("/project/subdir/"), nil, "project-subsubdir") - require.HTTPBodyContains(t, serve, "GET", makeURL("/project2/"), nil, "project2-main") - require.HTTPBodyContains(t, serve, "GET", makeURL("/project2/index"), nil, "project2-main") - require.HTTPBodyContains(t, serve, "GET", makeURL("/project2/index.html"), nil, "project2-main") - require.HTTPError(t, serve, "GET", makeURL("/private.project/"), nil) - require.HTTPError(t, serve, "GET", makeURL("//about.gitlab.com/%2e%2e"), nil) - require.HTTPError(t, serve, "GET", makeURL("/symlink"), nil) - require.HTTPError(t, serve, "GET", makeURL("/symlink/index.html"), nil) - require.HTTPError(t, serve, "GET", makeURL("/symlink/subdir/"), nil) - require.HTTPError(t, serve, "GET", makeURL("/project/fifo"), nil) - require.HTTPError(t, serve, "GET", makeURL("/not-existing-file"), nil) - require.HTTPRedirect(t, serve, "GET", makeURL("/project//about.gitlab.com/%2e%2e"), nil) -} - -func TestGroupServeHTTP(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - t.Run("group.test.io", func(t *testing.T) { testGroupServeHTTPHost(t, "group.test.io") }) - t.Run("group.test.io:8080", func(t *testing.T) { testGroupServeHTTPHost(t, "group.test.io:8080") }) -} - func TestDomainServeHTTP(t *testing.T) { cleanup := setUpTests(t) defer cleanup() testDomain := &Domain{ - group: group{name: "group"}, - projectName: "project2", - config: &domainConfig{ - Domain: "test.domain.com", - }, + Project: "project2", + Group: "group", + ProjectConfig: &ProjectConfig{DomainName: "test.domain.com"}, } require.HTTPBodyContains(t, serveFileOrNotFound(testDomain), "GET", "/", nil, "project2-main") @@ -108,9 +54,9 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Custom domain with HTTPS-only enabled", domain: &Domain{ - group: group{name: "group"}, - projectName: "project", - config: &domainConfig{HTTPSOnly: true}, + Project: "project", + Group: "group", + ProjectConfig: &ProjectConfig{HTTPSOnly: true}, }, url: "http://custom-domain", expected: true, @@ -118,78 +64,18 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Custom domain with HTTPS-only disabled", domain: &Domain{ - group: group{name: "group"}, - projectName: "project", - config: &domainConfig{HTTPSOnly: false}, + Project: "project", + Group: "group", + ProjectConfig: &ProjectConfig{HTTPSOnly: false}, }, url: "http://custom-domain", expected: false, }, - { - name: "Default group domain with HTTPS-only enabled", - domain: &Domain{ - projectName: "project", - group: group{ - name: "group", - projects: projects{"test-domain": &project{HTTPSOnly: true}}, - }, - }, - url: "http://test-domain", - expected: true, - }, - { - name: "Default group domain with HTTPS-only disabled", - domain: &Domain{ - projectName: "project", - group: group{ - name: "group", - projects: projects{"test-domain": &project{HTTPSOnly: false}}, - }, - }, - url: "http://test-domain", - expected: false, - }, - { - name: "Case-insensitive default group domain with HTTPS-only enabled", - domain: &Domain{ - projectName: "project", - group: group{ - name: "group", - projects: projects{"test-domain": &project{HTTPSOnly: true}}, - }, - }, - url: "http://Test-domain", - expected: true, - }, - { - name: "Other group domain with HTTPS-only enabled", - domain: &Domain{ - projectName: "project", - group: group{ - name: "group", - projects: projects{"project": &project{HTTPSOnly: true}}, - }, - }, - url: "http://test-domain/project", - expected: true, - }, - { - name: "Other group domain with HTTPS-only disabled", - domain: &Domain{ - projectName: "project", - group: group{ - name: "group", - projects: projects{"project": &project{HTTPSOnly: false}}, - }, - }, - url: "http://test-domain/project", - expected: false, - }, { name: "Unknown project", domain: &Domain{ - group: group{name: "group"}, - projectName: "project", + Project: "project", + Group: "group", }, url: "http://test-domain/project", expected: false, @@ -217,9 +103,9 @@ func TestHasAcmeChallenge(t *testing.T) { { name: "Project containing acme challenge", domain: &Domain{ - group: group{name: "group.acme"}, - projectName: "with.acme.challenge", - config: &domainConfig{HTTPSOnly: true}, + Group: "group.acme", + Project: "with.acme.challenge", + ProjectConfig: &ProjectConfig{HTTPSOnly: true}, }, token: "existingtoken", expected: true, @@ -227,9 +113,9 @@ func TestHasAcmeChallenge(t *testing.T) { { name: "Project containing acme challenge", domain: &Domain{ - group: group{name: "group.acme"}, - projectName: "with.acme.challenge", - config: &domainConfig{HTTPSOnly: true}, + Group: "group.acme", + Project: "with.acme.challenge", + ProjectConfig: &ProjectConfig{HTTPSOnly: true}, }, token: "foldertoken", expected: true, @@ -237,9 +123,9 @@ func TestHasAcmeChallenge(t *testing.T) { { name: "Project containing another token", domain: &Domain{ - group: group{name: "group.acme"}, - projectName: "with.acme.challenge", - config: &domainConfig{HTTPSOnly: true}, + Group: "group.acme", + Project: "with.acme.challenge", + ProjectConfig: &ProjectConfig{HTTPSOnly: true}, }, token: "notexistingtoken", expected: false, @@ -253,9 +139,8 @@ func TestHasAcmeChallenge(t *testing.T) { { name: "Domain without config", domain: &Domain{ - group: group{name: "group.acme"}, - projectName: "with.acme.challenge", - config: nil, + Group: "group.acme", + Project: "with.acme.challenge", }, token: "existingtoken", expected: false, @@ -295,112 +180,14 @@ func testHTTPGzip(t *testing.T, handler http.HandlerFunc, mode, url string, valu require.Equal(t, contentType, w.Header().Get("Content-Type")) } -func TestGroupServeHTTPGzip(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - testGroup := &Domain{ - projectName: "", - group: group{ - name: "group", - projects: map[string]*project{ - "group.test.io": &project{}, - "group.gitlab-example.com": &project{}, - "project": &project{}, - "project2": &project{}, - }, - }, - } - - testSet := []struct { - mode string // HTTP mode - url string // Test URL - acceptEncoding string // Accept encoding header - body interface{} // Expected body at above URL - contentType string // Expected content-type - ungzip bool // Expect the response to be gzipped? - }{ - // No gzip encoding requested - {"GET", "/index.html", "", "main-dir", "text/html; charset=utf-8", false}, - {"GET", "/index.html", "identity", "main-dir", "text/html; charset=utf-8", false}, - {"GET", "/index.html", "gzip; q=0", "main-dir", "text/html; charset=utf-8", false}, - // gzip encoding requested, - {"GET", "/index.html", "*", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "identity, gzip", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "gzip", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "gzip; q=1", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "gzip; q=0.9", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "gzip, deflate", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "gzip; q=1, deflate", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "gzip; q=0.9, deflate", "main-dir", "text/html; charset=utf-8", true}, - // gzip encoding requested, but url does not have compressed content on disk - {"GET", "/project2/index.html", "*", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "identity, gzip", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "gzip", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "gzip; q=1", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "gzip; q=0.9", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "gzip, deflate", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "gzip; q=1, deflate", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "gzip; q=0.9, deflate", "project2-main", "text/html; charset=utf-8", false}, - // malformed headers - {"GET", "/index.html", ";; gzip", "main-dir", "text/html; charset=utf-8", false}, - {"GET", "/index.html", "middle-out", "main-dir", "text/html; charset=utf-8", false}, - {"GET", "/index.html", "gzip; quality=1", "main-dir", "text/html; charset=utf-8", false}, - // Symlinked .gz files are not supported - {"GET", "/gz-symlink", "*", "data", "text/plain; charset=utf-8", false}, - // Unknown file-extension, with text content - {"GET", "/text.unknown", "*", "hello", "text/plain; charset=utf-8", true}, - {"GET", "/text-nogzip.unknown", "*", "hello", "text/plain; charset=utf-8", false}, - // Unknown file-extension, with PNG content - {"GET", "/image.unknown", "*", "GIF89a", "image/gif", true}, - {"GET", "/image-nogzip.unknown", "*", "GIF89a", "image/gif", false}, - } - - for _, tt := range testSet { - URL := "http://group.test.io" + tt.url - testHTTPGzip(t, serveFileOrNotFound(testGroup), tt.mode, URL, nil, tt.acceptEncoding, tt.body, tt.contentType, tt.ungzip) - } -} - -func TestGroup404ServeHTTP(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - testGroup := &Domain{ - projectName: "", - group: group{ - name: "group.404", - projects: map[string]*project{ - "domain.404": &project{}, - "group.404.test.io": &project{}, - "project.404": &project{}, - "project.404.symlink": &project{}, - "project.no.404": &project{}, - }, - }, - } - - testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.404/not/existing-file", nil, "Custom 404 project page") - testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.404/", nil, "Custom 404 project page") - testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/not/existing-file", nil, "Custom 404 group page") - testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/not-existing-file", nil, "Custom 404 group page") - testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/", nil, "Custom 404 group page") - require.HTTPBodyNotContains(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.404.symlink/not/existing-file", nil, "Custom 404 project page") - - // Ensure the namespace project's custom 404.html is not used by projects - testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.no.404/not/existing-file", nil, "The page you're looking for could not be found.") -} - func TestDomain404ServeHTTP(t *testing.T) { cleanup := setUpTests(t) defer cleanup() testDomain := &Domain{ - group: group{name: "group.404"}, - projectName: "domain.404", - config: &domainConfig{ - Domain: "domain.404.com", - }, + Group: "group.404", + Project: "domain.404", + ProjectConfig: &ProjectConfig{DomainName: "domain.404.com"}, } testhelpers.AssertHTTP404(t, serveFileOrNotFound(testDomain), "GET", "http://group.404.test.io/not-existing-file", nil, "Custom 404 group page") @@ -412,7 +199,7 @@ func TestPredefined404ServeHTTP(t *testing.T) { defer cleanup() testDomain := &Domain{ - group: group{name: "group"}, + Group: "group", } testhelpers.AssertHTTP404(t, serveFileOrNotFound(testDomain), "GET", "http://group.test.io/not-existing-file", nil, "The page you're looking for could not be found") @@ -420,8 +207,8 @@ func TestPredefined404ServeHTTP(t *testing.T) { func TestGroupCertificate(t *testing.T) { testGroup := &Domain{ - group: group{name: "group"}, - projectName: "", + Project: "", + Group: "group", } tls, err := testGroup.EnsureCertificate() @@ -431,11 +218,9 @@ func TestGroupCertificate(t *testing.T) { func TestDomainNoCertificate(t *testing.T) { testDomain := &Domain{ - group: group{name: "group"}, - projectName: "project2", - config: &domainConfig{ - Domain: "test.domain.com", - }, + Group: "group", + Project: "project2", + ProjectConfig: &ProjectConfig{DomainName: "test.domain.com"}, } tls, err := testDomain.EnsureCertificate() @@ -449,10 +234,10 @@ func TestDomainNoCertificate(t *testing.T) { func TestDomainCertificate(t *testing.T) { testDomain := &Domain{ - group: group{name: "group"}, - projectName: "project2", - config: &domainConfig{ - Domain: "test.domain.com", + Group: "group", + Project: "project2", + ProjectConfig: &ProjectConfig{ + DomainName: "test.domain.com", Certificate: fixture.Certificate, Key: fixture.Key, }, @@ -463,37 +248,6 @@ func TestDomainCertificate(t *testing.T) { require.NoError(t, err) } -func TestCacheControlHeaders(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - testGroup := &Domain{ - group: group{ - name: "group", - projects: map[string]*project{ - "group.test.io": &project{}, - }, - }, - } - w := httptest.NewRecorder() - req, err := http.NewRequest("GET", "http://group.test.io/", nil) - require.NoError(t, err) - - now := time.Now() - serveFileOrNotFound(testGroup)(w, req) - - require.Equal(t, http.StatusOK, w.Code) - require.Equal(t, "max-age=600", w.Header().Get("Cache-Control")) - - expires := w.Header().Get("Expires") - require.NotEmpty(t, expires) - - expiresTime, err := time.Parse(time.RFC1123, expires) - require.NoError(t, err) - - require.WithinDuration(t, now.UTC().Add(10*time.Minute), expiresTime.UTC(), time.Minute) -} - func TestOpenNoFollow(t *testing.T) { tmpfile, err := ioutil.TempFile("", "link-test") require.NoError(t, err) diff --git a/internal/domain/group.go b/internal/domain/group.go deleted file mode 100644 index 83b8d25563306e2a91e70bdd0d7abb214add9fb3..0000000000000000000000000000000000000000 --- a/internal/domain/group.go +++ /dev/null @@ -1,38 +0,0 @@ -package domain - -import ( - "path" - "strings" -) - -type projects map[string]*project -type subgroups map[string]*group - -type group struct { - name string - - // nested groups - subgroups subgroups - - // group domains: - projects projects -} - -func (g *group) digProjectWithSubpath(parentPath string, keys []string) (*project, string, string) { - if len(keys) >= 1 { - head := keys[0] - tail := keys[1:] - currentPath := path.Join(parentPath, head) - search := strings.ToLower(head) - - if project := g.projects[search]; project != nil { - return project, currentPath, path.Join(tail...) - } - - if subgroup := g.subgroups[search]; subgroup != nil { - return subgroup.digProjectWithSubpath(currentPath, tail) - } - } - - return nil, "", "" -} diff --git a/internal/domain/domain_config.go b/internal/source/disk/config.go similarity index 58% rename from internal/domain/domain_config.go rename to internal/source/disk/config.go index 2ab2ce6cb5a96a31c1a98534b18ff753b138e9e2..d2e6c123b9bd6dd5eb7420bff1c9bb19ad5fdb71 100644 --- a/internal/domain/domain_config.go +++ b/internal/source/disk/config.go @@ -1,4 +1,4 @@ -package domain +package disk import ( "encoding/json" @@ -7,6 +7,7 @@ import ( "strings" ) +// DomainConfig represents a custom domain config type domainConfig struct { Domain string Certificate string @@ -16,13 +17,23 @@ type domainConfig struct { AccessControl bool `json:"access_control"` } -type domainsConfig struct { +// MultiDomainConfig represents a group of custom domain configs +type multiDomainConfig struct { Domains []domainConfig HTTPSOnly bool `json:"https_only"` ID uint64 `json:"id"` AccessControl bool `json:"access_control"` } +// ProjectConfig is a project-level configuration +type projectConfig struct { + NamespaceProject bool + HTTPSOnly bool + AccessControl bool + ID uint64 +} + +// Valid validates a custom domain config for a root domain func (c *domainConfig) Valid(rootDomain string) bool { if c.Domain == "" { return false @@ -34,13 +45,13 @@ func (c *domainConfig) Valid(rootDomain string) bool { return !strings.HasSuffix(domain, rootDomain) } -func (c *domainsConfig) Read(group, project string) (err error) { +// Read reads a multi domain config and decodes it from a `config.json` +func (c *multiDomainConfig) Read(group, project string) error { configFile, err := os.Open(filepath.Join(group, project, "config.json")) if err != nil { return err } defer configFile.Close() - err = json.NewDecoder(configFile).Decode(c) - return + return json.NewDecoder(configFile).Decode(c) } diff --git a/internal/domain/domain_config_test.go b/internal/source/disk/config_test.go similarity index 92% rename from internal/domain/domain_config_test.go rename to internal/source/disk/config_test.go index 8cdcdeaa6bdb9ef9eec4161ac4ef37edf3bae9fb..1bb2364ab9bcfddf3c643dfe09334385814bd3b7 100644 --- a/internal/domain/domain_config_test.go +++ b/internal/source/disk/config_test.go @@ -1,4 +1,4 @@ -package domain +package disk import ( "io/ioutil" @@ -40,26 +40,26 @@ func TestDomainConfigRead(t *testing.T) { cleanup := setUpTests(t) defer cleanup() - d := domainsConfig{} + d := multiDomainConfig{} err := d.Read("test-group", "test-project") require.Error(t, err) os.MkdirAll(filepath.Dir(configFile), 0700) defer os.RemoveAll("test-group") - d = domainsConfig{} + d = multiDomainConfig{} err = d.Read("test-group", "test-project") require.Error(t, err) err = ioutil.WriteFile(configFile, []byte(invalidConfig), 0600) require.NoError(t, err) - d = domainsConfig{} + d = multiDomainConfig{} err = d.Read("test-group", "test-project") require.Error(t, err) err = ioutil.WriteFile(configFile, []byte(validConfig), 0600) require.NoError(t, err) - d = domainsConfig{} + d = multiDomainConfig{} err = d.Read("test-group", "test-project") require.NoError(t, err) } diff --git a/internal/source/disk/group.go b/internal/source/disk/group.go new file mode 100644 index 0000000000000000000000000000000000000000..0c8d08104c25fc3c8fc666e63d8d0ab1c89eca35 --- /dev/null +++ b/internal/source/disk/group.go @@ -0,0 +1,145 @@ +package disk + +import ( + "errors" + "net/http" + "path" + "strings" + + "gitlab.com/gitlab-org/gitlab-pages/internal/host" +) + +const ( + subgroupScanLimit int = 21 + // maxProjectDepth is set to the maximum nested project depth in gitlab (21) plus 3. + // One for the project, one for the first empty element of the split (URL.Path starts with /), + // and one for the real file path + maxProjectDepth int = subgroupScanLimit + 3 +) + +// Group represents a GitLab group with project configs and subgroups +type Group struct { + name string + + // nested groups + subgroups subgroups + + // group domains: + projects projects +} + +type projects map[string]*projectConfig +type subgroups map[string]*Group + +func (g *Group) digProjectWithSubpath(parentPath string, keys []string) (*projectConfig, string, string) { + if len(keys) >= 1 { + head := keys[0] + tail := keys[1:] + currentPath := path.Join(parentPath, head) + search := strings.ToLower(head) + + if project := g.projects[search]; project != nil { + return project, currentPath, path.Join(tail...) + } + + if subgroup := g.subgroups[search]; subgroup != nil { + return subgroup.digProjectWithSubpath(currentPath, tail) + } + } + + return nil, "", "" +} + +// Look up a project inside the domain based on the host and path. Returns the +// project and its name (if applicable) +func (g *Group) getProjectConfigWithSubpath(r *http.Request) (*projectConfig, string, string) { + // Check for a project specified in the URL: http://group.gitlab.io/projectA + // If present, these projects shadow the group domain. + split := strings.SplitN(r.URL.Path, "/", maxProjectDepth) + if len(split) >= 2 { + projectConfig, projectPath, urlPath := g.digProjectWithSubpath("", split[1:]) + if projectConfig != nil { + return projectConfig, projectPath, urlPath + } + } + + // Since the URL doesn't specify a project (e.g. http://mydomain.gitlab.io), + // return the group project if it exists. + if host := host.FromRequest(r); host != "" { + if groupProject := g.projects[host]; groupProject != nil { + return groupProject, host, strings.Join(split[1:], "/") + } + } + + return nil, "", "" +} + +// IsHTTPSOnly return true if project exists and has https-only setting +// configured +func (g *Group) IsHTTPSOnly(r *http.Request) bool { + project, _, _ := g.getProjectConfigWithSubpath(r) + + if project != nil { + return project.HTTPSOnly + } + + return false +} + +// HasAccessControl returns true if a group project has access control setting +// enabled +func (g *Group) HasAccessControl(r *http.Request) bool { + project, _, _ := g.getProjectConfigWithSubpath(r) + + if project != nil { + return project.AccessControl + } + + return false +} + +// IsNamespaceProject return true if per-request config belongs to a namespace +// project +func (g *Group) IsNamespaceProject(r *http.Request) bool { + project, _, _ := g.getProjectConfigWithSubpath(r) + + if project != nil { + return project.NamespaceProject + } + + return false +} + +// ProjectID return a per-request group project ID +func (g *Group) ProjectID(r *http.Request) uint64 { + project, _, _ := g.getProjectConfigWithSubpath(r) + + if project != nil { + return project.ID + } + + return 0 +} + +// ProjectExists return true if project config has been found +func (g *Group) ProjectExists(r *http.Request) bool { + project, _, _ := g.getProjectConfigWithSubpath(r) + + if project != nil { + return true + } + + return false +} + +// ProjectWithSubpath tries to find project and its config recursively for a +// given request to a group domain +func (g *Group) ProjectWithSubpath(r *http.Request) (string, string, error) { + project, projectName, subPath := g.getProjectConfigWithSubpath(r) + + if project != nil { + return projectName, subPath, nil + } + + return "", "", errors.New("project not found") +} diff --git a/internal/source/disk/group_domain_test.go b/internal/source/disk/group_domain_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3b4471f4feab1aa3528b64dc7e10de5ca8b6dff9 --- /dev/null +++ b/internal/source/disk/group_domain_test.go @@ -0,0 +1,441 @@ +package disk + +import ( + "compress/gzip" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "gitlab.com/gitlab-org/gitlab-pages/internal/domain" + "gitlab.com/gitlab-org/gitlab-pages/internal/fixture" + "gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers" +) + +func serveFileOrNotFound(domain *domain.Domain) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if !domain.ServeFileHTTP(w, r) { + domain.ServeNotFoundHTTP(w, r) + } + } +} + +func testGroupServeHTTPHost(t *testing.T, host string) { + testGroup := &domain.Domain{ + Project: "", + Group: "group", + GroupConfig: &Group{ + name: "group", + projects: map[string]*projectConfig{ + "group.test.io": &projectConfig{}, + "group.gitlab-example.com": &projectConfig{}, + "project": &projectConfig{}, + "project2": &projectConfig{}, + }, + }, + } + + makeURL := func(path string) string { + return "http://" + host + path + } + + serve := serveFileOrNotFound(testGroup) + + require.HTTPBodyContains(t, serve, "GET", makeURL("/"), nil, "main-dir") + require.HTTPBodyContains(t, serve, "GET", makeURL("/index"), nil, "main-dir") + require.HTTPBodyContains(t, serve, "GET", makeURL("/index.html"), nil, "main-dir") + testhelpers.AssertRedirectTo(t, serve, "GET", makeURL("/project"), nil, "//"+host+"/project/") + require.HTTPBodyContains(t, serve, "GET", makeURL("/project/"), nil, "project-subdir") + require.HTTPBodyContains(t, serve, "GET", makeURL("/project/index"), nil, "project-subdir") + require.HTTPBodyContains(t, serve, "GET", makeURL("/project/index/"), nil, "project-subdir") + require.HTTPBodyContains(t, serve, "GET", makeURL("/project/index.html"), nil, "project-subdir") + testhelpers.AssertRedirectTo(t, serve, "GET", makeURL("/project/subdir"), nil, "//"+host+"/project/subdir/") + require.HTTPBodyContains(t, serve, "GET", makeURL("/project/subdir/"), nil, "project-subsubdir") + require.HTTPBodyContains(t, serve, "GET", makeURL("/project2/"), nil, "project2-main") + require.HTTPBodyContains(t, serve, "GET", makeURL("/project2/index"), nil, "project2-main") + require.HTTPBodyContains(t, serve, "GET", makeURL("/project2/index.html"), nil, "project2-main") + require.HTTPError(t, serve, "GET", makeURL("/private.project/"), nil) + require.HTTPError(t, serve, "GET", makeURL("//about.gitlab.com/%2e%2e"), nil) + require.HTTPError(t, serve, "GET", makeURL("/symlink"), nil) + require.HTTPError(t, serve, "GET", makeURL("/symlink/index.html"), nil) + require.HTTPError(t, serve, "GET", makeURL("/symlink/subdir/"), nil) + require.HTTPError(t, serve, "GET", makeURL("/project/fifo"), nil) + require.HTTPError(t, serve, "GET", makeURL("/not-existing-file"), nil) + require.HTTPRedirect(t, serve, "GET", makeURL("/project//about.gitlab.com/%2e%2e"), nil) +} + +func TestGroupServeHTTP(t *testing.T) { + cleanup := setUpTests(t) + defer cleanup() + + t.Run("group.test.io", func(t *testing.T) { testGroupServeHTTPHost(t, "group.test.io") }) + t.Run("group.test.io:8080", func(t *testing.T) { testGroupServeHTTPHost(t, "group.test.io:8080") }) +} + +func TestDomainServeHTTP(t *testing.T) { + cleanup := setUpTests(t) + defer cleanup() + + testDomain := &domain.Domain{ + Group: "group", + Project: "project2", + ProjectConfig: &domain.ProjectConfig{DomainName: "test.domain.com"}, + } + + require.HTTPBodyContains(t, serveFileOrNotFound(testDomain), "GET", "/", nil, "project2-main") + require.HTTPBodyContains(t, serveFileOrNotFound(testDomain), "GET", "/index.html", nil, "project2-main") + require.HTTPRedirect(t, serveFileOrNotFound(testDomain), "GET", "/subdir", nil) + require.HTTPBodyContains(t, serveFileOrNotFound(testDomain), "GET", "/subdir", nil, + `Found`) + require.HTTPBodyContains(t, serveFileOrNotFound(testDomain), "GET", "/subdir/", nil, "project2-subdir") + require.HTTPBodyContains(t, serveFileOrNotFound(testDomain), "GET", "/subdir/index.html", nil, "project2-subdir") + require.HTTPError(t, serveFileOrNotFound(testDomain), "GET", "//about.gitlab.com/%2e%2e", nil) + require.HTTPError(t, serveFileOrNotFound(testDomain), "GET", "/not-existing-file", nil) +} + +func TestIsHTTPSOnly(t *testing.T) { + tests := []struct { + name string + domain *domain.Domain + url string + expected bool + }{ + { + name: "Default group domain with HTTPS-only enabled", + domain: &domain.Domain{ + Group: "group", + Project: "project", + GroupConfig: &Group{ + name: "group", + projects: projects{"test-domain": &projectConfig{HTTPSOnly: true}}, + }, + }, + url: "http://test-domain", + expected: true, + }, + { + name: "Default group domain with HTTPS-only disabled", + domain: &domain.Domain{ + Group: "group", + Project: "project", + GroupConfig: &Group{ + name: "group", + projects: projects{"test-domain": &projectConfig{HTTPSOnly: false}}, + }, + }, + url: "http://test-domain", + expected: false, + }, + { + name: "Case-insensitive default group domain with HTTPS-only enabled", + domain: &domain.Domain{ + Project: "project", + Group: "group", + GroupConfig: &Group{ + name: "group", + projects: projects{"test-domain": &projectConfig{HTTPSOnly: true}}, + }, + }, + url: "http://Test-domain", + expected: true, + }, + { + name: "Other group domain with HTTPS-only enabled", + domain: &domain.Domain{ + Project: "project", + Group: "group", + GroupConfig: &Group{ + name: "group", + projects: projects{"project": &projectConfig{HTTPSOnly: true}}, + }, + }, + url: "http://test-domain/project", + expected: true, + }, + { + name: "Other group domain with HTTPS-only disabled", + domain: &domain.Domain{ + Project: "project", + Group: "group", + GroupConfig: &Group{ + name: "group", + projects: projects{"project": &projectConfig{HTTPSOnly: false}}, + }, + }, + url: "http://test-domain/project", + expected: false, + }, + { + name: "Unknown project", + domain: &domain.Domain{ + Group: "group", + Project: "project", + }, + url: "http://test-domain/project", + expected: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req, _ := http.NewRequest(http.MethodGet, test.url, nil) + require.Equal(t, test.expected, test.domain.IsHTTPSOnly(req)) + }) + } +} + +func testHTTPGzip(t *testing.T, handler http.HandlerFunc, mode, url string, values url.Values, acceptEncoding string, str interface{}, contentType string, ungzip bool) { + w := httptest.NewRecorder() + req, err := http.NewRequest(mode, url+"?"+values.Encode(), nil) + require.NoError(t, err) + if acceptEncoding != "" { + req.Header.Add("Accept-Encoding", acceptEncoding) + } + handler(w, req) + + if ungzip { + reader, err := gzip.NewReader(w.Body) + require.NoError(t, err) + defer reader.Close() + + contentEncoding := w.Header().Get("Content-Encoding") + require.Equal(t, "gzip", contentEncoding, "Content-Encoding") + + bytes, err := ioutil.ReadAll(reader) + require.NoError(t, err) + require.Contains(t, string(bytes), str) + } else { + require.Contains(t, w.Body.String(), str) + } + + require.Equal(t, contentType, w.Header().Get("Content-Type")) +} + +func TestGroupServeHTTPGzip(t *testing.T) { + cleanup := setUpTests(t) + defer cleanup() + + testGroup := &domain.Domain{ + Project: "", + Group: "group", + GroupConfig: &Group{ + name: "group", + projects: map[string]*projectConfig{ + "group.test.io": &projectConfig{}, + "group.gitlab-example.com": &projectConfig{}, + "project": &projectConfig{}, + "project2": &projectConfig{}, + }, + }, + } + + testSet := []struct { + mode string // HTTP mode + url string // Test URL + acceptEncoding string // Accept encoding header + body interface{} // Expected body at above URL + contentType string // Expected content-type + ungzip bool // Expect the response to be gzipped? + }{ + // No gzip encoding requested + {"GET", "/index.html", "", "main-dir", "text/html; charset=utf-8", false}, + {"GET", "/index.html", "identity", "main-dir", "text/html; charset=utf-8", false}, + {"GET", "/index.html", "gzip; q=0", "main-dir", "text/html; charset=utf-8", false}, + // gzip encoding requested, + {"GET", "/index.html", "*", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "identity, gzip", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "gzip", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "gzip; q=1", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "gzip; q=0.9", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "gzip, deflate", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "gzip; q=1, deflate", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "gzip; q=0.9, deflate", "main-dir", "text/html; charset=utf-8", true}, + // gzip encoding requested, but url does not have compressed content on disk + {"GET", "/project2/index.html", "*", "project2-main", "text/html; charset=utf-8", false}, + {"GET", "/project2/index.html", "identity, gzip", "project2-main", "text/html; charset=utf-8", false}, + {"GET", "/project2/index.html", "gzip", "project2-main", "text/html; charset=utf-8", false}, + {"GET", "/project2/index.html", "gzip; q=1", "project2-main", "text/html; charset=utf-8", false}, + {"GET", "/project2/index.html", "gzip; q=0.9", "project2-main", "text/html; charset=utf-8", false}, + {"GET", "/project2/index.html", "gzip, deflate", "project2-main", "text/html; charset=utf-8", false}, + {"GET", "/project2/index.html", "gzip; q=1, deflate", "project2-main", "text/html; charset=utf-8", false}, + {"GET", "/project2/index.html", "gzip; q=0.9, deflate", "project2-main", "text/html; charset=utf-8", false}, + // malformed headers + {"GET", "/index.html", ";; gzip", "main-dir", "text/html; charset=utf-8", false}, + {"GET", "/index.html", "middle-out", "main-dir", "text/html; charset=utf-8", false}, + {"GET", "/index.html", "gzip; quality=1", "main-dir", "text/html; charset=utf-8", false}, + // Symlinked .gz files are not supported + {"GET", "/gz-symlink", "*", "data", "text/plain; charset=utf-8", false}, + // Unknown file-extension, with text content + {"GET", "/text.unknown", "*", "hello", "text/plain; charset=utf-8", true}, + {"GET", "/text-nogzip.unknown", "*", "hello", "text/plain; charset=utf-8", false}, + // Unknown file-extension, with PNG content + {"GET", "/image.unknown", "*", "GIF89a", "image/gif", true}, + {"GET", "/image-nogzip.unknown", "*", "GIF89a", "image/gif", false}, + } + + for _, tt := range testSet { + URL := "http://group.test.io" + tt.url + testHTTPGzip(t, serveFileOrNotFound(testGroup), tt.mode, URL, nil, tt.acceptEncoding, tt.body, tt.contentType, tt.ungzip) + } +} + +func TestGroup404ServeHTTP(t *testing.T) { + cleanup := setUpTests(t) + defer cleanup() + + testGroup := &domain.Domain{ + Project: "", + Group: "group.404", + GroupConfig: &Group{ + name: "group.404", + projects: map[string]*projectConfig{ + "domain.404": &projectConfig{}, + "group.404.test.io": &projectConfig{}, + "project.404": &projectConfig{}, + "project.404.symlink": &projectConfig{}, + "project.no.404": &projectConfig{}, + }, + }, + } + + testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.404/not/existing-file", nil, "Custom 404 project page") + testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.404/", nil, "Custom 404 project page") + testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/not/existing-file", nil, "Custom 404 group page") + testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/not-existing-file", nil, "Custom 404 group page") + testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/", nil, "Custom 404 group page") + require.HTTPBodyNotContains(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.404.symlink/not/existing-file", nil, "Custom 404 project page") + + // Ensure the namespace project's custom 404.html is not used by projects + testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.no.404/not/existing-file", nil, "The page you're looking for could not be found.") +} + +func TestDomain404ServeHTTP(t *testing.T) { + cleanup := setUpTests(t) + defer cleanup() + + testDomain := &domain.Domain{ + Project: "domain.404", + Group: "group.404", + ProjectConfig: &domain.ProjectConfig{DomainName: "domain.404.com"}, + } + + testhelpers.AssertHTTP404(t, serveFileOrNotFound(testDomain), "GET", "http://group.404.test.io/not-existing-file", nil, "Custom 404 group page") + testhelpers.AssertHTTP404(t, serveFileOrNotFound(testDomain), "GET", "http://group.404.test.io/", nil, "Custom 404 group page") +} + +func TestPredefined404ServeHTTP(t *testing.T) { + cleanup := setUpTests(t) + defer cleanup() + + testDomain := &domain.Domain{ + Group: "group", + } + + testhelpers.AssertHTTP404(t, serveFileOrNotFound(testDomain), "GET", "http://group.test.io/not-existing-file", nil, "The page you're looking for could not be found") +} + +func TestGroupCertificate(t *testing.T) { + testGroup := &domain.Domain{ + Group: "group", + Project: "", + } + + tls, err := testGroup.EnsureCertificate() + require.Nil(t, tls) + require.Error(t, err) +} + +func TestDomainNoCertificate(t *testing.T) { + testDomain := &domain.Domain{ + Group: "group", + Project: "project2", + ProjectConfig: &domain.ProjectConfig{DomainName: "test.domain.com"}, + } + + tls, err := testDomain.EnsureCertificate() + require.Nil(t, tls) + require.Error(t, err) + + _, err2 := testDomain.EnsureCertificate() + require.Error(t, err) + require.Equal(t, err, err2) +} + +func TestDomainCertificate(t *testing.T) { + testDomain := &domain.Domain{ + Project: "project2", + Group: "group", + ProjectConfig: &domain.ProjectConfig{DomainName: "test.domain.com", + Certificate: fixture.Certificate, + Key: fixture.Key, + }, + } + + tls, err := testDomain.EnsureCertificate() + require.NotNil(t, tls) + require.NoError(t, err) +} + +func TestCacheControlHeaders(t *testing.T) { + cleanup := setUpTests(t) + defer cleanup() + + testGroup := &domain.Domain{ + Group: "group", + GroupConfig: &Group{ + name: "group", + projects: map[string]*projectConfig{ + "group.test.io": &projectConfig{}, + }, + }, + } + w := httptest.NewRecorder() + req, err := http.NewRequest("GET", "http://group.test.io/", nil) + require.NoError(t, err) + + now := time.Now() + serveFileOrNotFound(testGroup)(w, req) + + require.Equal(t, http.StatusOK, w.Code) + require.Equal(t, "max-age=600", w.Header().Get("Cache-Control")) + + expires := w.Header().Get("Expires") + require.NotEmpty(t, expires) + + expiresTime, err := time.Parse(time.RFC1123, expires) + require.NoError(t, err) + + require.WithinDuration(t, now.UTC().Add(10*time.Minute), expiresTime.UTC(), time.Minute) +} + +var chdirSet = false + +func setUpTests(t require.TestingT) func() { + return chdirInPath(t, "../../../shared/pages") +} + +func chdirInPath(t require.TestingT, path string) func() { + noOp := func() {} + if chdirSet { + return noOp + } + + cwd, err := os.Getwd() + require.NoError(t, err, "Cannot Getwd") + + err = os.Chdir(path) + require.NoError(t, err, "Cannot Chdir") + + chdirSet = true + return func() { + err := os.Chdir(cwd) + require.NoError(t, err, "Cannot Chdir in cleanup") + + chdirSet = false + } +} diff --git a/internal/domain/group_test.go b/internal/source/disk/group_test.go similarity index 81% rename from internal/domain/group_test.go rename to internal/source/disk/group_test.go index 8f75fe96a6072f6b283ac8177e184459c81233af..d0fb49bd90bd2d3e61a4b610fd9f25b554abf302 100644 --- a/internal/domain/group_test.go +++ b/internal/source/disk/group_test.go @@ -1,4 +1,4 @@ -package domain +package disk import ( "strings" @@ -8,25 +8,25 @@ import ( ) func TestGroupDig(t *testing.T) { - matchingProject := &project{ID: 1} + matchingProject := &projectConfig{ID: 1} tests := []struct { name string - g group + g Group path string - expectedProject *project + expectedProject *projectConfig expectedProjectPath string expectedPath string }{ { name: "empty group", path: "projectb/demo/features.html", - g: group{}, + g: Group{}, }, { name: "group with project", path: "projectb/demo/features.html", - g: group{ + g: Group{ projects: projects{"projectb": matchingProject}, }, expectedProject: matchingProject, @@ -36,7 +36,7 @@ func TestGroupDig(t *testing.T) { { name: "group with project and no path in URL", path: "projectb", - g: group{ + g: Group{ projects: projects{"projectb": matchingProject}, }, expectedProject: matchingProject, @@ -45,11 +45,11 @@ func TestGroupDig(t *testing.T) { { name: "group with subgroup and project", path: "projectb/demo/features.html", - g: group{ + g: Group{ projects: projects{"projectb": matchingProject}, subgroups: subgroups{ - "sub1": &group{ - projects: projects{"another": &project{}}, + "sub1": &Group{ + projects: projects{"another": &projectConfig{}}, }, }, }, @@ -60,13 +60,13 @@ func TestGroupDig(t *testing.T) { { name: "group with project inside a subgroup", path: "sub1/projectb/demo/features.html", - g: group{ + g: Group{ subgroups: subgroups{ - "sub1": &group{ + "sub1": &Group{ projects: projects{"projectb": matchingProject}, }, }, - projects: projects{"another": &project{}}, + projects: projects{"another": &projectConfig{}}, }, expectedProject: matchingProject, expectedProjectPath: "sub1/projectb", @@ -75,10 +75,10 @@ func TestGroupDig(t *testing.T) { { name: "group with matching subgroup but no project", path: "sub1/projectb/demo/features.html", - g: group{ + g: Group{ subgroups: subgroups{ - "sub1": &group{ - projects: projects{"another": &project{}}, + "sub1": &Group{ + projects: projects{"another": &projectConfig{}}, }, }, }, diff --git a/internal/domain/map.go b/internal/source/disk/map.go similarity index 86% rename from internal/domain/map.go rename to internal/source/disk/map.go index 7934860bb9e5e2d8afe8e08d22688aa5663e884e..2a6ada2d64ce562d3bebe0bd9a3b6756e6ab0b9b 100644 --- a/internal/domain/map.go +++ b/internal/source/disk/map.go @@ -1,4 +1,4 @@ -package domain +package disk import ( "bytes" @@ -12,22 +12,23 @@ import ( "github.com/karrick/godirwalk" log "github.com/sirupsen/logrus" + "gitlab.com/gitlab-org/gitlab-pages/internal/domain" "gitlab.com/gitlab-org/gitlab-pages/metrics" ) -// Map maps domain names to D instances. -type Map map[string]*Domain +// Map maps domain names to Domain instances. +type Map map[string]*domain.Domain type domainsUpdater func(Map) -func (dm Map) updateDomainMap(domainName string, domain *Domain) { +func (dm Map) updateDomainMap(domainName string, domain *domain.Domain) { if old, ok := dm[domainName]; ok { log.WithFields(log.Fields{ "domain_name": domainName, - "new_group": domain.group, - "new_project_name": domain.projectName, - "old_group": old.group, - "old_project_name": old.projectName, + "new_group": domain.Group, + "new_project_name": domain.Project, + "old_group": old.Group, + "old_project_name": old.Project, }).Error("Duplicate domain") } @@ -35,10 +36,17 @@ func (dm Map) updateDomainMap(domainName string, domain *Domain) { } func (dm Map) addDomain(rootDomain, groupName, projectName string, config *domainConfig) { - newDomain := &Domain{ - group: group{name: groupName}, - projectName: projectName, - config: config, + newDomain := &domain.Domain{ + Group: groupName, + Project: projectName, + ProjectConfig: &domain.ProjectConfig{ + DomainName: config.Domain, + Certificate: config.Certificate, + Key: config.Key, + HTTPSOnly: config.HTTPSOnly, + ProjectID: config.ID, + AccessControl: config.AccessControl, + }, } var domainName string @@ -51,8 +59,9 @@ func (dm Map) updateGroupDomain(rootDomain, groupName, projectPath string, https groupDomain := dm[domainName] if groupDomain == nil { - groupDomain = &Domain{ - group: group{ + groupDomain = &domain.Domain{ + Group: groupName, + GroupConfig: &Group{ name: groupName, projects: make(projects), subgroups: make(subgroups), @@ -62,13 +71,13 @@ func (dm Map) updateGroupDomain(rootDomain, groupName, projectPath string, https split := strings.SplitN(strings.ToLower(projectPath), "/", maxProjectDepth) projectName := split[len(split)-1] - g := &groupDomain.group + g := groupDomain.GroupConfig.(*Group) for i := 0; i < len(split)-1; i++ { subgroupName := split[i] subgroup := g.subgroups[subgroupName] if subgroup == nil { - subgroup = &group{ + subgroup = &Group{ name: subgroupName, projects: make(projects), subgroups: make(subgroups), @@ -79,7 +88,7 @@ func (dm Map) updateGroupDomain(rootDomain, groupName, projectPath string, https g = subgroup } - g.projects[projectName] = &project{ + g.projects[projectName] = &projectConfig{ NamespaceProject: domainName == projectName, HTTPSOnly: httpsOnly, AccessControl: accessControl, @@ -89,7 +98,7 @@ func (dm Map) updateGroupDomain(rootDomain, groupName, projectPath string, https dm[domainName] = groupDomain } -func (dm Map) readProjectConfig(rootDomain string, group, projectName string, config *domainsConfig) { +func (dm Map) readProjectConfig(rootDomain string, group, projectName string, config *multiDomainConfig) { if config == nil { // This is necessary to preserve the previous behaviour where a // group domain is created even if no config.json files are @@ -131,7 +140,7 @@ func readProject(group, parent, projectName string, level int, fanIn chan<- jobR // We read the config.json file _before_ fanning in, because it does disk // IO and it does not need access to the domains map. - config := &domainsConfig{} + config := &multiDomainConfig{} if err := config.Read(group, projectPath); err != nil { config = nil } @@ -163,7 +172,7 @@ func readProjects(group, parent string, level int, buf []byte, fanIn chan<- jobR type jobResult struct { group string project string - config *domainsConfig + config *multiDomainConfig } // ReadGroups walks the pages directory and populates dm with all the domains it finds. diff --git a/internal/domain/map_test.go b/internal/source/disk/map_test.go similarity index 93% rename from internal/domain/map_test.go rename to internal/source/disk/map_test.go index de5e49552e542583a696a38adcbb3c4a1e3f78b6..9aa960722f3ec2d2bc3d2921df6f346d67a36c33 100644 --- a/internal/domain/map_test.go +++ b/internal/source/disk/map_test.go @@ -1,4 +1,4 @@ -package domain +package disk import ( "crypto/rand" @@ -68,16 +68,15 @@ func TestReadProjects(t *testing.T) { } // Check that multiple domains in the same project are recorded faithfully - exp1 := &domainConfig{Domain: "test.domain.com"} - require.Equal(t, exp1, dm["test.domain.com"].config) - - exp2 := &domainConfig{Domain: "other.domain.com", Certificate: "test", Key: "key"} - require.Equal(t, exp2, dm["other.domain.com"].config) + require.Equal(t, "test.domain.com", dm["test.domain.com"].ProjectConfig.DomainName) + require.Equal(t, "other.domain.com", dm["other.domain.com"].ProjectConfig.DomainName) + require.Equal(t, "test", dm["other.domain.com"].ProjectConfig.Certificate) + require.Equal(t, "key", dm["other.domain.com"].ProjectConfig.Key) // check subgroups domain, ok := dm["group.test.io"] require.True(t, ok, "missing group.test.io domain") - subgroup, ok := domain.subgroups["subgroup"] + subgroup, ok := domain.GroupConfig.(*Group).subgroups["subgroup"] require.True(t, ok, "missing group.test.io subgroup") _, ok = subgroup.projects["project"] require.True(t, ok, "missing project for subgroup in group.test.io domain") @@ -118,7 +117,7 @@ func TestReadProjectsMaxDepth(t *testing.T) { // check subgroups domain, ok := dm["group-0.test.io"] require.True(t, ok, "missing group-0.test.io domain") - subgroup := &domain.group + subgroup := domain.GroupConfig.(*Group) for i := 0; i < levels; i++ { subgroup, ok = subgroup.subgroups["sub"] if i <= subgroupScanLimit { diff --git a/internal/source/domains.go b/internal/source/domains.go new file mode 100644 index 0000000000000000000000000000000000000000..cd2d89c55f16c07029e1c2e262fb5e34d762dace --- /dev/null +++ b/internal/source/domains.go @@ -0,0 +1,65 @@ +package source + +import ( + "strings" + "sync" + "time" + + "gitlab.com/gitlab-org/gitlab-pages/internal/domain" + "gitlab.com/gitlab-org/gitlab-pages/internal/source/disk" +) + +// Domains struct represents a map of all domains supported by pages. It is +// currently reading them from disk. +type Domains struct { + dm disk.Map + lock *sync.RWMutex +} + +// NewDomains is a factory method for domains initializing a mutex. It should +// not initialize `dm` as we later check the readiness by comparing it with a +// nil value. +func NewDomains() *Domains { + return &Domains{ + lock: &sync.RWMutex{}, + } +} + +// GetDomain returns a domain from the domains map +func (d *Domains) GetDomain(host string) *domain.Domain { + host = strings.ToLower(host) + d.lock.RLock() + defer d.lock.RUnlock() + domain, _ := d.dm[host] + + return domain +} + +// HasDomain checks for presence of a domain in the domains map +func (d *Domains) HasDomain(host string) bool { + d.lock.RLock() + defer d.lock.RUnlock() + + host = strings.ToLower(host) + _, isPresent := d.dm[host] + + return isPresent +} + +// Ready checks if the domains source is ready for work +func (d *Domains) Ready() bool { + return d.dm != nil +} + +// Watch starts the domain source, in this case it is reading domains from +// groups on disk concurrently +func (d *Domains) Watch(rootDomain string) { + go disk.Watch(rootDomain, d.updateDomains, time.Second) +} + +func (d *Domains) updateDomains(dm disk.Map) { + d.lock.Lock() + defer d.lock.Unlock() + + d.dm = dm +}