diff --git a/app.go b/app.go index 9be4409c27ee786440b65e69080d92e7dee4aea3..b6ac89ab5e30c0e05871bee2cafe1726ef0e8889 100644 --- a/app.go +++ b/app.go @@ -258,7 +258,6 @@ func (a *theApp) serveFileOrNotFoundHandler() http.Handler { // because the projects override the paths of the namespace project and they might be private even though // namespace project is public. if domain.IsNamespaceProject(r) { - if a.Auth.CheckAuthenticationWithoutProject(w, r) { return } diff --git a/go.mod b/go.mod index 2c4252a450171b9854fc5d4367ce01d4f36c1de7..046398e28c937147121894bd3e577b61fe42278d 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,8 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 github.com/karrick/godirwalk v1.10.12 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/namsral/flag v1.7.4-pre github.com/prometheus/client_golang v1.1.0 github.com/rs/cors v1.7.0 diff --git a/internal/domain/domain_config.go b/internal/domain/config.go similarity index 79% rename from internal/domain/domain_config.go rename to internal/domain/config.go index 2ab2ce6cb5a96a31c1a98534b18ff753b138e9e2..0682853b19ca9377aa9c96b8917a52a9e74a8584 100644 --- a/internal/domain/domain_config.go +++ b/internal/domain/config.go @@ -7,7 +7,7 @@ import ( "strings" ) -type domainConfig struct { +type Config struct { Domain string Certificate string Key string @@ -16,14 +16,14 @@ type domainConfig struct { AccessControl bool `json:"access_control"` } -type domainsConfig struct { - Domains []domainConfig +type legacyDomainsConfig struct { + Domains []Config HTTPSOnly bool `json:"https_only"` ID uint64 `json:"id"` AccessControl bool `json:"access_control"` } -func (c *domainConfig) Valid(rootDomain string) bool { +func (c *Config) Valid(rootDomain string) bool { if c.Domain == "" { return false } @@ -34,7 +34,7 @@ func (c *domainConfig) Valid(rootDomain string) bool { return !strings.HasSuffix(domain, rootDomain) } -func (c *domainsConfig) Read(group, project string) (err error) { +func (c *legacyDomainsConfig) Read(group, project string) (err error) { configFile, err := os.Open(filepath.Join(group, project, "config.json")) if err != nil { return err diff --git a/internal/domain/domain_config_test.go b/internal/domain/config_test.go similarity index 94% rename from internal/domain/domain_config_test.go rename to internal/domain/config_test.go index 8cdcdeaa6bdb9ef9eec4161ac4ef37edf3bae9fb..f7b6435b50edd67388afc1488023e2e3b3fbc16f 100644 --- a/internal/domain/domain_config_test.go +++ b/internal/domain/config_test.go @@ -13,6 +13,10 @@ const configFile = "test-group/test-project/config.json" const invalidConfig = `{"Domains":{}}` const validConfig = `{"Domains":[{"Domain":"test"}]}` +// temporary type alias +type domainsConfig = legacyDomainsConfig +type domainConfig = Config + func TestDomainConfigValidness(t *testing.T) { d := domainConfig{} require.False(t, d.Valid("gitlab.io")) diff --git a/internal/domain/domain.go b/internal/domain/domain.go index 2f499ff971b58095cd3fe0a6a4db07c10ab19f30..97b95f79c06963e333db8a4944c8fbded3d45e34 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -3,22 +3,13 @@ package domain import ( "crypto/tls" "errors" - "fmt" - "io" - "mime" "net/http" - "os" - "path/filepath" - "strconv" "strings" "sync" "time" - "golang.org/x/sys/unix" - "gitlab.com/gitlab-org/gitlab-pages/internal/host" - "gitlab.com/gitlab-org/gitlab-pages/internal/httperrors" - "gitlab.com/gitlab-org/gitlab-pages/internal/httputil" + "gitlab.com/gitlab-org/gitlab-pages/internal/source" ) const ( @@ -29,15 +20,8 @@ const ( maxProjectDepth int = subgroupScanLimit + 3 ) -type locationDirectoryError struct { - FullPath string - RelativePath string -} - -type locationFileNoExtensionError struct { - FullPath string -} - +// Project represents a projects associated with a domain / requested domain +// path type project struct { NamespaceProject bool HTTPSOnly bool @@ -47,11 +31,12 @@ type project struct { // Domain is a domain that gitlab-pages can serve. type Domain struct { - group + source *source.Disk - // custom domains: - projectName string - config *domainConfig + // custom domains + project string + group group + config *Config certificate *tls.Certificate certificateError error @@ -60,50 +45,15 @@ type Domain struct { // String implements Stringer. func (d *Domain) String() string { - if d.group.name != "" && d.projectName != "" { - return d.group.name + "/" + d.projectName + if d.group.name != "" && d.project != "" { + return d.group.name + "/" + d.project } if d.group.name != "" { return d.group.name } - return d.projectName -} - -func (l *locationDirectoryError) Error() string { - return "location error accessing directory where file expected" -} - -func (l *locationFileNoExtensionError) Error() string { - return "error accessing a path without an extension" -} - -func acceptsGZip(r *http.Request) bool { - if r.Header.Get("Range") != "" { - return false - } - - offers := []string{"gzip", "identity"} - acceptedEncoding := httputil.NegotiateContentEncoding(r, offers) - return acceptedEncoding == "gzip" -} - -func handleGZip(w http.ResponseWriter, r *http.Request, fullPath string) string { - if !acceptsGZip(r) { - return fullPath - } - - gzipPath := fullPath + ".gz" - - // Ensure the .gz file is not a symlink - if fi, err := os.Lstat(gzipPath); err != nil || !fi.Mode().IsRegular() { - return fullPath - } - - w.Header().Set("Content-Encoding", "gzip") - - return gzipPath + return d.project } // Look up a project inside the domain based on the host and path. Returns the @@ -113,7 +63,7 @@ func (d *Domain) getProjectWithSubpath(r *http.Request) (*project, string, strin // If present, these projects shadow the group domain. split := strings.SplitN(r.URL.Path, "/", maxProjectDepth) if len(split) >= 2 { - project, projectPath, urlPath := d.digProjectWithSubpath("", split[1:]) + project, projectPath, urlPath := d.group.digProjectWithSubpath("", split[1:]) if project != nil { return project, projectPath, urlPath } @@ -122,7 +72,7 @@ func (d *Domain) getProjectWithSubpath(r *http.Request) (*project, string, strin // Since the URL doesn't specify a project (e.g. http://mydomain.gitlab.io), // return the group project if it exists. if host := host.FromRequest(r); host != "" { - if groupProject := d.projects[host]; groupProject != nil { + if groupProject := d.group.projects[host]; groupProject != nil { return groupProject, host, strings.Join(split[1:], "/") } } @@ -175,24 +125,7 @@ func (d *Domain) HasAcmeChallenge(token string) bool { return false } - if d.config == nil { - return false - } - - _, err := d.resolvePath(d.projectName, ".well-known/acme-challenge", token) - - // there is an acme challenge on disk - if err == nil { - return true - } - - _, err = d.resolvePath(d.projectName, ".well-known/acme-challenge", token, "index.html") - - if err == nil { - return true - } - - return false + return d.source.HasAcmeChallenge(token) } // IsNamespaceProject figures out if the request is to a namespace project @@ -249,233 +182,6 @@ func (d *Domain) HasProject(r *http.Request) bool { return false } -// Detect file's content-type either by extension or mime-sniffing. -// Implementation is adapted from Golang's `http.serveContent()` -// See https://github.com/golang/go/blob/902fc114272978a40d2e65c2510a18e870077559/src/net/http/fs.go#L194 -func (d *Domain) detectContentType(path string) (string, error) { - contentType := mime.TypeByExtension(filepath.Ext(path)) - - if contentType == "" { - var buf [512]byte - - file, err := os.Open(path) - if err != nil { - return "", err - } - - defer file.Close() - - // Using `io.ReadFull()` because `file.Read()` may be chunked. - // Ignoring errors because we don't care if the 512 bytes cannot be read. - n, _ := io.ReadFull(file, buf[:]) - contentType = http.DetectContentType(buf[:n]) - } - - return contentType, nil -} - -func (d *Domain) serveFile(w http.ResponseWriter, r *http.Request, origPath string) error { - fullPath := handleGZip(w, r, origPath) - - file, err := openNoFollow(fullPath) - if err != nil { - return err - } - - defer file.Close() - - fi, err := file.Stat() - if err != nil { - return err - } - - if !d.IsAccessControlEnabled(r) { - // Set caching headers - w.Header().Set("Cache-Control", "max-age=600") - w.Header().Set("Expires", time.Now().Add(10*time.Minute).Format(time.RFC1123)) - } - - contentType, err := d.detectContentType(origPath) - if err != nil { - return err - } - - w.Header().Set("Content-Type", contentType) - http.ServeContent(w, r, origPath, fi.ModTime(), file) - - return nil -} - -func (d *Domain) serveCustomFile(w http.ResponseWriter, r *http.Request, code int, origPath string) error { - fullPath := handleGZip(w, r, origPath) - - // Open and serve content of file - file, err := openNoFollow(fullPath) - if err != nil { - return err - } - defer file.Close() - - fi, err := file.Stat() - if err != nil { - return err - } - - contentType, err := d.detectContentType(origPath) - if err != nil { - return err - } - - w.Header().Set("Content-Type", contentType) - w.Header().Set("Content-Length", strconv.FormatInt(fi.Size(), 10)) - w.WriteHeader(code) - - if r.Method != "HEAD" { - _, err := io.CopyN(w, file, fi.Size()) - return err - } - - return nil -} - -// Resolve the HTTP request to a path on disk, converting requests for -// directories to requests for index.html inside the directory if appropriate. -func (d *Domain) resolvePath(projectName string, subPath ...string) (string, error) { - publicPath := filepath.Join(d.group.name, projectName, "public") - - // Don't use filepath.Join as cleans the path, - // where we want to traverse full path as supplied by user - // (including ..) - testPath := publicPath + "/" + strings.Join(subPath, "/") - fullPath, err := filepath.EvalSymlinks(testPath) - if err != nil { - if endsWithoutHTMLExtension(testPath) { - return "", &locationFileNoExtensionError{ - FullPath: fullPath, - } - } - - return "", err - } - - // The requested path resolved to somewhere outside of the public/ directory - if !strings.HasPrefix(fullPath, publicPath+"/") && fullPath != publicPath { - return "", fmt.Errorf("%q should be in %q", fullPath, publicPath) - } - - fi, err := os.Lstat(fullPath) - if err != nil { - return "", err - } - - // The requested path is a directory, so try index.html via recursion - if fi.IsDir() { - return "", &locationDirectoryError{ - FullPath: fullPath, - RelativePath: strings.TrimPrefix(fullPath, publicPath), - } - } - - // The file exists, but is not a supported type to serve. Perhaps a block - // special device or something else that may be a security risk. - if !fi.Mode().IsRegular() { - return "", fmt.Errorf("%s: is not a regular file", fullPath) - } - - return fullPath, nil -} - -func (d *Domain) tryNotFound(w http.ResponseWriter, r *http.Request, projectName string) error { - page404, err := d.resolvePath(projectName, "404.html") - if err != nil { - return err - } - - err = d.serveCustomFile(w, r, http.StatusNotFound, page404) - if err != nil { - return err - } - return nil -} - -func (d *Domain) tryFile(w http.ResponseWriter, r *http.Request, projectName string, subPath ...string) error { - fullPath, err := d.resolvePath(projectName, subPath...) - - if locationError, _ := err.(*locationDirectoryError); locationError != nil { - if endsWithSlash(r.URL.Path) { - fullPath, err = d.resolvePath(projectName, filepath.Join(subPath...), "index.html") - } else { - // Concat Host with URL.Path - redirectPath := "//" + r.Host + "/" - redirectPath += strings.TrimPrefix(r.URL.Path, "/") - - // Ensure that there's always "/" at end - redirectPath = strings.TrimSuffix(redirectPath, "/") + "/" - http.Redirect(w, r, redirectPath, 302) - return nil - } - } - - if locationError, _ := err.(*locationFileNoExtensionError); locationError != nil { - fullPath, err = d.resolvePath(projectName, strings.TrimSuffix(filepath.Join(subPath...), "/")+".html") - } - - if err != nil { - return err - } - - return d.serveFile(w, r, fullPath) -} - -func (d *Domain) serveFileFromGroup(w http.ResponseWriter, r *http.Request) bool { - project, projectName, subPath := d.getProjectWithSubpath(r) - if project == nil { - httperrors.Serve404(w) - return true - } - - if d.tryFile(w, r, projectName, subPath) == nil { - return true - } - - return false -} - -func (d *Domain) serveNotFoundFromGroup(w http.ResponseWriter, r *http.Request) { - project, projectName, _ := d.getProjectWithSubpath(r) - if project == nil { - httperrors.Serve404(w) - return - } - - // Try serving custom not-found page - if d.tryNotFound(w, r, projectName) == nil { - return - } - - // Generic 404 - httperrors.Serve404(w) -} - -func (d *Domain) serveFileFromConfig(w http.ResponseWriter, r *http.Request) bool { - // Try to serve file for http://host/... => /group/project/... - if d.tryFile(w, r, d.projectName, r.URL.Path) == nil { - return true - } - - return false -} - -func (d *Domain) serveNotFoundFromConfig(w http.ResponseWriter, r *http.Request) { - // Try serving not found page for http://host/ => /group/project/404.html - if d.tryNotFound(w, r, d.projectName) == nil { - return - } - - // Serve generic not found - httperrors.Serve404(w) -} - // EnsureCertificate parses the PEM-encoded certificate for the domain func (d *Domain) EnsureCertificate() (*tls.Certificate, error) { if d.config == nil { @@ -493,42 +199,28 @@ func (d *Domain) EnsureCertificate() (*tls.Certificate, error) { return d.certificate, d.certificateError } -// ServeFileHTTP implements http.Handler. Returns true if something was served, false if not. -func (d *Domain) ServeFileHTTP(w http.ResponseWriter, r *http.Request) bool { - if d == nil { - httperrors.Serve404(w) - return true - } +func (d *Domain) getServing(w http.ResponseWriter, r *http.Request) *source.Serving { + _, project, subPath := d.getProjectWithSubpath(r) - if d.config != nil { - return d.serveFileFromConfig(w, r) + return &source.Serving{ + Writer: w, Request: r, Project: project, SubPath: subPath, } - - return d.serveFileFromGroup(w, r) } -// ServeNotFoundHTTP implements http.Handler. Serves the not found pages from the projects. -func (d *Domain) ServeNotFoundHTTP(w http.ResponseWriter, r *http.Request) { - if d == nil { - httperrors.Serve404(w) - return +func (d *Domain) ServeFileHTTP(w http.ResponseWriter, r *http.Request) bool { + if !d.IsAccessControlEnabled(r) { + // Set caching headers + w.Header().Set("Cache-Control", "max-age=600") + w.Header().Set("Expires", time.Now().Add(10*time.Minute).Format(time.RFC1123)) } - if d.config != nil { - d.serveNotFoundFromConfig(w, r) - } else { - d.serveNotFoundFromGroup(w, r) - } -} + serving := d.getServing(w, r) -func endsWithSlash(path string) bool { - return strings.HasSuffix(path, "/") + return d.source.ServeFileHTTP(serving) } -func endsWithoutHTMLExtension(path string) bool { - return !strings.HasSuffix(path, ".html") -} +func (d *Domain) ServeNotFoundHTTP(w http.ResponseWriter, r *http.Request) { + serving := d.getServing(w, r) -func openNoFollow(path string) (*os.File, error) { - return os.OpenFile(path, os.O_RDONLY|unix.O_NOFOLLOW, 0) + d.source.ServeNotFoundHTTP(serving) } diff --git a/internal/domain/domain_test.go b/internal/domain/domain_test.go index d5db33c96feecad577b63d0b90e93a962454771f..90b836e5e803a2ca1211d86ce9224664d52d64de 100644 --- a/internal/domain/domain_test.go +++ b/internal/domain/domain_test.go @@ -26,7 +26,7 @@ func serveFileOrNotFound(domain *Domain) http.HandlerFunc { func testGroupServeHTTPHost(t *testing.T, host string) { testGroup := &Domain{ - projectName: "", + project: "", group: group{ name: "group", projects: map[string]*project{ @@ -80,8 +80,8 @@ func TestDomainServeHTTP(t *testing.T) { defer cleanup() testDomain := &Domain{ - group: group{name: "group"}, - projectName: "project2", + group: group{name: "group"}, + project: "project2", config: &domainConfig{ Domain: "test.domain.com", }, @@ -108,9 +108,9 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Custom domain with HTTPS-only enabled", domain: &Domain{ - group: group{name: "group"}, - projectName: "project", - config: &domainConfig{HTTPSOnly: true}, + group: group{name: "group"}, + project: "project", + config: &domainConfig{HTTPSOnly: true}, }, url: "http://custom-domain", expected: true, @@ -118,9 +118,9 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Custom domain with HTTPS-only disabled", domain: &Domain{ - group: group{name: "group"}, - projectName: "project", - config: &domainConfig{HTTPSOnly: false}, + group: group{name: "group"}, + project: "project", + config: &domainConfig{HTTPSOnly: false}, }, url: "http://custom-domain", expected: false, @@ -128,7 +128,7 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Default group domain with HTTPS-only enabled", domain: &Domain{ - projectName: "project", + project: "project", group: group{ name: "group", projects: projects{"test-domain": &project{HTTPSOnly: true}}, @@ -140,7 +140,7 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Default group domain with HTTPS-only disabled", domain: &Domain{ - projectName: "project", + project: "project", group: group{ name: "group", projects: projects{"test-domain": &project{HTTPSOnly: false}}, @@ -152,7 +152,7 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Case-insensitive default group domain with HTTPS-only enabled", domain: &Domain{ - projectName: "project", + project: "project", group: group{ name: "group", projects: projects{"test-domain": &project{HTTPSOnly: true}}, @@ -164,7 +164,7 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Other group domain with HTTPS-only enabled", domain: &Domain{ - projectName: "project", + project: "project", group: group{ name: "group", projects: projects{"project": &project{HTTPSOnly: true}}, @@ -176,7 +176,7 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Other group domain with HTTPS-only disabled", domain: &Domain{ - projectName: "project", + project: "project", group: group{ name: "group", projects: projects{"project": &project{HTTPSOnly: false}}, @@ -188,8 +188,8 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Unknown project", domain: &Domain{ - group: group{name: "group"}, - projectName: "project", + group: group{name: "group"}, + project: "project", }, url: "http://test-domain/project", expected: false, @@ -217,9 +217,9 @@ func TestHasAcmeChallenge(t *testing.T) { { name: "Project containing acme challenge", domain: &Domain{ - group: group{name: "group.acme"}, - projectName: "with.acme.challenge", - config: &domainConfig{HTTPSOnly: true}, + group: group{name: "group.acme"}, + project: "with.acme.challenge", + config: &domainConfig{HTTPSOnly: true}, }, token: "existingtoken", expected: true, @@ -227,9 +227,9 @@ func TestHasAcmeChallenge(t *testing.T) { { name: "Project containing acme challenge", domain: &Domain{ - group: group{name: "group.acme"}, - projectName: "with.acme.challenge", - config: &domainConfig{HTTPSOnly: true}, + group: group{name: "group.acme"}, + project: "with.acme.challenge", + config: &domainConfig{HTTPSOnly: true}, }, token: "foldertoken", expected: true, @@ -237,9 +237,9 @@ func TestHasAcmeChallenge(t *testing.T) { { name: "Project containing another token", domain: &Domain{ - group: group{name: "group.acme"}, - projectName: "with.acme.challenge", - config: &domainConfig{HTTPSOnly: true}, + group: group{name: "group.acme"}, + project: "with.acme.challenge", + config: &domainConfig{HTTPSOnly: true}, }, token: "notexistingtoken", expected: false, @@ -253,9 +253,9 @@ func TestHasAcmeChallenge(t *testing.T) { { name: "Domain without config", domain: &Domain{ - group: group{name: "group.acme"}, - projectName: "with.acme.challenge", - config: nil, + group: group{name: "group.acme"}, + project: "with.acme.challenge", + config: nil, }, token: "existingtoken", expected: false, @@ -300,7 +300,7 @@ func TestGroupServeHTTPGzip(t *testing.T) { defer cleanup() testGroup := &Domain{ - projectName: "", + project: "", group: group{ name: "group", projects: map[string]*project{ @@ -367,7 +367,7 @@ func TestGroup404ServeHTTP(t *testing.T) { defer cleanup() testGroup := &Domain{ - projectName: "", + project: "", group: group{ name: "group.404", projects: map[string]*project{ @@ -396,8 +396,8 @@ func TestDomain404ServeHTTP(t *testing.T) { defer cleanup() testDomain := &Domain{ - group: group{name: "group.404"}, - projectName: "domain.404", + group: group{name: "group.404"}, + project: "domain.404", config: &domainConfig{ Domain: "domain.404.com", }, @@ -420,8 +420,8 @@ func TestPredefined404ServeHTTP(t *testing.T) { func TestGroupCertificate(t *testing.T) { testGroup := &Domain{ - group: group{name: "group"}, - projectName: "", + group: group{name: "group"}, + project: "", } tls, err := testGroup.EnsureCertificate() @@ -431,8 +431,8 @@ func TestGroupCertificate(t *testing.T) { func TestDomainNoCertificate(t *testing.T) { testDomain := &Domain{ - group: group{name: "group"}, - projectName: "project2", + group: group{name: "group"}, + project: "project2", config: &domainConfig{ Domain: "test.domain.com", }, @@ -449,8 +449,8 @@ func TestDomainNoCertificate(t *testing.T) { func TestDomainCertificate(t *testing.T) { testDomain := &Domain{ - group: group{name: "group"}, - projectName: "project2", + group: group{name: "group"}, + project: "project2", config: &domainConfig{ Domain: "test.domain.com", Certificate: fixture.Certificate, @@ -494,29 +494,6 @@ func TestCacheControlHeaders(t *testing.T) { require.WithinDuration(t, now.UTC().Add(10*time.Minute), expiresTime.UTC(), time.Minute) } -func TestOpenNoFollow(t *testing.T) { - tmpfile, err := ioutil.TempFile("", "link-test") - require.NoError(t, err) - defer tmpfile.Close() - - orig := tmpfile.Name() - softLink := orig + ".link" - defer os.Remove(orig) - - source, err := openNoFollow(orig) - require.NoError(t, err) - require.NotNil(t, source) - defer source.Close() - - err = os.Symlink(orig, softLink) - require.NoError(t, err) - defer os.Remove(softLink) - - link, err := openNoFollow(softLink) - require.Error(t, err) - require.Nil(t, link) -} - var chdirSet = false func setUpTests(t require.TestingT) func() { diff --git a/internal/domain/map.go b/internal/domain/map.go index 7934860bb9e5e2d8afe8e08d22688aa5663e884e..2e7bbf28cc112a8284ec87c16c03349dd4fb8d6d 100644 --- a/internal/domain/map.go +++ b/internal/domain/map.go @@ -12,6 +12,7 @@ import ( "github.com/karrick/godirwalk" log "github.com/sirupsen/logrus" + "gitlab.com/gitlab-org/gitlab-pages/internal/source" "gitlab.com/gitlab-org/gitlab-pages/metrics" ) @@ -25,20 +26,25 @@ func (dm Map) updateDomainMap(domainName string, domain *Domain) { log.WithFields(log.Fields{ "domain_name": domainName, "new_group": domain.group, - "new_project_name": domain.projectName, + "new_project_name": domain.project, "old_group": old.group, - "old_project_name": old.projectName, + "old_project_name": old.project, }).Error("Duplicate domain") } dm[domainName] = domain } -func (dm Map) addDomain(rootDomain, groupName, projectName string, config *domainConfig) { +func (dm Map) addDomain(rootDomain, groupName, projectName string, config *Config) { newDomain := &Domain{ - group: group{name: groupName}, - projectName: projectName, - config: config, + group: group{name: groupName}, + project: projectName, + config: config, + source: &source.Disk{ + Project: projectName, + Group: groupName, + CustomDomain: (config != nil), + }, } var domainName string @@ -89,7 +95,7 @@ func (dm Map) updateGroupDomain(rootDomain, groupName, projectPath string, https dm[domainName] = groupDomain } -func (dm Map) readProjectConfig(rootDomain string, group, projectName string, config *domainsConfig) { +func (dm Map) readProjectConfig(rootDomain string, group, projectName string, config *legacyDomainsConfig) { if config == nil { // This is necessary to preserve the previous behaviour where a // group domain is created even if no config.json files are @@ -131,7 +137,7 @@ func readProject(group, parent, projectName string, level int, fanIn chan<- jobR // We read the config.json file _before_ fanning in, because it does disk // IO and it does not need access to the domains map. - config := &domainsConfig{} + config := &legacyDomainsConfig{} if err := config.Read(group, projectPath); err != nil { config = nil } @@ -163,7 +169,7 @@ func readProjects(group, parent string, level int, buf []byte, fanIn chan<- jobR type jobResult struct { group string project string - config *domainsConfig + config *legacyDomainsConfig } // ReadGroups walks the pages directory and populates dm with all the domains it finds. diff --git a/internal/domain/map_test.go b/internal/domain/map_test.go index de5e49552e542583a696a38adcbb3c4a1e3f78b6..2d61d248f6728ea1e536bf37646f5cb19ed024da 100644 --- a/internal/domain/map_test.go +++ b/internal/domain/map_test.go @@ -77,7 +77,7 @@ func TestReadProjects(t *testing.T) { // check subgroups domain, ok := dm["group.test.io"] require.True(t, ok, "missing group.test.io domain") - subgroup, ok := domain.subgroups["subgroup"] + subgroup, ok := domain.group.subgroups["subgroup"] require.True(t, ok, "missing group.test.io subgroup") _, ok = subgroup.projects["project"] require.True(t, ok, "missing project for subgroup in group.test.io domain") diff --git a/internal/source/disk.go b/internal/source/disk.go new file mode 100644 index 0000000000000000000000000000000000000000..0614d07e129500a15eb09a89de52843777ca3875 --- /dev/null +++ b/internal/source/disk.go @@ -0,0 +1,354 @@ +package source + +import ( + "fmt" + "io" + "mime" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + + "gitlab.com/gitlab-org/gitlab-pages/internal/httperrors" + "gitlab.com/gitlab-org/gitlab-pages/internal/httputil" + "golang.org/x/sys/unix" +) + +// File struct represent the on-disk source for a GitLab Pages Domain. +type Disk struct { + Project string + Group string + CustomDomain bool +} + +type locationDirectoryError struct { + FullPath string + RelativePath string +} + +type locationFileNoExtensionError struct { + FullPath string +} + +func (l *locationDirectoryError) Error() string { + return "location error accessing directory where file expected" +} + +func (l *locationFileNoExtensionError) Error() string { + return "error accessing a path without an extension" +} + +// Writes to the http.ResponseWritter. Returns true if something was served, false if not. +func (d *Disk) ServeFileHTTP(serving *Serving) bool { + if d == nil { // TODO do we need that + httperrors.Serve404(serving.Writer) + return true + } + + if d.CustomDomain { + return d.serveFileFromConfig(serving) + } + + return d.serveFileFromGroup(serving) +} + +// Writes to the http.ResponseWriter. Serves the not found page from a project. +func (d *Disk) ServeNotFoundHTTP(serving *Serving) { + if d == nil { // TODO do we need that + httperrors.Serve404(serving.Writer) + return + } + + if d.CustomDomain { + d.serveNotFoundFromConfig(serving) + } else { + d.serveNotFoundFromGroup(serving) + } +} + +func (d *Disk) serveFileFromGroup(serving *Serving) bool { + if !serving.IsProjectFound() { + httperrors.Serve404(serving.Writer) + return true + } + + if d.tryFile(serving.Writer, serving.Request, serving.Project, serving.SubPath) == nil { + return true + } + + return false +} + +func (d *Disk) serveNotFoundFromGroup(serving *Serving) { + if !serving.IsProjectFound() { + httperrors.Serve404(serving.Writer) + return + } + + // Try serving custom not-found page + if d.tryNotFound(serving.Writer, serving.Request, serving.Project) == nil { + return + } + + // Generic 404 + httperrors.Serve404(serving.Writer) +} + +// TODO can we use r.project instead of d.project? +func (d *Disk) serveFileFromConfig(serving *Serving) bool { + // Try to serve file for http://host/... => /group/project/... + if d.tryFile(serving.Writer, serving.Request, serving.Project, serving.requestPath()) == nil { + return true + } + + return false +} + +// TODO can we use r.project instead of d.project? +func (d *Disk) serveNotFoundFromConfig(serving *Serving) { + // Try serving not found page for http://host/ => /group/project/404.html + if d.tryNotFound(serving.Writer, serving.Request, serving.Project) == nil { + return + } + + // Serve generic not found + httperrors.Serve404(serving.Writer) +} + +func (d *Disk) tryFile(w http.ResponseWriter, r *http.Request, projectName string, subPath ...string) error { + fullPath, err := d.resolvePath(projectName, subPath...) + + if locationError, _ := err.(*locationDirectoryError); locationError != nil { + if endsWithSlash(r.URL.Path) { + fullPath, err = d.resolvePath(projectName, filepath.Join(subPath...), "index.html") + } else { + // Concat Host with URL.Path + redirectPath := "//" + r.Host + "/" + redirectPath += strings.TrimPrefix(r.URL.Path, "/") + + // Ensure that there's always "/" at end + redirectPath = strings.TrimSuffix(redirectPath, "/") + "/" + http.Redirect(w, r, redirectPath, 302) + return nil + } + } + + if locationError, _ := err.(*locationFileNoExtensionError); locationError != nil { + fullPath, err = d.resolvePath(projectName, strings.TrimSuffix(filepath.Join(subPath...), "/")+".html") + } + + if err != nil { + return err + } + + return d.serveFile(w, r, fullPath) +} + +func (d *Disk) tryNotFound(w http.ResponseWriter, r *http.Request, projectName string) error { + page404, err := d.resolvePath(projectName, "404.html") + if err != nil { + return err + } + + err = d.serveCustomFile(w, r, http.StatusNotFound, page404) + if err != nil { + return err + } + return nil +} + +// Resolve the HTTP request to a path on disk, converting requests for +// directories to requests for index.html inside the directory if appropriate. +func (d *Disk) resolvePath(projectName string, subPath ...string) (string, error) { + publicPath := filepath.Join(d.Group, projectName, "public") + + // Don't use filepath.Join as cleans the path, + // where we want to traverse full path as supplied by user + // (including ..) + testPath := publicPath + "/" + strings.Join(subPath, "/") + fullPath, err := filepath.EvalSymlinks(testPath) + if err != nil { + if endsWithoutHTMLExtension(testPath) { + return "", &locationFileNoExtensionError{ + FullPath: fullPath, + } + } + + return "", err + } + + // The requested path resolved to somewhere outside of the public/ directory + if !strings.HasPrefix(fullPath, publicPath+"/") && fullPath != publicPath { + return "", fmt.Errorf("%q should be in %q", fullPath, publicPath) + } + + fi, err := os.Lstat(fullPath) + if err != nil { + return "", err + } + + // The requested path is a directory, so try index.html via recursion + if fi.IsDir() { + return "", &locationDirectoryError{ + FullPath: fullPath, + RelativePath: strings.TrimPrefix(fullPath, publicPath), + } + } + + // The file exists, but is not a supported type to serve. Perhaps a block + // special device or something else that may be a security risk. + if !fi.Mode().IsRegular() { + return "", fmt.Errorf("%s: is not a regular file", fullPath) + } + + return fullPath, nil +} + +func (d *Disk) serveCustomFile(w http.ResponseWriter, r *http.Request, code int, origPath string) error { + fullPath := handleGZip(w, r, origPath) + + // Open and serve content of file + file, err := openNoFollow(fullPath) + if err != nil { + return err + } + defer file.Close() + + fi, err := file.Stat() + if err != nil { + return err + } + + contentType, err := d.detectContentType(origPath) + if err != nil { + return err + } + + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", strconv.FormatInt(fi.Size(), 10)) + w.WriteHeader(code) + + if r.Method != "HEAD" { + _, err := io.CopyN(w, file, fi.Size()) + return err + } + + return nil +} + +func (d *Disk) serveFile(w http.ResponseWriter, r *http.Request, origPath string) error { + fullPath := handleGZip(w, r, origPath) + + file, err := openNoFollow(fullPath) + if err != nil { + return err + } + + defer file.Close() + + fi, err := file.Stat() + if err != nil { + return err + } + + contentType, err := d.detectContentType(origPath) + if err != nil { + return err + } + + w.Header().Set("Content-Type", contentType) + http.ServeContent(w, r, origPath, fi.ModTime(), file) + + return nil +} + +// Detect file's content-type either by extension or mime-sniffing. +// Implementation is adapted from Golang's `http.serveContent()` +// See https://github.com/golang/go/blob/902fc114272978a40d2e65c2510a18e870077559/src/net/http/fs.go#L194 +func (d *Disk) detectContentType(path string) (string, error) { + contentType := mime.TypeByExtension(filepath.Ext(path)) + + if contentType == "" { + var buf [512]byte + + file, err := os.Open(path) + if err != nil { + return "", err + } + + defer file.Close() + + // Using `io.ReadFull()` because `file.Read()` may be chunked. + // Ignoring errors because we don't care if the 512 bytes cannot be read. + n, _ := io.ReadFull(file, buf[:]) + contentType = http.DetectContentType(buf[:n]) + } + + return contentType, nil +} + +func acceptsGZip(r *http.Request) bool { + if r.Header.Get("Range") != "" { + return false + } + + offers := []string{"gzip", "identity"} + acceptedEncoding := httputil.NegotiateContentEncoding(r, offers) + return acceptedEncoding == "gzip" +} + +func handleGZip(w http.ResponseWriter, r *http.Request, fullPath string) string { + if !acceptsGZip(r) { + return fullPath + } + + gzipPath := fullPath + ".gz" + + // Ensure the .gz file is not a symlink + if fi, err := os.Lstat(gzipPath); err != nil || !fi.Mode().IsRegular() { + return fullPath + } + + w.Header().Set("Content-Encoding", "gzip") + + return gzipPath +} + +func endsWithSlash(path string) bool { + return strings.HasSuffix(path, "/") +} + +func endsWithoutHTMLExtension(path string) bool { + return !strings.HasSuffix(path, ".html") +} + +func openNoFollow(path string) (*os.File, error) { + return os.OpenFile(path, os.O_RDONLY|unix.O_NOFOLLOW, 0) +} + +// HasAcmeChallenge checks domain directory contains particular acme challenge +func (d *Disk) HasAcmeChallenge(token string) bool { + if d == nil { + return false + } + + if !d.CustomDomain { + return false + } + + _, err := d.resolvePath(d.Project, ".well-known/acme-challenge", token) + + // there is an acme challenge on disk + if err == nil { + return true + } + + _, err = d.resolvePath(d.Project, ".well-known/acme-challenge", token, "index.html") + + if err == nil { + return true + } + + return false +} diff --git a/internal/source/disk_test.go b/internal/source/disk_test.go new file mode 100644 index 0000000000000000000000000000000000000000..cbc1fd9305068e5186a3d3ef3080b4cbd41c40a7 --- /dev/null +++ b/internal/source/disk_test.go @@ -0,0 +1,32 @@ +package source + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestOpenNoFollow(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "link-test") + require.NoError(t, err) + defer tmpfile.Close() + + orig := tmpfile.Name() + softLink := orig + ".link" + defer os.Remove(orig) + + source, err := openNoFollow(orig) + require.NoError(t, err) + require.NotNil(t, source) + defer source.Close() + + err = os.Symlink(orig, softLink) + require.NoError(t, err) + defer os.Remove(softLink) + + link, err := openNoFollow(softLink) + require.Error(t, err) + require.Nil(t, link) +} diff --git a/internal/source/serving.go b/internal/source/serving.go new file mode 100644 index 0000000000000000000000000000000000000000..6a77c219fe7528245f6b82948502de6dfec3e7d3 --- /dev/null +++ b/internal/source/serving.go @@ -0,0 +1,24 @@ +package source + +import ( + "net/http" +) + +// Serving represents a request and a response we use to serve pages for a +// given project and optional subpath in case of a domain with a subgroup +// access component +type Serving struct { + Request *http.Request + Writer http.ResponseWriter + Project string // project name + SubPath string // request path +} + +// TODO test, refactor related code +func (s *Serving) IsProjectFound() bool { + return len(s.Project) > 0 +} + +func (s *Serving) requestPath() string { + return s.Request.URL.Path +} diff --git a/internal/source/source.go b/internal/source/source.go new file mode 100644 index 0000000000000000000000000000000000000000..43f6be1534e21800c6adde27efd652bc81c393e2 --- /dev/null +++ b/internal/source/source.go @@ -0,0 +1,9 @@ +package source + +import "net/http" + +// Source interface is a common interface between all sources in the source +// package. +type Source interface { // TODO define me better + Serve(w http.ResponseWriter, r *http.Request) bool +}