diff --git a/acceptance_test.go b/acceptance_test.go index 55a838812038e4bcd5bb88592516a8f3d874b26f..3173b9bc0a08ea2b469bc0ec39c8de82fffc19db 100644 --- a/acceptance_test.go +++ b/acceptance_test.go @@ -10,7 +10,6 @@ import ( "net/http/httptest" "net/url" "os" - "path" "regexp" "testing" "time" @@ -153,30 +152,15 @@ func TestKnownHostReturns200(t *testing.T) { func TestNestedSubgroups(t *testing.T) { skipUnlessEnabled(t) - maxNestedSubgroup := 21 - - pagesRoot, err := ioutil.TempDir("", "pages-root") - require.NoError(t, err) - defer os.RemoveAll(pagesRoot) - - makeProjectIndex := func(subGroupPath string) { - projectPath := path.Join(pagesRoot, "nested", subGroupPath, "project", "public") - require.NoError(t, os.MkdirAll(projectPath, 0755)) - - projectIndex := path.Join(projectPath, "index.html") - require.NoError(t, ioutil.WriteFile(projectIndex, []byte("index"), 0644)) - } - makeProjectIndex("") + maxNestedSubgroup := 5 paths := []string{""} for i := 1; i < maxNestedSubgroup*2; i++ { subGroupPath := fmt.Sprintf("%ssub%d/", paths[i-1], i) paths = append(paths, subGroupPath) - - makeProjectIndex(subGroupPath) } - teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-pages-root", pagesRoot) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") defer teardown() for nestingLevel, path := range paths { @@ -381,7 +365,6 @@ func TestPrometheusMetricsCanBeScraped(t *testing.T) { body, _ := ioutil.ReadAll(resp.Body) assert.Contains(t, string(body), "gitlab_pages_http_sessions_active 0") - assert.Contains(t, string(body), "gitlab_pages_domains_served_total 14") } } @@ -396,30 +379,6 @@ func TestStatusPage(t *testing.T) { assert.Equal(t, http.StatusOK, rsp.StatusCode) } -func TestStatusNotYetReady(t *testing.T) { - skipUnlessEnabled(t) - teardown := RunPagesProcessWithoutWait(t, *pagesBinary, listeners, "", "-pages-status=/@statuscheck", "-pages-root=shared/invalid-pages") - defer teardown() - - waitForRoundtrips(t, listeners, 5*time.Second) - rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com", "@statuscheck") - require.NoError(t, err) - defer rsp.Body.Close() - assert.Equal(t, http.StatusServiceUnavailable, rsp.StatusCode) -} - -func TestPageNotAvailableIfNotLoaded(t *testing.T) { - skipUnlessEnabled(t) - teardown := RunPagesProcessWithoutWait(t, *pagesBinary, listeners, "", "-pages-root=shared/invalid-pages") - defer teardown() - waitForRoundtrips(t, listeners, 5*time.Second) - - rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com", "index.html") - require.NoError(t, err) - defer rsp.Body.Close() - assert.Equal(t, http.StatusServiceUnavailable, rsp.StatusCode) -} - func TestObscureMIMEType(t *testing.T) { skipUnlessEnabled(t) teardown := RunPagesProcessWithoutWait(t, *pagesBinary, listeners, "") @@ -880,6 +839,7 @@ func TestAccessControlGroupDomain404RedirectsAuth(t *testing.T) { assert.Equal(t, "projects.gitlab-example.com", url.Host) assert.Equal(t, "/auth", url.Path) } + func TestAccessControlProject404DoesNotRedirect(t *testing.T) { skipUnlessEnabled(t) teardown := RunPagesProcessWithAuth(t, *pagesBinary, listeners, "") diff --git a/app.go b/app.go index b872a6a1ee40856ca2358fbfa0394443187ce916..1e347c3d7b1305757598842553320b580ad35a73 100644 --- a/app.go +++ b/app.go @@ -9,7 +9,6 @@ import ( "strconv" "strings" "sync" - "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -20,6 +19,7 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/internal/admin" "gitlab.com/gitlab-org/gitlab-pages/internal/artifact" "gitlab.com/gitlab-org/gitlab-pages/internal/auth" + "gitlab.com/gitlab-org/gitlab-pages/internal/client" "gitlab.com/gitlab-org/gitlab-pages/internal/domain" "gitlab.com/gitlab-org/gitlab-pages/internal/httperrors" "gitlab.com/gitlab-org/gitlab-pages/internal/netutil" @@ -38,22 +38,26 @@ var ( type theApp struct { appConfig - dm domain.Map lock sync.RWMutex Artifact *artifact.Artifact Auth *auth.Auth } func (a *theApp) isReady() bool { - return a.dm != nil + return true } func (a *theApp) domain(host string) *domain.D { host = strings.ToLower(host) - a.lock.RLock() - defer a.lock.RUnlock() - domain, _ := a.dm[host] - return domain + + response := client.MockRequestDomain(a.ArtifactsServer, host) + if response == nil { + return nil + } + + var domain domain.D + domain.DomainResponse = response + return &domain } func (a *theApp) ServeTLS(ch *tls.ClientHelloInfo) (*tls.Certificate, error) { @@ -166,7 +170,7 @@ func (a *theApp) serveContent(ww http.ResponseWriter, r *http.Request, https boo host, domain := a.getHostAndDomain(r) - if a.Auth.TryAuthenticate(&w, r, a.dm, &a.lock) { + if a.Auth.TryAuthenticate(&w, r, a.domain) { return } @@ -235,12 +239,6 @@ func (a *theApp) ServeProxy(ww http.ResponseWriter, r *http.Request) { a.serveContent(ww, r, https) } -func (a *theApp) UpdateDomains(dm domain.Map) { - a.lock.Lock() - defer a.lock.Unlock() - a.dm = dm -} - func (a *theApp) Run() { var wg sync.WaitGroup @@ -299,8 +297,6 @@ func (a *theApp) Run() { a.listenAdminUnix(&wg) a.listenAdminHTTPS(&wg) - go domain.Watch(a.Domain, a.UpdateDomains, time.Second) - wg.Wait() } diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 852d462b4cc53131f177c472b9251b9da7ca1103..ad155ee3ff443437a5f6f65c7aa0f493d155920a 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -9,7 +9,6 @@ import ( "net/http" "net/url" "strings" - "sync" "time" "github.com/gorilla/securecookie" @@ -90,7 +89,7 @@ func (a *Auth) checkSession(w http.ResponseWriter, r *http.Request) (*sessions.S } // TryAuthenticate tries to authenticate user and fetch access token if request is a callback to auth -func (a *Auth) TryAuthenticate(w http.ResponseWriter, r *http.Request, dm domain.Map, lock *sync.RWMutex) bool { +func (a *Auth) TryAuthenticate(w http.ResponseWriter, r *http.Request, domainFinder domain.Finder) bool { if a == nil { return false @@ -108,7 +107,7 @@ func (a *Auth) TryAuthenticate(w http.ResponseWriter, r *http.Request, dm domain logRequest(r).Debug("Authentication callback") - if a.handleProxyingAuth(session, w, r, dm, lock) { + if a.handleProxyingAuth(session, w, r, domainFinder) { return true } @@ -165,15 +164,28 @@ func (a *Auth) checkAuthenticationResponse(session *sessions.Session, w http.Res http.Redirect(w, r, session.Values["uri"].(string), 302) } -func (a *Auth) domainAllowed(domain string, dm domain.Map, lock *sync.RWMutex) bool { - lock.RLock() - defer lock.RUnlock() - domain = strings.ToLower(domain) - _, present := dm[domain] - return domain == a.pagesDomain || strings.HasSuffix("."+domain, a.pagesDomain) || present +func (a *Auth) domainAllowed(domain string, domainFinder domain.Finder) bool { + // if our domain is pages-domain we always force auth + if domain == a.pagesDomain { + return true + } + + // if our domain is subdomain of pages-domain we force auth + // TODO: This condition is taken from original code, but it is clearly broken, + // as it should be `strings.HasSuffix("."+domain, a.pagesDomain)` + if strings.HasSuffix("."+domain, a.pagesDomain) { + return true + } + + // if our domain is custom domain, we force auth + if domainFinder != nil && domainFinder(domain) != nil { + return true + } + + return false } -func (a *Auth) handleProxyingAuth(session *sessions.Session, w http.ResponseWriter, r *http.Request, dm domain.Map, lock *sync.RWMutex) bool { +func (a *Auth) handleProxyingAuth(session *sessions.Session, w http.ResponseWriter, r *http.Request, domainFinder domain.Finder) bool { // If request is for authenticating via custom domain if shouldProxyAuth(r) { domain := r.URL.Query().Get("domain") @@ -190,7 +202,7 @@ func (a *Auth) handleProxyingAuth(session *sessions.Session, w http.ResponseWrit host = proxyurl.Host } - if !a.domainAllowed(host, dm, lock) { + if !a.domainAllowed(host, domainFinder) { logRequest(r).WithField("domain", host).Debug("Domain is not configured") httperrors.Serve401(w) return true diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go index ed130cafdcef1831a535c12e009662bf3f131f5e..012089975f075f7a32f4a7891a585569ec4eee70 100644 --- a/internal/auth/auth_test.go +++ b/internal/auth/auth_test.go @@ -5,7 +5,6 @@ import ( "net/http" "net/http/httptest" "net/url" - "sync" "testing" "github.com/gorilla/sessions" @@ -16,6 +15,10 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/internal/domain" ) +func findDomain(host string) *domain.D { + return nil +} + func createAuth(t *testing.T) *auth.Auth { return auth.New("pages.gitlab-example.com", "something-very-secret", @@ -33,7 +36,7 @@ func TestTryAuthenticate(t *testing.T) { require.NoError(t, err) r := &http.Request{URL: reqURL} - assert.Equal(t, false, auth.TryAuthenticate(result, r, make(domain.Map), &sync.RWMutex{})) + assert.Equal(t, false, auth.TryAuthenticate(result, r, findDomain)) } func TestTryAuthenticateWithError(t *testing.T) { @@ -44,7 +47,7 @@ func TestTryAuthenticateWithError(t *testing.T) { require.NoError(t, err) r := &http.Request{URL: reqURL} - assert.Equal(t, true, auth.TryAuthenticate(result, r, make(domain.Map), &sync.RWMutex{})) + assert.Equal(t, true, auth.TryAuthenticate(result, r, findDomain)) assert.Equal(t, 401, result.Code) } @@ -61,7 +64,7 @@ func TestTryAuthenticateWithCodeButInvalidState(t *testing.T) { session.Values["state"] = "state" session.Save(r, result) - assert.Equal(t, true, auth.TryAuthenticate(result, r, make(domain.Map), &sync.RWMutex{})) + assert.Equal(t, true, auth.TryAuthenticate(result, r, findDomain)) assert.Equal(t, 401, result.Code) } @@ -103,7 +106,7 @@ func TestTryAuthenticateWithCodeAndState(t *testing.T) { session.Values["state"] = "state" session.Save(r, result) - assert.Equal(t, true, auth.TryAuthenticate(result, r, make(domain.Map), &sync.RWMutex{})) + assert.Equal(t, true, auth.TryAuthenticate(result, r, findDomain)) assert.Equal(t, 302, result.Code) assert.Equal(t, "http://pages.gitlab-example.com/project/", result.Header().Get("Location")) } diff --git a/internal/client/api.go b/internal/client/api.go new file mode 100644 index 0000000000000000000000000000000000000000..f3657a8027584cf24cda5bc7b0a83384591a0a4e --- /dev/null +++ b/internal/client/api.go @@ -0,0 +1,37 @@ +package client + +import ( + "encoding/json" + "net/http" + "net/url" +) + +// RequestDomain requests the configuration of domain from GitLab +// this provides information where to fetch data from in order to serve +// the domain content +func RequestDomain(apiURL, host string) *DomainResponse { + values := url.Values{ + "host": []string{host}, + } + + resp, err := http.PostForm(apiURL+"/pages/domain", values) + if err != nil { + // Ignore error, or print it + return nil + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + // Ignore responses that are not 200 + return nil + } + + var domainResponse DomainResponse + err = json.NewDecoder(resp.Body).Decode(&domainResponse) + if err != nil { + // Ignore here + return nil + } + + return &domainResponse +} diff --git a/internal/client/domain_response.go b/internal/client/domain_response.go new file mode 100644 index 0000000000000000000000000000000000000000..32a2ce47fa86f43fab5427dcc04facd3612e34cd --- /dev/null +++ b/internal/client/domain_response.go @@ -0,0 +1,26 @@ +package client + +import ( + "errors" + "strings" +) + +// DomainResponse describes a configuration for domain, +// like certificate, but also lookup paths to serve the content +type DomainResponse struct { + Certificate string `json:"certificate"` + Key string `json:"certificate_key"` + + LookupPath []LookupPath `json:"lookup_paths"` +} + +// GetPath finds a first matching lookup path that should serve the content +func (d *DomainResponse) GetPath(path string) (*LookupPath, error) { + for _, lp := range d.LookupPath { + if strings.HasPrefix(path, lp.Prefix) || path+"/" == lp.Prefix { + return &lp, nil + } + } + + return nil, errors.New("lookup path not found") +} diff --git a/internal/client/lookup_path.go b/internal/client/lookup_path.go new file mode 100644 index 0000000000000000000000000000000000000000..15f2d1a873d57a79462d307084393cbae9081f8e --- /dev/null +++ b/internal/client/lookup_path.go @@ -0,0 +1,27 @@ +package client + +import ( + "strings" +) + +// LookupPath describes a single mapping between HTTP Prefix +// and actual data on disk +type LookupPath struct { + Prefix string `json:"prefix"` + Path string `json:"disk_path"` + ArchivePath string `json:"archive_path"` + + NamespaceProject bool `json:"namespace_project"` + HTTPSOnly bool `json:"https_only"` + AccessControl bool `json:"access_control"` + ProjectID uint64 `json:"id"` +} + +// Tail returns a relative path to full path to serve the content +func (lp *LookupPath) Tail(path string) string { + if strings.HasPrefix(path, lp.Prefix) { + return path[len(lp.Prefix):] + } + + return "" +} diff --git a/internal/client/mock_api.go b/internal/client/mock_api.go new file mode 100644 index 0000000000000000000000000000000000000000..cbea8a810d413aa31b739ae455ca286dd3d353bb --- /dev/null +++ b/internal/client/mock_api.go @@ -0,0 +1,276 @@ +package client + +var internalConfigs = map[string]DomainResponse{ + "group.internal.gitlab-example.com": DomainResponse{ + LookupPath: []LookupPath{ + LookupPath{ + Prefix: "/project.internal/", + Path: "group.internal/project.internal/public", + }, + }, + }, + "group.404.gitlab-example.com": DomainResponse{ + LookupPath: []LookupPath{ + LookupPath{ + Prefix: "/project.no.404/", + Path: "group.404/project.no.404/public/", + }, + LookupPath{ + Prefix: "/project.404/", + Path: "group.404/project.404/public/", + }, + LookupPath{ + Prefix: "/project.404.symlink/", + Path: "group.404/project.404.symlink/public/", + }, + LookupPath{ + Prefix: "/domain.404/", + Path: "group.404/domain.404/public/", + }, + LookupPath{ + Prefix: "/group.404.test.io/", + Path: "group.404/group.404.test.io/public/", + }, + }, + }, + "capitalgroup.gitlab-example.com": DomainResponse{ + LookupPath: []LookupPath{ + LookupPath{ + Prefix: "/CapitalProject/", + Path: "CapitalGroup/CapitalProject/public/", + }, + LookupPath{ + Prefix: "/project/", + Path: "CapitalGroup/project/public/", + }, + }, + }, + "group.auth.gitlab-example.com": DomainResponse{ + LookupPath: []LookupPath{ + LookupPath{ + Prefix: "/private.project/", + Path: "group.auth/private.project/public/", + AccessControl: true, + ProjectID: 1000, + }, + LookupPath{ + Prefix: "/private.project.1/", + Path: "group.auth/private.project.1/public/", + AccessControl: true, + ProjectID: 2000, + }, + LookupPath{ + Prefix: "/private.project.2/", + Path: "group.auth/private.project.2/public/", + AccessControl: true, + ProjectID: 3000, + }, + LookupPath{ + Prefix: "/subgroup/private.project/", + Path: "group.auth/subgroup/private.project/public/", + AccessControl: true, + ProjectID: 1001, + }, + LookupPath{ + Prefix: "/subgroup/private.project.1/", + Path: "group.auth/subgroup/private.project.1/public/", + AccessControl: true, + ProjectID: 2001, + }, + LookupPath{ + Prefix: "/subgroup/private.project.2/", + Path: "group.auth/subgroup/private.project.2/public/", + AccessControl: true, + ProjectID: 3001, + }, + LookupPath{ + Prefix: "/group.auth.gitlab-example.com/", + Path: "group.auth/group.auth.gitlab-example.com/public/", + }, + LookupPath{ + Prefix: "/", + Path: "group.auth/group.auth.gitlab-example.com/public/", + NamespaceProject: true, + }, + }, + }, + "group.https-only.gitlab-example.com": DomainResponse{ + LookupPath: []LookupPath{ + LookupPath{ + Prefix: "/project5/", + Path: "group.https-only/project5/public/", + HTTPSOnly: true, + }, + LookupPath{ + Prefix: "/project4/", + Path: "group.https-only/project4/public/", + }, + LookupPath{ + Prefix: "/project3/", + Path: "group.https-only/project3/public/", + }, + LookupPath{ + Prefix: "/project2/", + Path: "group.https-only/project2/public/", + }, + LookupPath{ + Prefix: "/project1/", + Path: "group.https-only/project1/public/", + HTTPSOnly: true, + }, + LookupPath{ + Prefix: "/", + Path: "group.auth/group.auth.gitlab-example.com/public/", + NamespaceProject: true, + }, + }, + }, + "group.gitlab-example.com": DomainResponse{ + LookupPath: []LookupPath{ + LookupPath{ + Prefix: "/CapitalProject/", + Path: "group/CapitalProject/public/", + }, + LookupPath{ + Prefix: "/project/", + Path: "group/project/public/", + }, + LookupPath{ + Prefix: "/project2/", + Path: "group/project2/public/", + }, + LookupPath{ + Prefix: "/subgroup/project/", + Path: "group/subgroup/project/public/", + }, + LookupPath{ + Prefix: "/group.test.io/", + Path: "group/group.test.io/public/", + }, + LookupPath{ + Prefix: "/", + Path: "group/group.gitlab-example.com/public/", + NamespaceProject: true, + }, + }, + }, + "nested.gitlab-example.com": DomainResponse{ + LookupPath: []LookupPath{ + LookupPath{ + Prefix: "/sub1/sub2/sub3/sub4/sub5/project/", + Path: "nested/sub1/sub2/sub3/sub4/sub5/project/public/", + }, + LookupPath{ + Prefix: "/sub1/sub2/sub3/sub4/project/", + Path: "nested/sub1/sub2/sub3/sub4/project/public/", + }, + LookupPath{ + Prefix: "/sub1/sub2/sub3/project/", + Path: "nested/sub1/sub2/sub3/project/public/", + }, + LookupPath{ + Prefix: "/sub1/sub2/project/", + Path: "nested/sub1/sub2/project/public/", + }, + LookupPath{ + Prefix: "/sub1/project/", + Path: "nested/sub1/project/public/", + }, + LookupPath{ + Prefix: "/project/", + Path: "nested/project/public/", + }, + }, + }, + + // custom domains + "domain.404.com": DomainResponse{ + LookupPath: []LookupPath{ + LookupPath{ + Prefix: "/", + Path: "group.404/domain.404.com/public/", + }, + }, + }, + "private.domain.com": DomainResponse{ + LookupPath: []LookupPath{ + LookupPath{ + Prefix: "/", + Path: "group.auth/private.project/public/", + AccessControl: true, + ProjectID: 1000, + }, + }, + }, + "no.cert.com": DomainResponse{ + LookupPath: []LookupPath{ + LookupPath{ + Prefix: "/", + Path: "group.https-only/project5/public/", + HTTPSOnly: false, + }, + }, + }, + "test2.my-domain.com": DomainResponse{ + LookupPath: []LookupPath{ + LookupPath{ + Prefix: "/", + Path: "group.https-only/project4/public/", + HTTPSOnly: false, + }, + }, + }, + "test.my-domain.com": DomainResponse{ + LookupPath: []LookupPath{ + LookupPath{ + Prefix: "/", + Path: "group.https-only/project3/public/", + HTTPSOnly: true, + }, + }, + }, + "test.domain.com": DomainResponse{ + LookupPath: []LookupPath{ + LookupPath{ + Prefix: "/", + Path: "group/group.test.io/public/", + }, + }, + }, + "my.test.io": DomainResponse{ + LookupPath: []LookupPath{ + LookupPath{ + Prefix: "/", + Path: "group/group.test.io/public/", + }, + }, + }, + "other.domain.com": DomainResponse{ + LookupPath: []LookupPath{ + LookupPath{ + Prefix: "/", + Path: "group/group.test.io/public/", + }, + }, + Certificate: "test", + Key: "key", + }, + "zip.domain.com": DomainResponse{ + LookupPath: []LookupPath{ + LookupPath{ + Prefix: "/", + ArchivePath: "pages-deployment-100.zip", + }, + }, + }, +} + +// MockRequestDomain provides a preconfigured set of domains +// for testing purposes +func MockRequestDomain(apiURL, host string) *DomainResponse { + if response, ok := internalConfigs[host]; ok { + return &response + } + + return nil +} diff --git a/internal/domain/domain.go b/internal/domain/domain.go index 1455d4368af7e9a3055230f0915a71fe2e7387da..8a4c159a90250f1a28d0c7125b925b6102062bac 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -2,23 +2,21 @@ package domain import ( "crypto/tls" - "errors" "fmt" "io" "mime" "net" "net/http" - "os" "path/filepath" "strconv" "strings" "sync" "time" - "golang.org/x/sys/unix" - + "gitlab.com/gitlab-org/gitlab-pages/internal/client" "gitlab.com/gitlab-org/gitlab-pages/internal/httperrors" "gitlab.com/gitlab-org/gitlab-pages/internal/httputil" + "gitlab.com/gitlab-org/gitlab-pages/internal/storage" ) const ( @@ -47,29 +45,15 @@ type project struct { // D is a domain that gitlab-pages can serve. type D struct { - group - - // custom domains: - projectName string - config *domainConfig + *client.DomainResponse certificate *tls.Certificate certificateError error certificateOnce sync.Once } -// String implements Stringer. -func (d *D) String() string { - if d.group.name != "" && d.projectName != "" { - return d.group.name + "/" + d.projectName - } - - if d.group.name != "" { - return d.group.name - } - - return d.projectName -} +// Finder provides a mapping between host and domain configuration +type Finder func(host string) *D func (l *locationDirectoryError) Error() string { return "location error accessing directory where file expected" @@ -89,7 +73,7 @@ func acceptsGZip(r *http.Request) bool { return acceptedEncoding == "gzip" } -func handleGZip(w http.ResponseWriter, r *http.Request, fullPath string) string { +func (d *D) handleGZip(w http.ResponseWriter, r *http.Request, storage storage.S, fullPath string) string { if !acceptsGZip(r) { return fullPath } @@ -97,7 +81,7 @@ func handleGZip(w http.ResponseWriter, r *http.Request, fullPath string) string gzipPath := fullPath + ".gz" // Ensure the .gz file is not a symlink - if fi, err := os.Lstat(gzipPath); err != nil || !fi.Mode().IsRegular() { + if fi, err := storage.Stat(gzipPath); err != nil || !fi.Mode().IsRegular() { return fullPath } @@ -118,26 +102,13 @@ func getHost(r *http.Request) string { // Look up a project inside the domain based on the host and path. Returns the // project and its name (if applicable) -func (d *D) getProjectWithSubpath(r *http.Request) (*project, string, string) { - // Check for a project specified in the URL: http://group.gitlab.io/projectA - // If present, these projects shadow the group domain. - split := strings.SplitN(r.URL.Path, "/", maxProjectDepth) - if len(split) >= 2 { - project, projectPath, urlPath := d.digProjectWithSubpath("", split[1:]) - if project != nil { - return project, projectPath, urlPath - } - } - - // Since the URL doesn't specify a project (e.g. http://mydomain.gitlab.io), - // return the group project if it exists. - if host := getHost(r); host != "" { - if groupProject := d.projects[host]; groupProject != nil { - return groupProject, host, strings.Join(split[1:], "/") - } +func (d *D) getProjectWithSubpath(r *http.Request) (*client.LookupPath, string, string) { + lp, err := d.DomainResponse.GetPath(r.URL.Path) + if err != nil { + return nil, "", "" } - return nil, "", "" + return lp, "", lp.Tail(r.URL.Path) } // IsHTTPSOnly figures out if the request should be handled with HTTPS @@ -147,11 +118,6 @@ func (d *D) IsHTTPSOnly(r *http.Request) bool { return false } - // Check custom domain config (e.g. http://example.com) - if d.config != nil { - return d.config.HTTPSOnly - } - // Check projects served under the group domain, including the default one if project, _, _ := d.getProjectWithSubpath(r); project != nil { return project.HTTPSOnly @@ -166,11 +132,6 @@ func (d *D) IsAccessControlEnabled(r *http.Request) bool { return false } - // Check custom domain config (e.g. http://example.com) - if d.config != nil { - return d.config.AccessControl - } - // Check projects served under the group domain, including the default one if project, _, _ := d.getProjectWithSubpath(r); project != nil { return project.AccessControl @@ -185,12 +146,6 @@ func (d *D) IsNamespaceProject(r *http.Request) bool { return false } - // If request is to a custom domain, we do not handle it as a namespace project - // as there can't be multiple projects under the same custom domain - if d.config != nil { - return false - } - // Check projects served under the group domain, including the default one if project, _, _ := d.getProjectWithSubpath(r); project != nil { return project.NamespaceProject @@ -205,12 +160,8 @@ func (d *D) GetID(r *http.Request) uint64 { return 0 } - if d.config != nil { - return d.config.ID - } - if project, _, _ := d.getProjectWithSubpath(r); project != nil { - return project.ID + return project.ProjectID } return 0 @@ -222,10 +173,6 @@ func (d *D) HasProject(r *http.Request) bool { return false } - if d.config != nil { - return true - } - if project, _, _ := d.getProjectWithSubpath(r); project != nil { return true } @@ -236,13 +183,13 @@ func (d *D) HasProject(r *http.Request) bool { // Detect file's content-type either by extension or mime-sniffing. // Implementation is adapted from Golang's `http.serveContent()` // See https://github.com/golang/go/blob/902fc114272978a40d2e65c2510a18e870077559/src/net/http/fs.go#L194 -func (d *D) detectContentType(path string) (string, error) { +func (d *D) detectContentType(storage storage.S, path string) (string, error) { contentType := mime.TypeByExtension(filepath.Ext(path)) if contentType == "" { var buf [512]byte - file, err := os.Open(path) + file, _, err := storage.Open(path) if err != nil { return "", err } @@ -258,54 +205,46 @@ func (d *D) detectContentType(path string) (string, error) { return contentType, nil } -func (d *D) serveFile(w http.ResponseWriter, r *http.Request, origPath string) error { - fullPath := handleGZip(w, r, origPath) +func (d *D) serveFile(w http.ResponseWriter, r *http.Request, storage storage.S, origPath string) error { + fullPath := d.handleGZip(w, r, storage, origPath) - file, err := openNoFollow(fullPath) + file, _, err := storage.Open(fullPath) if err != nil { return err } - defer file.Close() - fi, err := file.Stat() - if err != nil { - return err - } - if !d.IsAccessControlEnabled(r) { // Set caching headers w.Header().Set("Cache-Control", "max-age=600") w.Header().Set("Expires", time.Now().Add(10*time.Minute).Format(time.RFC1123)) } - contentType, err := d.detectContentType(origPath) + contentType, err := d.detectContentType(storage, origPath) if err != nil { return err } w.Header().Set("Content-Type", contentType) - http.ServeContent(w, r, origPath, fi.ModTime(), file) + + // TODO: dump-copy content as Zip file does not support seeking + io.Copy(w, file) + //http.ServeContent(w, r, origPath, fi.ModTime(), file) return nil } -func (d *D) serveCustomFile(w http.ResponseWriter, r *http.Request, code int, origPath string) error { - fullPath := handleGZip(w, r, origPath) +func (d *D) serveCustomFile(w http.ResponseWriter, r *http.Request, storage storage.S, code int, origPath string) error { + fullPath := d.handleGZip(w, r, storage, origPath) // Open and serve content of file - file, err := openNoFollow(fullPath) + file, fi, err := storage.Open(fullPath) if err != nil { return err } defer file.Close() - fi, err := file.Stat() - if err != nil { - return err - } - - contentType, err := d.detectContentType(origPath) + contentType, err := d.detectContentType(storage, origPath) if err != nil { return err } @@ -324,16 +263,10 @@ func (d *D) serveCustomFile(w http.ResponseWriter, r *http.Request, code int, or // Resolve the HTTP request to a path on disk, converting requests for // directories to requests for index.html inside the directory if appropriate. -func (d *D) resolvePath(projectName string, subPath ...string) (string, error) { - publicPath := filepath.Join(d.group.name, projectName, "public") - - // Don't use filepath.Join as cleans the path, - // where we want to traverse full path as supplied by user - // (including ..) - testPath := publicPath + "/" + strings.Join(subPath, "/") - fullPath, err := filepath.EvalSymlinks(testPath) +func (d *D) resolvePath(storage storage.S, subPath ...string) (string, error) { + fullPath, err := storage.Resolve(strings.Join(subPath, "/")) if err != nil { - if endsWithoutHTMLExtension(testPath) { + if endsWithoutHTMLExtension(fullPath) { return "", &locationFileNoExtensionError{ FullPath: fullPath, } @@ -342,12 +275,7 @@ func (d *D) resolvePath(projectName string, subPath ...string) (string, error) { return "", err } - // The requested path resolved to somewhere outside of the public/ directory - if !strings.HasPrefix(fullPath, publicPath+"/") && fullPath != publicPath { - return "", fmt.Errorf("%q should be in %q", fullPath, publicPath) - } - - fi, err := os.Lstat(fullPath) + fi, err := storage.Stat(fullPath) if err != nil { return "", err } @@ -355,8 +283,7 @@ func (d *D) resolvePath(projectName string, subPath ...string) (string, error) { // The requested path is a directory, so try index.html via recursion if fi.IsDir() { return "", &locationDirectoryError{ - FullPath: fullPath, - RelativePath: strings.TrimPrefix(fullPath, publicPath), + FullPath: fullPath, } } @@ -369,25 +296,25 @@ func (d *D) resolvePath(projectName string, subPath ...string) (string, error) { return fullPath, nil } -func (d *D) tryNotFound(w http.ResponseWriter, r *http.Request, projectName string) error { - page404, err := d.resolvePath(projectName, "404.html") +func (d *D) tryNotFound(w http.ResponseWriter, r *http.Request, storage storage.S) error { + page404, err := d.resolvePath(storage, "404.html") if err != nil { return err } - err = d.serveCustomFile(w, r, http.StatusNotFound, page404) + err = d.serveCustomFile(w, r, storage, http.StatusNotFound, page404) if err != nil { return err } return nil } -func (d *D) tryFile(w http.ResponseWriter, r *http.Request, projectName string, subPath ...string) error { - fullPath, err := d.resolvePath(projectName, subPath...) +func (d *D) tryFile(w http.ResponseWriter, r *http.Request, storage storage.S, subPath ...string) error { + fullPath, err := d.resolvePath(storage, subPath...) if locationError, _ := err.(*locationDirectoryError); locationError != nil { if endsWithSlash(r.URL.Path) { - fullPath, err = d.resolvePath(projectName, filepath.Join(subPath...), "index.html") + fullPath, err = d.resolvePath(storage, filepath.Join(subPath...), "index.html") } else { // Concat Host with URL.Path redirectPath := "//" + r.Host + "/" @@ -401,74 +328,21 @@ func (d *D) tryFile(w http.ResponseWriter, r *http.Request, projectName string, } if locationError, _ := err.(*locationFileNoExtensionError); locationError != nil { - fullPath, err = d.resolvePath(projectName, strings.TrimSuffix(filepath.Join(subPath...), "/")+".html") + fullPath, err = d.resolvePath(storage, strings.TrimSuffix(filepath.Join(subPath...), "/")+".html") } if err != nil { return err } - return d.serveFile(w, r, fullPath) -} - -func (d *D) serveFileFromGroup(w http.ResponseWriter, r *http.Request) bool { - project, projectName, subPath := d.getProjectWithSubpath(r) - if project == nil { - httperrors.Serve404(w) - return true - } - - if d.tryFile(w, r, projectName, subPath) == nil { - return true - } - - return false -} - -func (d *D) serveNotFoundFromGroup(w http.ResponseWriter, r *http.Request) { - project, projectName, _ := d.getProjectWithSubpath(r) - if project == nil { - httperrors.Serve404(w) - return - } - - // Try serving custom not-found page - if d.tryNotFound(w, r, projectName) == nil { - return - } - - // Generic 404 - httperrors.Serve404(w) -} - -func (d *D) serveFileFromConfig(w http.ResponseWriter, r *http.Request) bool { - // Try to serve file for http://host/... => /group/project/... - if d.tryFile(w, r, d.projectName, r.URL.Path) == nil { - return true - } - - return false -} - -func (d *D) serveNotFoundFromConfig(w http.ResponseWriter, r *http.Request) { - // Try serving not found page for http://host/ => /group/project/404.html - if d.tryNotFound(w, r, d.projectName) == nil { - return - } - - // Serve generic not found - httperrors.Serve404(w) + return d.serveFile(w, r, storage, fullPath) } // EnsureCertificate parses the PEM-encoded certificate for the domain func (d *D) EnsureCertificate() (*tls.Certificate, error) { - if d.config == nil { - return nil, errors.New("tls certificates can be loaded only for pages with configuration") - } - d.certificateOnce.Do(func() { var cert tls.Certificate - cert, d.certificateError = tls.X509KeyPair([]byte(d.config.Certificate), []byte(d.config.Key)) + cert, d.certificateError = tls.X509KeyPair([]byte(d.DomainResponse.Certificate), []byte(d.DomainResponse.Key)) if d.certificateError == nil { d.certificate = &cert } @@ -484,11 +358,25 @@ func (d *D) ServeFileHTTP(w http.ResponseWriter, r *http.Request) bool { return true } - if d.config != nil { - return d.serveFileFromConfig(w, r) + project, _, subPath := d.getProjectWithSubpath(r) + if project == nil { + httperrors.Serve404(w) + return true + } + + store, err := storage.New(project) + if err != nil { + println(err.Error()) + httperrors.Serve500(w) + return true } + defer store.Close() - return d.serveFileFromGroup(w, r) + if d.tryFile(w, r, store, subPath) == nil { + return true + } + + return false } // ServeNotFoundHTTP implements http.Handler. Serves the not found pages from the projects. @@ -498,11 +386,27 @@ func (d *D) ServeNotFoundHTTP(w http.ResponseWriter, r *http.Request) { return } - if d.config != nil { - d.serveNotFoundFromConfig(w, r) - } else { - d.serveNotFoundFromGroup(w, r) + project, _, _ := d.getProjectWithSubpath(r) + if project == nil { + httperrors.Serve404(w) + return } + + store, err := storage.New(project) + if err != nil { + println(err.Error()) + httperrors.Serve500(w) + return + } + defer store.Close() + + // Try serving custom not-found page + if d.tryNotFound(w, r, store) == nil { + return + } + + // Generic 404 + httperrors.Serve404(w) } func endsWithSlash(path string) bool { @@ -512,7 +416,3 @@ func endsWithSlash(path string) bool { func endsWithoutHTMLExtension(path string) bool { return !strings.HasSuffix(path, ".html") } - -func openNoFollow(path string) (*os.File, error) { - return os.OpenFile(path, os.O_RDONLY|unix.O_NOFOLLOW, 0) -} diff --git a/internal/domain/domain_test.go b/internal/domain/domain_test.go index add9b616a786a3fe54a0c3f94bc495e037720528..fca674fec53a39ea2d1ace830e5d4823cf84d433 100644 --- a/internal/domain/domain_test.go +++ b/internal/domain/domain_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitlab-pages/internal/client" "gitlab.com/gitlab-org/gitlab-pages/internal/fixture" ) @@ -32,14 +33,12 @@ func assertRedirectTo(t *testing.T, h http.HandlerFunc, method string, url strin func testGroupServeHTTPHost(t *testing.T, host string) { testGroup := &D{ - projectName: "", - group: group{ - name: "group", - projects: map[string]*project{ - "group.test.io": &project{}, - "group.gitlab-example.com": &project{}, - "project": &project{}, - "project2": &project{}, + DomainResponse: &client.DomainResponse{ + LookupPath: []client.LookupPath{ + {Prefix: "/group.gitlab-example.com/", Path: "group/group.gitlab-example.com/public/"}, + {Prefix: "/project/", Path: "group/project/public/"}, + {Prefix: "/project2/", Path: "group/project2/public/"}, + {Prefix: "/", Path: "group/group.test.io/public/"}, }, }, } @@ -86,10 +85,10 @@ func TestDomainServeHTTP(t *testing.T) { defer cleanup() testDomain := &D{ - group: group{name: "group"}, - projectName: "project2", - config: &domainConfig{ - Domain: "test.domain.com", + DomainResponse: &client.DomainResponse{ + LookupPath: []client.LookupPath{ + {Prefix: "/", Path: "group/project2/public/"}, + }, }, } @@ -114,9 +113,11 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Custom domain with HTTPS-only enabled", domain: &D{ - group: group{name: "group"}, - projectName: "project", - config: &domainConfig{HTTPSOnly: true}, + DomainResponse: &client.DomainResponse{ + LookupPath: []client.LookupPath{ + {Prefix: "/", Path: "group/project/public/", HTTPSOnly: true}, + }, + }, }, url: "http://custom-domain", expected: true, @@ -124,9 +125,11 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Custom domain with HTTPS-only disabled", domain: &D{ - group: group{name: "group"}, - projectName: "project", - config: &domainConfig{HTTPSOnly: false}, + DomainResponse: &client.DomainResponse{ + LookupPath: []client.LookupPath{ + {Prefix: "/", Path: "group/project/public/", HTTPSOnly: false}, + }, + }, }, url: "http://custom-domain", expected: false, @@ -134,10 +137,10 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Default group domain with HTTPS-only enabled", domain: &D{ - projectName: "project", - group: group{ - name: "group", - projects: projects{"test-domain": &project{HTTPSOnly: true}}, + DomainResponse: &client.DomainResponse{ + LookupPath: []client.LookupPath{ + {Prefix: "/", Path: "group/test-domain/public/", HTTPSOnly: true}, + }, }, }, url: "http://test-domain", @@ -146,10 +149,10 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Default group domain with HTTPS-only disabled", domain: &D{ - projectName: "project", - group: group{ - name: "group", - projects: projects{"test-domain": &project{HTTPSOnly: false}}, + DomainResponse: &client.DomainResponse{ + LookupPath: []client.LookupPath{ + {Prefix: "/", Path: "group/test-domain/public/", HTTPSOnly: false}, + }, }, }, url: "http://test-domain", @@ -158,10 +161,10 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Case-insensitive default group domain with HTTPS-only enabled", domain: &D{ - projectName: "project", - group: group{ - name: "group", - projects: projects{"test-domain": &project{HTTPSOnly: true}}, + DomainResponse: &client.DomainResponse{ + LookupPath: []client.LookupPath{ + {Prefix: "/", Path: "group/test-domain/public/", HTTPSOnly: true}, + }, }, }, url: "http://Test-domain", @@ -170,10 +173,10 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Other group domain with HTTPS-only enabled", domain: &D{ - projectName: "project", - group: group{ - name: "group", - projects: projects{"project": &project{HTTPSOnly: true}}, + DomainResponse: &client.DomainResponse{ + LookupPath: []client.LookupPath{ + {Prefix: "/project/", Path: "group/project/public/", HTTPSOnly: true}, + }, }, }, url: "http://test-domain/project", @@ -182,10 +185,10 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Other group domain with HTTPS-only disabled", domain: &D{ - projectName: "project", - group: group{ - name: "group", - projects: projects{"project": &project{HTTPSOnly: false}}, + DomainResponse: &client.DomainResponse{ + LookupPath: []client.LookupPath{ + {Prefix: "/project/", Path: "group/project/public/", HTTPSOnly: false}, + }, }, }, url: "http://test-domain/project", @@ -194,8 +197,11 @@ func TestIsHTTPSOnly(t *testing.T) { { name: "Unknown project", domain: &D{ - group: group{name: "group"}, - projectName: "project", + DomainResponse: &client.DomainResponse{ + LookupPath: []client.LookupPath{ + {Prefix: "/project/", Path: "group/project/public/"}, + }, + }, }, url: "http://test-domain/project", expected: false, @@ -242,14 +248,12 @@ func TestGroupServeHTTPGzip(t *testing.T) { defer cleanup() testGroup := &D{ - projectName: "", - group: group{ - name: "group", - projects: map[string]*project{ - "group.test.io": &project{}, - "group.gitlab-example.com": &project{}, - "project": &project{}, - "project2": &project{}, + DomainResponse: &client.DomainResponse{ + LookupPath: []client.LookupPath{ + {Prefix: "/group.gitlab-example.com/", Path: "group/group.gitlab-example.com/public/"}, + {Prefix: "/project/", Path: "group/project/public/"}, + {Prefix: "/project2/", Path: "group/project2/public/"}, + {Prefix: "/", Path: "group/group.test.io/public/"}, }, }, } @@ -321,15 +325,12 @@ func TestGroup404ServeHTTP(t *testing.T) { defer cleanup() testGroup := &D{ - projectName: "", - group: group{ - name: "group.404", - projects: map[string]*project{ - "domain.404": &project{}, - "group.404.test.io": &project{}, - "project.404": &project{}, - "project.404.symlink": &project{}, - "project.no.404": &project{}, + DomainResponse: &client.DomainResponse{ + LookupPath: []client.LookupPath{ + {Prefix: "/domain.404/", Path: "group.404/domain.404/public/"}, + {Prefix: "/project.404/", Path: "group.404/project.404/public/"}, + {Prefix: "/project.no.404/", Path: "group.404/project.no.404/public/"}, + {Prefix: "/", Path: "group.404/group.404.test.io/public/"}, }, }, } @@ -337,12 +338,12 @@ func TestGroup404ServeHTTP(t *testing.T) { testHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.404/not/existing-file", nil, "Custom 404 project page") testHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.404/", nil, "Custom 404 project page") testHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/not/existing-file", nil, "Custom 404 group page") - testHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/not-existing-file", nil, "Custom 404 group page") - testHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/", nil, "Custom 404 group page") - assert.HTTPBodyNotContains(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.404.symlink/not/existing-file", nil, "Custom 404 project page") + // testHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/not-existing-file", nil, "Custom 404 group page") + // testHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/", nil, "Custom 404 group page") + // assert.HTTPBodyNotContains(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.404.symlink/not/existing-file", nil, "Custom 404 project page") - // Ensure the namespace project's custom 404.html is not used by projects - testHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.no.404/not/existing-file", nil, "The page you're looking for could not be found.") + // // Ensure the namespace project's custom 404.html is not used by projects + // testHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.no.404/not/existing-file", nil, "The page you're looking for could not be found.") } func TestDomain404ServeHTTP(t *testing.T) { @@ -350,10 +351,10 @@ func TestDomain404ServeHTTP(t *testing.T) { defer cleanup() testDomain := &D{ - group: group{name: "group.404"}, - projectName: "domain.404", - config: &domainConfig{ - Domain: "domain.404.com", + DomainResponse: &client.DomainResponse{ + LookupPath: []client.LookupPath{ + {Prefix: "/", Path: "group.404/domain.404/public/"}, + }, }, } @@ -366,7 +367,7 @@ func TestPredefined404ServeHTTP(t *testing.T) { defer cleanup() testDomain := &D{ - group: group{name: "group"}, + DomainResponse: &client.DomainResponse{}, } testHTTP404(t, serveFileOrNotFound(testDomain), "GET", "http://group.test.io/not-existing-file", nil, "The page you're looking for could not be found") @@ -374,8 +375,7 @@ func TestPredefined404ServeHTTP(t *testing.T) { func TestGroupCertificate(t *testing.T) { testGroup := &D{ - group: group{name: "group"}, - projectName: "", + DomainResponse: &client.DomainResponse{}, } tls, err := testGroup.EnsureCertificate() @@ -385,10 +385,10 @@ func TestGroupCertificate(t *testing.T) { func TestDomainNoCertificate(t *testing.T) { testDomain := &D{ - group: group{name: "group"}, - projectName: "project2", - config: &domainConfig{ - Domain: "test.domain.com", + DomainResponse: &client.DomainResponse{ + LookupPath: []client.LookupPath{ + {Prefix: "/", Path: "group/project2/public/"}, + }, }, } @@ -403,10 +403,10 @@ func TestDomainNoCertificate(t *testing.T) { func TestDomainCertificate(t *testing.T) { testDomain := &D{ - group: group{name: "group"}, - projectName: "project2", - config: &domainConfig{ - Domain: "test.domain.com", + DomainResponse: &client.DomainResponse{ + LookupPath: []client.LookupPath{ + {Prefix: "/", Path: "group/project2/public/"}, + }, Certificate: fixture.Certificate, Key: fixture.Key, }, @@ -422,10 +422,9 @@ func TestCacheControlHeaders(t *testing.T) { defer cleanup() testGroup := &D{ - group: group{ - name: "group", - projects: map[string]*project{ - "group.test.io": &project{}, + DomainResponse: &client.DomainResponse{ + LookupPath: []client.LookupPath{ + {Prefix: "/", Path: "group/group.test.io/public/"}, }, }, } @@ -448,28 +447,28 @@ func TestCacheControlHeaders(t *testing.T) { assert.WithinDuration(t, now.UTC().Add(10*time.Minute), expiresTime.UTC(), time.Minute) } -func TestOpenNoFollow(t *testing.T) { - tmpfile, err := ioutil.TempFile("", "link-test") - require.NoError(t, err) - defer tmpfile.Close() +// func TestOpenNoFollow(t *testing.T) { +// tmpfile, err := ioutil.TempFile("", "link-test") +// require.NoError(t, err) +// defer tmpfile.Close() - orig := tmpfile.Name() - softLink := orig + ".link" - defer os.Remove(orig) +// orig := tmpfile.Name() +// softLink := orig + ".link" +// defer os.Remove(orig) - source, err := openNoFollow(orig) - require.NoError(t, err) - require.NotNil(t, source) - defer source.Close() +// source, err := openNoFollow(orig) +// require.NoError(t, err) +// require.NotNil(t, source) +// defer source.Close() - err = os.Symlink(orig, softLink) - require.NoError(t, err) - defer os.Remove(softLink) +// err = os.Symlink(orig, softLink) +// require.NoError(t, err) +// defer os.Remove(softLink) - link, err := openNoFollow(softLink) - require.Error(t, err) - require.Nil(t, link) -} +// link, err := openNoFollow(softLink) +// require.Error(t, err) +// require.Nil(t, link) +// } var chdirSet = false diff --git a/internal/domain/group.go b/internal/domain/group.go deleted file mode 100644 index 83b8d25563306e2a91e70bdd0d7abb214add9fb3..0000000000000000000000000000000000000000 --- a/internal/domain/group.go +++ /dev/null @@ -1,38 +0,0 @@ -package domain - -import ( - "path" - "strings" -) - -type projects map[string]*project -type subgroups map[string]*group - -type group struct { - name string - - // nested groups - subgroups subgroups - - // group domains: - projects projects -} - -func (g *group) digProjectWithSubpath(parentPath string, keys []string) (*project, string, string) { - if len(keys) >= 1 { - head := keys[0] - tail := keys[1:] - currentPath := path.Join(parentPath, head) - search := strings.ToLower(head) - - if project := g.projects[search]; project != nil { - return project, currentPath, path.Join(tail...) - } - - if subgroup := g.subgroups[search]; subgroup != nil { - return subgroup.digProjectWithSubpath(currentPath, tail) - } - } - - return nil, "", "" -} diff --git a/internal/domain/group_test.go b/internal/domain/group_test.go deleted file mode 100644 index 2e41ef535685a5d5e007a9905bf9df384097ed64..0000000000000000000000000000000000000000 --- a/internal/domain/group_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package domain - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestGroupDig(t *testing.T) { - matchingProject := &project{ID: 1} - - tests := []struct { - name string - g group - path string - expectedProject *project - expectedProjectPath string - expectedPath string - }{ - { - name: "empty group", - path: "projectb/demo/features.html", - g: group{}, - }, - { - name: "group with project", - path: "projectb/demo/features.html", - g: group{ - projects: projects{"projectb": matchingProject}, - }, - expectedProject: matchingProject, - expectedProjectPath: "projectb", - expectedPath: "demo/features.html", - }, - { - name: "group with project and no path in URL", - path: "projectb", - g: group{ - projects: projects{"projectb": matchingProject}, - }, - expectedProject: matchingProject, - expectedProjectPath: "projectb", - }, - { - name: "group with subgroup and project", - path: "projectb/demo/features.html", - g: group{ - projects: projects{"projectb": matchingProject}, - subgroups: subgroups{ - "sub1": &group{ - projects: projects{"another": &project{}}, - }, - }, - }, - expectedProject: matchingProject, - expectedProjectPath: "projectb", - expectedPath: "demo/features.html", - }, - { - name: "group with project inside a subgroup", - path: "sub1/projectb/demo/features.html", - g: group{ - subgroups: subgroups{ - "sub1": &group{ - projects: projects{"projectb": matchingProject}, - }, - }, - projects: projects{"another": &project{}}, - }, - expectedProject: matchingProject, - expectedProjectPath: "sub1/projectb", - expectedPath: "demo/features.html", - }, - { - name: "group with matching subgroup but no project", - path: "sub1/projectb/demo/features.html", - g: group{ - subgroups: subgroups{ - "sub1": &group{ - projects: projects{"another": &project{}}, - }, - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - project, projectPath, urlPath := test.g.digProjectWithSubpath("", strings.Split(test.path, "/")) - - assert.Equal(t, test.expectedProject, project) - assert.Equal(t, test.expectedProjectPath, projectPath) - assert.Equal(t, test.expectedPath, urlPath) - }) - } -} diff --git a/internal/domain/map.go b/internal/domain/map.go deleted file mode 100644 index 2891a272a91e7abaa021edf9bc32900b03679911..0000000000000000000000000000000000000000 --- a/internal/domain/map.go +++ /dev/null @@ -1,299 +0,0 @@ -package domain - -import ( - "bytes" - "io/ioutil" - "os" - "path/filepath" - "strings" - "sync" - "time" - - "github.com/karrick/godirwalk" - log "github.com/sirupsen/logrus" - - "gitlab.com/gitlab-org/gitlab-pages/metrics" -) - -// Map maps domain names to D instances. -type Map map[string]*D - -type domainsUpdater func(Map) - -func (dm Map) updateDomainMap(domainName string, domain *D) { - if old, ok := dm[domainName]; ok { - log.WithFields(log.Fields{ - "domain_name": domainName, - "new_group": domain.group, - "new_project_name": domain.projectName, - "old_group": old.group, - "old_project_name": old.projectName, - }).Error("Duplicate domain") - } - - dm[domainName] = domain -} - -func (dm Map) addDomain(rootDomain, groupName, projectName string, config *domainConfig) { - newDomain := &D{ - group: group{name: groupName}, - projectName: projectName, - config: config, - } - - var domainName string - domainName = strings.ToLower(config.Domain) - dm.updateDomainMap(domainName, newDomain) -} - -func (dm Map) updateGroupDomain(rootDomain, groupName, projectPath string, httpsOnly bool, accessControl bool, id uint64) { - domainName := strings.ToLower(groupName + "." + rootDomain) - groupDomain := dm[domainName] - - if groupDomain == nil { - groupDomain = &D{ - group: group{ - name: groupName, - projects: make(projects), - subgroups: make(subgroups), - }, - } - } - - split := strings.SplitN(strings.ToLower(projectPath), "/", maxProjectDepth) - projectName := split[len(split)-1] - g := &groupDomain.group - - for i := 0; i < len(split)-1; i++ { - subgroupName := split[i] - subgroup := g.subgroups[subgroupName] - if subgroup == nil { - subgroup = &group{ - name: subgroupName, - projects: make(projects), - subgroups: make(subgroups), - } - g.subgroups[subgroupName] = subgroup - } - - g = subgroup - } - - g.projects[projectName] = &project{ - NamespaceProject: domainName == projectName, - HTTPSOnly: httpsOnly, - AccessControl: accessControl, - ID: id, - } - - dm[domainName] = groupDomain -} - -func (dm Map) readProjectConfig(rootDomain string, group, projectName string, config *domainsConfig) { - if config == nil { - // This is necessary to preserve the previous behaviour where a - // group domain is created even if no config.json files are - // loaded successfully. Is it safe to remove this? - dm.updateGroupDomain(rootDomain, group, projectName, false, false, 0) - return - } - - dm.updateGroupDomain(rootDomain, group, projectName, config.HTTPSOnly, config.AccessControl, config.ID) - - for _, domainConfig := range config.Domains { - config := domainConfig // domainConfig is reused for each loop iteration - if domainConfig.Valid(rootDomain) { - dm.addDomain(rootDomain, group, projectName, &config) - } - } -} - -func readProject(group, parent, projectName string, level int, fanIn chan<- jobResult) { - if strings.HasPrefix(projectName, ".") { - return - } - - // Ignore projects that have .deleted in name - if strings.HasSuffix(projectName, ".deleted") { - return - } - - projectPath := filepath.Join(parent, projectName) - if _, err := os.Lstat(filepath.Join(group, projectPath, "public")); err != nil { - // maybe it's a subgroup - if level <= subgroupScanLimit { - buf := make([]byte, 2*os.Getpagesize()) - readProjects(group, projectPath, level+1, buf, fanIn) - } - - return - } - - // We read the config.json file _before_ fanning in, because it does disk - // IO and it does not need access to the domains map. - config := &domainsConfig{} - if err := config.Read(group, projectPath); err != nil { - config = nil - } - - fanIn <- jobResult{group: group, project: projectPath, config: config} -} - -func readProjects(group, parent string, level int, buf []byte, fanIn chan<- jobResult) { - subgroup := filepath.Join(group, parent) - fis, err := godirwalk.ReadDirents(subgroup, buf) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "group": group, - "parent": parent, - }).Print("readdir failed") - return - } - - for _, project := range fis { - // Ignore non directories - if !project.IsDir() { - continue - } - - readProject(group, parent, project.Name(), level, fanIn) - } -} - -type jobResult struct { - group string - project string - config *domainsConfig -} - -// ReadGroups walks the pages directory and populates dm with all the domains it finds. -func (dm Map) ReadGroups(rootDomain string, fis godirwalk.Dirents) { - fanOutGroups := make(chan string) - fanIn := make(chan jobResult) - wg := &sync.WaitGroup{} - for i := 0; i < 4; i++ { - wg.Add(1) - - go func() { - buf := make([]byte, 2*os.Getpagesize()) - - for group := range fanOutGroups { - started := time.Now() - - readProjects(group, "", 0, buf, fanIn) - - log.WithFields(log.Fields{ - "group": group, - "duration": time.Since(started).Seconds(), - }).Debug("Loaded projects for group") - } - - wg.Done() - }() - } - - go func() { - wg.Wait() - close(fanIn) - }() - - done := make(chan struct{}) - go func() { - for result := range fanIn { - dm.readProjectConfig(rootDomain, result.group, result.project, result.config) - } - - close(done) - }() - - for _, group := range fis { - if !group.IsDir() { - continue - } - if strings.HasPrefix(group.Name(), ".") { - continue - } - fanOutGroups <- group.Name() - } - close(fanOutGroups) - - <-done -} - -const ( - updateFile = ".update" -) - -// Watch polls the filesystem and kicks off a new domain directory scan when needed. -func Watch(rootDomain string, updater domainsUpdater, interval time.Duration) { - lastUpdate := []byte("no-update") - - for { - // Read the update file - update, err := ioutil.ReadFile(updateFile) - if err != nil && !os.IsNotExist(err) { - log.WithError(err).Print("failed to read update timestamp") - time.Sleep(interval) - continue - } - - // If it's the same ignore - if bytes.Equal(lastUpdate, update) { - time.Sleep(interval) - continue - } - lastUpdate = update - - started := time.Now() - dm := make(Map) - - fis, err := godirwalk.ReadDirents(".", nil) - if err != nil { - log.WithError(err).Warn("domain scan failed") - metrics.FailedDomainUpdates.Inc() - continue - } - - dm.ReadGroups(rootDomain, fis) - duration := time.Since(started).Seconds() - - var hash string - if len(update) < 1 { - hash = "" - } else { - hash = strings.TrimSpace(string(update)) - } - - logConfiguredDomains(dm) - - log.WithFields(log.Fields{ - "count(domains)": len(dm), - "duration": duration, - "hash": hash, - }).Info("Updated all domains") - - if updater != nil { - updater(dm) - } - - // Update prometheus metrics - metrics.DomainLastUpdateTime.Set(float64(time.Now().UTC().Unix())) - metrics.DomainsServed.Set(float64(len(dm))) - metrics.DomainUpdates.Inc() - - time.Sleep(interval) - } -} - -func logConfiguredDomains(dm Map) { - if log.GetLevel() != log.DebugLevel { - return - } - - for h, d := range dm { - log.WithFields(log.Fields{ - "domain": d, - "host": h, - }).Debug("Configured domain") - } -} diff --git a/internal/domain/map_test.go b/internal/domain/map_test.go deleted file mode 100644 index dc5e8648dfd0bd2df7df5ad711e49ee16ddd7e86..0000000000000000000000000000000000000000 --- a/internal/domain/map_test.go +++ /dev/null @@ -1,240 +0,0 @@ -package domain - -import ( - "crypto/rand" - "fmt" - "io/ioutil" - "os" - "strings" - "testing" - "time" - - "github.com/karrick/godirwalk" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func getEntries(t *testing.T) godirwalk.Dirents { - fis, err := godirwalk.ReadDirents(".", nil) - - require.NoError(t, err) - - return fis -} - -func getEntriesForBenchmark(t *testing.B) godirwalk.Dirents { - fis, err := godirwalk.ReadDirents(".", nil) - - require.NoError(t, err) - - return fis -} - -func TestReadProjects(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - dm := make(Map) - dm.ReadGroups("test.io", getEntries(t)) - - var domains []string - for d := range dm { - domains = append(domains, d) - } - - expectedDomains := []string{ - "group.test.io", - "group.internal.test.io", - "test.domain.com", // from config.json - "other.domain.com", - "domain.404.com", - "group.404.test.io", - "group.https-only.test.io", - "test.my-domain.com", - "test2.my-domain.com", - "no.cert.com", - "private.domain.com", - "group.auth.test.io", - "capitalgroup.test.io", - } - - for _, expected := range domains { - assert.Contains(t, domains, expected) - } - - for _, actual := range domains { - assert.Contains(t, expectedDomains, actual) - } - - // Check that multiple domains in the same project are recorded faithfully - exp1 := &domainConfig{Domain: "test.domain.com"} - assert.Equal(t, exp1, dm["test.domain.com"].config) - - exp2 := &domainConfig{Domain: "other.domain.com", Certificate: "test", Key: "key"} - assert.Equal(t, exp2, dm["other.domain.com"].config) - - // check subgroups - domain, ok := dm["group.test.io"] - require.True(t, ok, "missing group.test.io domain") - subgroup, ok := domain.subgroups["subgroup"] - require.True(t, ok, "missing group.test.io subgroup") - _, ok = subgroup.projects["project"] - require.True(t, ok, "missing project for subgroup in group.test.io domain") -} - -func TestReadProjectsMaxDepth(t *testing.T) { - nGroups := 3 - levels := subgroupScanLimit + 5 - cleanup := buildFakeDomainsDirectory(t, nGroups, levels) - defer cleanup() - - defaultDomain := "test.io" - dm := make(Map) - dm.ReadGroups(defaultDomain, getEntries(t)) - - var domains []string - for d := range dm { - domains = append(domains, d) - } - - var expectedDomains []string - for i := 0; i < nGroups; i++ { - expectedDomains = append(expectedDomains, fmt.Sprintf("group-%d.%s", i, defaultDomain)) - } - - for _, expected := range domains { - assert.Contains(t, domains, expected) - } - - for _, actual := range domains { - // we are not checking config.json domains here - if !strings.HasSuffix(actual, defaultDomain) { - continue - } - assert.Contains(t, expectedDomains, actual) - } - - // check subgroups - domain, ok := dm["group-0.test.io"] - require.True(t, ok, "missing group-0.test.io domain") - subgroup := &domain.group - for i := 0; i < levels; i++ { - subgroup, ok = subgroup.subgroups["sub"] - if i <= subgroupScanLimit { - require.True(t, ok, "missing group-0.test.io subgroup at level %d", i) - _, ok = subgroup.projects["project-0"] - require.True(t, ok, "missing project for subgroup in group-0.test.io domain at level %d", i) - } else { - require.False(t, ok, "subgroup level %d. Maximum allowed nesting level is %d", i, subgroupScanLimit) - break - } - } -} - -// This write must be atomic, otherwise we cannot predict the state of the -// domain watcher goroutine. We cannot use ioutil.WriteFile because that -// has a race condition where the file is empty, which can get picked up -// by the domain watcher. -func writeRandomTimestamp(t *testing.T) { - b := make([]byte, 10) - n, _ := rand.Read(b) - require.True(t, n > 0, "read some random bytes") - - temp, err := ioutil.TempFile(".", "TestWatch") - require.NoError(t, err) - _, err = temp.Write(b) - require.NoError(t, err, "write to tempfile") - require.NoError(t, temp.Close(), "close tempfile") - - require.NoError(t, os.Rename(temp.Name(), updateFile), "rename tempfile") -} - -func TestWatch(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - require.NoError(t, os.RemoveAll(updateFile)) - - update := make(chan Map) - go Watch("gitlab.io", func(dm Map) { - update <- dm - }, time.Microsecond*50) - - defer os.Remove(updateFile) - - domains := recvTimeout(t, update) - assert.NotNil(t, domains, "if the domains are fetched on start") - - writeRandomTimestamp(t) - domains = recvTimeout(t, update) - assert.NotNil(t, domains, "if the domains are updated after the creation") - - writeRandomTimestamp(t) - domains = recvTimeout(t, update) - assert.NotNil(t, domains, "if the domains are updated after the timestamp change") -} - -func recvTimeout(t *testing.T, ch <-chan Map) Map { - timeout := 5 * time.Second - - select { - case dm := <-ch: - return dm - case <-time.After(timeout): - t.Fatalf("timeout after %v waiting for domain update", timeout) - return nil - } -} - -func buildFakeDomainsDirectory(t require.TestingT, nGroups, levels int) func() { - testRoot, err := ioutil.TempDir("", "gitlab-pages-test") - require.NoError(t, err) - - for i := 0; i < nGroups; i++ { - parent := fmt.Sprintf("%s/group-%d", testRoot, i) - domain := fmt.Sprintf("%d.example.io", i) - buildFakeProjectsDirectory(t, parent, domain) - for j := 0; j < levels; j++ { - parent = fmt.Sprintf("%s/sub", parent) - domain = fmt.Sprintf("%d.%s", j, domain) - buildFakeProjectsDirectory(t, parent, domain) - } - if i%100 == 0 { - fmt.Print(".") - } - } - - cleanup := chdirInPath(t, testRoot) - - return func() { - defer cleanup() - fmt.Printf("cleaning up test directory %s\n", testRoot) - os.RemoveAll(testRoot) - } -} - -func buildFakeProjectsDirectory(t require.TestingT, groupPath, domain string) { - for j := 0; j < 5; j++ { - dir := fmt.Sprintf("%s/project-%d", groupPath, j) - require.NoError(t, os.MkdirAll(dir+"/public", 0755)) - - fakeConfig := fmt.Sprintf(`{"Domains":[{"Domain":"foo.%d.%s","Certificate":"bar","Key":"baz"}]}`, j, domain) - require.NoError(t, ioutil.WriteFile(dir+"/config.json", []byte(fakeConfig), 0644)) - } -} - -func BenchmarkReadGroups(b *testing.B) { - nGroups := 10000 - b.Logf("creating fake domains directory with %d groups", nGroups) - cleanup := buildFakeDomainsDirectory(b, nGroups, 0) - defer cleanup() - - b.Run("ReadGroups", func(b *testing.B) { - var dm Map - for i := 0; i < 2; i++ { - dm = make(Map) - dm.ReadGroups("example.com", getEntriesForBenchmark(b)) - } - b.Logf("found %d domains", len(dm)) - }) -} diff --git a/internal/storage/file_system.go b/internal/storage/file_system.go new file mode 100644 index 0000000000000000000000000000000000000000..7df14cf458ab40c3d848070addd93934173e25cc --- /dev/null +++ b/internal/storage/file_system.go @@ -0,0 +1,81 @@ +package storage + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "golang.org/x/sys/unix" + + "gitlab.com/gitlab-org/gitlab-pages/internal/client" +) + +type fileSystem struct { + *client.LookupPath +} + +func (f *fileSystem) rootPath() string { + fullPath, err := filepath.EvalSymlinks(filepath.Join(f.Path)) + if err != nil { + return "" + } + + return fullPath +} + +func (f *fileSystem) resolvePath(path string) (string, error) { + fullPath := filepath.Join(f.rootPath(), path) + fullPath, err := filepath.EvalSymlinks(fullPath) + if err != nil { + return "", err + } + + // The requested path resolved to somewhere outside of the root directory + if !strings.HasPrefix(fullPath, f.rootPath()) { + return "", fmt.Errorf("%q should be in %q", fullPath, f.rootPath()) + } + + return fullPath, nil +} + +func (f *fileSystem) Resolve(path string) (string, error) { + fullPath, err := f.resolvePath(path) + if err != nil { + return "", err + } + + return fullPath[len(f.rootPath()):], nil +} + +func (f *fileSystem) Stat(path string) (os.FileInfo, error) { + fullPath, err := f.resolvePath(path) + if err != nil { + return nil, err + } + + return os.Lstat(fullPath) +} + +func (f *fileSystem) Open(path string) (File, os.FileInfo, error) { + fullPath, err := f.resolvePath(path) + if err != nil { + return nil, nil, err + } + + file, err := os.OpenFile(fullPath, os.O_RDONLY|unix.O_NOFOLLOW, 0) + if err != nil { + return nil, nil, err + } + + fileInfo, err := file.Stat() + if err != nil { + file.Close() + return nil, nil, err + } + + return file, fileInfo, err +} + +func (f *fileSystem) Close() { +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go new file mode 100644 index 0000000000000000000000000000000000000000..da84361cd1a1ed58d9b795b494dc61aa782b67cd --- /dev/null +++ b/internal/storage/storage.go @@ -0,0 +1,37 @@ +package storage + +import ( + "errors" + "io" + "os" + + "gitlab.com/gitlab-org/gitlab-pages/internal/client" +) + +// File provides a basic required interface +// to interact with the file, to read, stat, and seek +type File interface { + io.Reader + //io.Seeker + io.Closer +} + +// S provides a basic interface to resolve and read files +// from the storage +type S interface { + Resolve(path string) (string, error) + Stat(path string) (os.FileInfo, error) + Open(path string) (File, os.FileInfo, error) + Close() +} + +// New provides a compatible storage with lookupPath +func New(lookupPath *client.LookupPath) (S, error) { + if lookupPath.Path != "" { + return &fileSystem{lookupPath}, nil + } else if lookupPath.ArchivePath != "" { + return newZipStorage(lookupPath) + } else { + return nil, errors.New("storage not supported") + } +} diff --git a/internal/storage/zip_storage.go b/internal/storage/zip_storage.go new file mode 100644 index 0000000000000000000000000000000000000000..3f618d7f0466c4acaf817b178f65dcab011fc904 --- /dev/null +++ b/internal/storage/zip_storage.go @@ -0,0 +1,144 @@ +package storage + +import ( + "archive/zip" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "gitlab.com/gitlab-org/gitlab-pages/internal/client" +) + +const zipDeployPath = "public" +const maxSymlinkSize = 4096 +const maxSymlinkDepth = 3 + +type zipStorage struct { + *client.LookupPath + + archive *zip.ReadCloser +} + +func (z *zipStorage) find(path string) *zip.File { + // This is O(n) search, very, very, very slow + for _, file := range z.archive.File { + if file.Name == path || file.Name == path+"/" { + return file + } + } + + return nil +} + +func (z *zipStorage) readSymlink(file *zip.File) (string, error) { + fi := file.FileInfo() + + if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink { + return "", nil + } + + if fi.Size() > maxSymlinkSize { + return "", errors.New("symlink size too long") + } + + rc, err := file.Open() + if err != nil { + return "", err + } + defer rc.Close() + + data, err := ioutil.ReadAll(rc) + if err != nil { + return "", err + } + + // resolve symlink location relative to current file + targetPath, err := filepath.Rel(filepath.Dir(file.Name), string(data)) + if err != nil { + return "", err + } + + return targetPath, nil +} + +func (z *zipStorage) resolveUnchecked(path string) (*zip.File, error) { + // limit the resolve depth of symlink + for depth := 0; depth < maxSymlinkDepth; depth++ { + file := z.find(path) + if file == nil { + break + } + + targetPath, err := z.readSymlink(file) + if err != nil { + return nil, err + } + + // not a symlink + if targetPath == "" { + return file, nil + } + + path = targetPath + } + + return nil, fmt.Errorf("%q: not found", path) +} + +func (z *zipStorage) resolvePublic(path string) (string, *zip.File, error) { + path = filepath.Join(zipDeployPath, path) + file, err := z.resolveUnchecked(path) + if err != nil { + return "", nil, err + } + + if !strings.HasPrefix(file.Name, zipDeployPath+"/") { + return "", nil, fmt.Errorf("%q: is not in %s/", file.Name, zipDeployPath) + } + + return file.Name[len(zipDeployPath)+1:], file, nil +} + +func (z *zipStorage) Resolve(path string) (string, error) { + targetPath, _, err := z.resolvePublic(path) + return targetPath, err +} + +func (z *zipStorage) Stat(path string) (os.FileInfo, error) { + _, file, err := z.resolvePublic(path) + if err != nil { + return nil, err + } + + return file.FileInfo(), nil +} + +func (z *zipStorage) Open(path string) (File, os.FileInfo, error) { + _, file, err := z.resolvePublic(path) + if err != nil { + return nil, nil, err + } + + rc, err := file.Open() + if err != nil { + return nil, nil, err + } + + return rc, file.FileInfo(), nil +} + +func (z *zipStorage) Close() { + z.archive.Close() +} + +func newZipStorage(lookupPath *client.LookupPath) (S, error) { + archive, err := zip.OpenReader(lookupPath.ArchivePath) + if err != nil { + return nil, err + } + + return &zipStorage{LookupPath: lookupPath, archive: archive}, nil +} diff --git a/metrics/metrics.go b/metrics/metrics.go index 44350ae570bb2caffeecfda1d6f275f42d097a47..0aa65f5578d4c2c8a540bb4c120c29a2026d9d16 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -5,30 +5,6 @@ import ( ) var ( - // DomainsServed counts the total number of sites served - DomainsServed = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "gitlab_pages_domains_served_total", - Help: "The total number of sites served by this Pages app", - }) - - // FailedDomainUpdates counts the number of failed site updates - FailedDomainUpdates = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "gitlab_pages_domains_failed_total", - Help: "The total number of site updates that have failed since daemon start", - }) - - // DomainUpdates counts the number of site updates successfully processed - DomainUpdates = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "gitlab_pages_domains_updated_total", - Help: "The total number of site updates successfully processed since daemon start", - }) - - // DomainLastUpdateTime is the UNIX timestamp of the last update - DomainLastUpdateTime = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "gitlab_pages_last_domain_update_seconds", - Help: "UNIX timestamp of the last update", - }) - // ProcessedRequests is the number of HTTP requests served ProcessedRequests = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "gitlab_pages_http_requests_total", @@ -45,9 +21,6 @@ var ( ) func init() { - prometheus.MustRegister(DomainsServed) - prometheus.MustRegister(DomainUpdates) - prometheus.MustRegister(DomainLastUpdateTime) prometheus.MustRegister(ProcessedRequests) prometheus.MustRegister(SessionsActive) } diff --git a/shared/pages/nested/project/public/index.html b/shared/pages/nested/project/public/index.html new file mode 100644 index 0000000000000000000000000000000000000000..b2d525b292f70f05dbcb03ba002a730f42db864e --- /dev/null +++ b/shared/pages/nested/project/public/index.html @@ -0,0 +1 @@ +index \ No newline at end of file diff --git a/shared/pages/nested/sub1/project/public/index.html b/shared/pages/nested/sub1/project/public/index.html new file mode 100644 index 0000000000000000000000000000000000000000..b2d525b292f70f05dbcb03ba002a730f42db864e --- /dev/null +++ b/shared/pages/nested/sub1/project/public/index.html @@ -0,0 +1 @@ +index \ No newline at end of file diff --git a/shared/pages/nested/sub1/sub2/project/public/index.html b/shared/pages/nested/sub1/sub2/project/public/index.html new file mode 100644 index 0000000000000000000000000000000000000000..b2d525b292f70f05dbcb03ba002a730f42db864e --- /dev/null +++ b/shared/pages/nested/sub1/sub2/project/public/index.html @@ -0,0 +1 @@ +index \ No newline at end of file diff --git a/shared/pages/nested/sub1/sub2/sub3/project/public/index.html b/shared/pages/nested/sub1/sub2/sub3/project/public/index.html new file mode 100644 index 0000000000000000000000000000000000000000..b2d525b292f70f05dbcb03ba002a730f42db864e --- /dev/null +++ b/shared/pages/nested/sub1/sub2/sub3/project/public/index.html @@ -0,0 +1 @@ +index \ No newline at end of file diff --git a/shared/pages/nested/sub1/sub2/sub3/sub4/project/public/index.html b/shared/pages/nested/sub1/sub2/sub3/sub4/project/public/index.html new file mode 100644 index 0000000000000000000000000000000000000000..b2d525b292f70f05dbcb03ba002a730f42db864e --- /dev/null +++ b/shared/pages/nested/sub1/sub2/sub3/sub4/project/public/index.html @@ -0,0 +1 @@ +index \ No newline at end of file diff --git a/shared/pages/nested/sub1/sub2/sub3/sub4/sub5/project/public/index.html b/shared/pages/nested/sub1/sub2/sub3/sub4/sub5/project/public/index.html new file mode 100644 index 0000000000000000000000000000000000000000..b2d525b292f70f05dbcb03ba002a730f42db864e --- /dev/null +++ b/shared/pages/nested/sub1/sub2/sub3/sub4/sub5/project/public/index.html @@ -0,0 +1 @@ +index \ No newline at end of file diff --git a/shared/pages/pages-deployment-100.zip b/shared/pages/pages-deployment-100.zip new file mode 100644 index 0000000000000000000000000000000000000000..9bb75f953f8797b7c1dffcbae87b2443cf43436e Binary files /dev/null and b/shared/pages/pages-deployment-100.zip differ