diff --git a/.gitlab/ci/test.yml b/.gitlab/ci/test.yml index f0e484b9403dc55e18a536f8116aec6f7f201941..dd759eddb9840209f8cf31fd364e4c44c3c2a68a 100644 --- a/.gitlab/ci/test.yml +++ b/.gitlab/ci/test.yml @@ -15,21 +15,30 @@ - make test ARGS='-short' - make junit-report -.tests-acceptance-deamon: +.tests-acceptance-inplace: extends: .tests-common script: - - echo "Running just the acceptance tests daemonized (tmpdir)...." - - TEST_DAEMONIZE=tmpdir make acceptance - echo "Running just the acceptance tests daemonized (inplace)...." - TEST_DAEMONIZE=inplace make acceptance - make junit-report +.tests-acceptance-tmpdir: + extends: .tests-common + script: + - echo "Running just the acceptance tests daemonized (tmpdir)...." + - TEST_DAEMONIZE=tmpdir make acceptance + - make junit-report + test:1.14: extends: .tests-unit image: golang:1.14 test-acceptance:1.14: - extends: .tests-acceptance-deamon + extends: .tests-acceptance-tmpdir + image: golang:1.14 + +test-acceptance-inplace:1.14: + extends: .tests-acceptance-inplace image: golang:1.14 test:1.15: @@ -37,7 +46,11 @@ test:1.15: image: golang:1.15 test-acceptance:1.15: - extends: .tests-acceptance-deamon + extends: .tests-acceptance-tmpdir + image: golang:1.15 + +test-acceptance-inplace:1.15: + extends: .tests-acceptance-inplace image: golang:1.15 test:1.16: @@ -45,7 +58,11 @@ test:1.16: image: golang:1.16 test-acceptance:1.16: - extends: .tests-acceptance-deamon + extends: .tests-acceptance-tmpdir + image: golang:1.16 + +test-acceptance-inplace:1.16: + extends: .tests-acceptance-inplace image: golang:1.16 race: diff --git a/daemon.go b/daemon.go index be790417a17b7064ce261a7a7b49d86d7ca1c856..623b8212f7bc462cd3861bf3e43cb9ed7fa99d46 100644 --- a/daemon.go +++ b/daemon.go @@ -291,10 +291,11 @@ func daemonize(config *config.Config) error { } log.WithFields(log.Fields{ - "uid": uid, - "gid": gid, - "in-place": inPlace, - "pages-root": pagesRoot, + "uid": uid, + "gid": gid, + "in-place": inPlace, + "pages-root": pagesRoot, + "enable-disk": config.GitLab.EnableDisk, }).Info("running the daemon as unprivileged user") cmd, err := daemonReexec(uid, gid, daemonRunProgram) @@ -303,28 +304,30 @@ func daemonize(config *config.Config) error { } defer killProcess(cmd) - // Run daemon in chroot environment - var wrapper *jail.Jail - if inPlace { - wrapper, err = chrootDaemon(cmd) - } else { - wrapper, err = jailDaemon(pagesRoot, cmd) - } - if err != nil { - log.WithError(err).Print("chroot failed") - return err - } - defer wrapper.Dispose() + if config.GitLab.EnableDisk { + // Run daemon in chroot environment + var wrapper *jail.Jail + if inPlace { + wrapper, err = chrootDaemon(cmd) + } else { + wrapper, err = jailDaemon(pagesRoot, cmd) + } + if err != nil { + log.WithError(err).Print("chroot failed") + return err + } + defer wrapper.Dispose() - // Unshare mount namespace - // 1. If this fails, in a worst case changes to mounts will propagate to other processes - // 2. Ensures that jail mount is not propagated to the parent mount namespace - // to avoid populating `tmp` directory with old mounts - _ = wrapper.Unshare() + // Unshare mount namespace + // 1. If this fails, in a worst case changes to mounts will propagate to other processes + // 2. Ensures that jail mount is not propagated to the parent mount namespace + // to avoid populating `tmp` directory with old mounts + _ = wrapper.Unshare() - if err := wrapper.Build(); err != nil { - log.WithError(err).Print("chroot build failed") - return err + if err := wrapper.Build(); err != nil { + log.WithError(err).Print("chroot build failed") + return err + } } // Create a pipe to pass the configuration diff --git a/test/acceptance/artifacts_test.go b/test/acceptance/artifacts_test.go index 2f578a7383b04ff8316e2b508565c32986ebde6b..398b62a649ad459e40cf470ad3afd080e37e325b 100644 --- a/test/acceptance/artifacts_test.go +++ b/test/acceptance/artifacts_test.go @@ -15,8 +15,6 @@ import ( ) func TestArtifactProxyRequest(t *testing.T) { - skipUnlessEnabled(t, "not-inplace-chroot") - transport := (TestHTTPSClient.Transport).(*http.Transport).Clone() transport.ResponseHeaderTimeout = 5 * time.Second @@ -161,8 +159,6 @@ func TestArtifactProxyRequest(t *testing.T) { } func TestPrivateArtifactProxyRequest(t *testing.T) { - skipUnlessEnabled(t, "not-inplace-chroot") - setupTransport(t) testServer := makeGitLabPagesAccessStub(t) diff --git a/test/acceptance/auth_test.go b/test/acceptance/auth_test.go index 980fe377ea2229d8a791397faef33189a26b60c7..aa1f82fe3e337dc927145a81b51efd0269b75260 100644 --- a/test/acceptance/auth_test.go +++ b/test/acceptance/auth_test.go @@ -264,6 +264,7 @@ func TestAccessControlUnderCustomDomain(t *testing.T) { func TestCustomErrorPageWithAuth(t *testing.T) { skipUnlessEnabled(t, "not-inplace-chroot") + testServer := makeGitLabPagesAccessStub(t) testServer.Start() defer testServer.Close() diff --git a/test/acceptance/helpers_test.go b/test/acceptance/helpers_test.go index b267f1a26d85f0b03bee08e7958bc7d4dbf5fe08..4b6af86760cc3c516ff712db77a8d468886c04a0 100644 --- a/test/acceptance/helpers_test.go +++ b/test/acceptance/helpers_test.go @@ -245,7 +245,13 @@ func RunPagesProcessWithStubGitLabServer(t *testing.T, opts ...processOption) *L source := NewGitlabDomainsSourceStub(t, processCfg.gitlabStubOpts) gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t) - processCfg.extraArgs = append(processCfg.extraArgs, "-pages-root", wd, "-internal-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", "gitlab") + processCfg.extraArgs = append(processCfg.extraArgs, + "-pages-root", wd, + "-internal-gitlab-server", source.URL, + "-api-secret-key", gitLabAPISecretKey, + "-domain-config-source", "gitlab", + "-enable-disk=false", + ) logBuf, cleanup := runPagesProcess(t, processCfg.wait, processCfg.pagesBinary, processCfg.listeners, "", processCfg.envs, processCfg.extraArgs...) diff --git a/test/acceptance/serving_test.go b/test/acceptance/serving_test.go index 32291e529c585afa4d9e5bf28cfd654ac7dc1f6d..bf75ed5e67e0cb4d107ec0abe97b70500db43f32 100644 --- a/test/acceptance/serving_test.go +++ b/test/acceptance/serving_test.go @@ -57,7 +57,9 @@ func TestGroupDomainReturns200(t *testing.T) { func TestKnownHostReturns200(t *testing.T) { skipUnlessEnabled(t) - RunPagesProcessWithStubGitLabServer(t) + RunPagesProcessWithStubGitLabServer(t, + withListeners([]ListenSpec{httpListener}), + ) tests := []struct { name string @@ -96,20 +98,19 @@ func TestKnownHostReturns200(t *testing.T) { content: "A subgroup project\n", }, } - for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - for _, spec := range supportedListeners() { - rsp, err := GetPageFromListener(t, spec, tt.host, tt.path) - require.NoError(t, err) - require.Equal(t, http.StatusOK, rsp.StatusCode) + //for _, spec := range supportedListeners() { + rsp, err := GetPageFromListener(t, httpListener, tt.host, tt.path) + require.NoError(t, err) + require.Equal(t, http.StatusOK, rsp.StatusCode) - body, err := ioutil.ReadAll(rsp.Body) - require.NoError(t, err) - require.Equal(t, tt.content, string(body)) + body, err := ioutil.ReadAll(rsp.Body) + require.NoError(t, err) + rsp.Body.Close() - rsp.Body.Close() - } + require.Equal(t, tt.content, string(body)) + //} }) } } @@ -776,9 +777,7 @@ func TestServerRepliesWithHeaders(t *testing.T) { func TestDiskDisabledFailsToServeFileAndLocalContent(t *testing.T) { skipUnlessEnabled(t) - logBuf := RunPagesProcessWithStubGitLabServer(t, - withExtraArgument("enable-disk", "false"), - ) + logBuf := RunPagesProcessWithStubGitLabServer(t) //withExtraArgument("enable-disk", "false"), for host, suffix := range map[string]string{ // API serves "source": { "type": "local" } diff --git a/test/acceptance/stub_test.go b/test/acceptance/stub_test.go index 6346fe4edff832e25eb7ad7d846e4819b2e7a190..91ad3a491eccace572d6c22d6797310f1b4bcd14 100644 --- a/test/acceptance/stub_test.go +++ b/test/acceptance/stub_test.go @@ -50,6 +50,7 @@ func withEnv(envs []string) processOption { func withExtraArgument(key, value string) processOption { return func(config *processConfig) { + // note: boolean arguments need to be defined using the `=` sign config.extraArgs = append(config.extraArgs, fmt.Sprintf("-%s=%s", key, value)) } } diff --git a/test/acceptance/testdata/api_responses.go b/test/acceptance/testdata/api_responses.go index 07349466fb98f2c5a73e58cd5181909d5dabffe0..39cefd90c526460bf5f26a3d1d60ec0d94a5651a 100644 --- a/test/acceptance/testdata/api_responses.go +++ b/test/acceptance/testdata/api_responses.go @@ -2,6 +2,9 @@ package testdata import ( "fmt" + "net/http" + "net/http/httptest" + "net/url" "os" "path/filepath" "strings" @@ -21,10 +24,10 @@ var DomainResponses = map[string]responseFn{ "zip-from-disk.gitlab.io": customDomain(projectConfig{ projectID: 123, pathOnDisk: "@hashed/zip-from-disk.gitlab.io", - }), - "zip-from-disk-not-found.gitlab.io": customDomain(projectConfig{}), + }, true), + "zip-from-disk-not-found.gitlab.io": customDomain(projectConfig{}, true), // outside of working dir - "zip-not-allowed-path.gitlab.io": customDomain(projectConfig{pathOnDisk: "../../../../"}), + "zip-not-allowed-path.gitlab.io": customDomain(projectConfig{pathOnDisk: "../../../../"}, true), "group.gitlab-example.com": generateVirtualDomainFromDir("group", "group.gitlab-example.com", nil), "CapitalGroup.gitlab-example.com": generateVirtualDomainFromDir("CapitalGroup", "CapitalGroup.gitlab-example.com", nil), "group.404.gitlab-example.com": generateVirtualDomainFromDir("group.404", "group.404.gitlab-example.com", nil), @@ -41,11 +44,11 @@ var DomainResponses = map[string]responseFn{ "domain.404.com": customDomain(projectConfig{ projectID: 1000, pathOnDisk: "group.404/domain.404", - }), + }, false), "withacmechallenge.domain.com": customDomain(projectConfig{ projectID: 1234, pathOnDisk: "group.acme/with.acme.challenge", - }), + }, false), // NOTE: before adding more domains here, generate the zip archive by running (per project) // make zip PROJECT_SUBDIR=group/serving // make zip PROJECT_SUBDIR=group/project2 @@ -58,7 +61,8 @@ func generateVirtualDomainFromDir(dir, rootDomain string, perPrefixConfig map[st t.Helper() var foundZips []string - + //path "group.https-only.gitlab-example.com/project2/public.zip" + //handler"group.https-only.gitlab-example.com/project2/public.zip" // walk over dir and save any paths containing a `.zip` file // $(GITLAB_PAGES_DIR)/shared/pages + "/" + group @@ -78,6 +82,8 @@ func generateVirtualDomainFromDir(dir, rootDomain string, perPrefixConfig map[st return nil }) + testServerURL := newZipFileServer(t) + lookupPaths := make([]api.LookupPath, 0, len(foundZips)) // generate lookup paths for _, project := range foundZips { @@ -105,7 +111,7 @@ func generateVirtualDomainFromDir(dir, rootDomain string, perPrefixConfig map[st Prefix: prefix, Source: api.Source{ Type: "zip", - Path: fmt.Sprintf("file://%s", wd+"/"+dir+project), + Path: fmt.Sprintf("%s/zip?file=%s", testServerURL, url.QueryEscape(wd+"/"+dir+project)), }, } @@ -127,10 +133,17 @@ type projectConfig struct { } // customDomain with per project config -func customDomain(config projectConfig) responseFn { +func customDomain(config projectConfig, serveFromDisk bool) responseFn { return func(t *testing.T, wd string) api.VirtualDomain { t.Helper() + path := fmt.Sprintf("file://%s/%s/public.zip", wd, config.pathOnDisk) + if !serveFromDisk { + cleanPath := filepath.Clean(wd + "/" + config.pathOnDisk + "/public.zip") + testServerURL := newZipFileServer(t) + path = fmt.Sprintf("%s/zip?file=%s", testServerURL, url.QueryEscape(cleanPath)) + } + return api.VirtualDomain{ Certificate: "", Key: "", @@ -145,10 +158,39 @@ func customDomain(config projectConfig) responseFn { Prefix: "/", Source: api.Source{ Type: "zip", - Path: fmt.Sprintf("file://%s/%s/public.zip", wd, config.pathOnDisk), + Path: path, }, }, }, } } } + +func newZipFileServer(t *testing.T) string { + t.Helper() + + mux := http.NewServeMux() + + mux.HandleFunc("/zip", func(w http.ResponseWriter, r *http.Request) { + file := r.URL.Query().Get("file") + + fi, err := os.Lstat(file) + require.NoError(t, err) + require.False(t, fi.IsDir()) + fmt.Printf("file: %q\nREQ HEADERS?: %+v\n\n", file, r.Header) + http.ServeFile(w, r, file) + + if f, ok := w.(http.Flusher); ok { + fmt.Printf("ARE WE FLUSHING?\n\n") + f.Flush() + } + }) + + testServer := httptest.NewServer(mux) + + t.Cleanup(func() { + testServer.Close() + }) + + return testServer.URL +}