From 5a85f639293269fc5101662bea9f23ec99a46ef3 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Thu, 24 Jun 2021 12:21:50 +1000 Subject: [PATCH 01/11] WIP: do not create jail when disk is disabled --- daemon.go | 51 +++++++++++++++++---------------- test/acceptance/helpers_test.go | 8 +++++- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/daemon.go b/daemon.go index be790417a..623b8212f 100644 --- a/daemon.go +++ b/daemon.go @@ -291,10 +291,11 @@ func daemonize(config *config.Config) error { } log.WithFields(log.Fields{ - "uid": uid, - "gid": gid, - "in-place": inPlace, - "pages-root": pagesRoot, + "uid": uid, + "gid": gid, + "in-place": inPlace, + "pages-root": pagesRoot, + "enable-disk": config.GitLab.EnableDisk, }).Info("running the daemon as unprivileged user") cmd, err := daemonReexec(uid, gid, daemonRunProgram) @@ -303,28 +304,30 @@ func daemonize(config *config.Config) error { } defer killProcess(cmd) - // Run daemon in chroot environment - var wrapper *jail.Jail - if inPlace { - wrapper, err = chrootDaemon(cmd) - } else { - wrapper, err = jailDaemon(pagesRoot, cmd) - } - if err != nil { - log.WithError(err).Print("chroot failed") - return err - } - defer wrapper.Dispose() + if config.GitLab.EnableDisk { + // Run daemon in chroot environment + var wrapper *jail.Jail + if inPlace { + wrapper, err = chrootDaemon(cmd) + } else { + wrapper, err = jailDaemon(pagesRoot, cmd) + } + if err != nil { + log.WithError(err).Print("chroot failed") + return err + } + defer wrapper.Dispose() - // Unshare mount namespace - // 1. If this fails, in a worst case changes to mounts will propagate to other processes - // 2. Ensures that jail mount is not propagated to the parent mount namespace - // to avoid populating `tmp` directory with old mounts - _ = wrapper.Unshare() + // Unshare mount namespace + // 1. If this fails, in a worst case changes to mounts will propagate to other processes + // 2. Ensures that jail mount is not propagated to the parent mount namespace + // to avoid populating `tmp` directory with old mounts + _ = wrapper.Unshare() - if err := wrapper.Build(); err != nil { - log.WithError(err).Print("chroot build failed") - return err + if err := wrapper.Build(); err != nil { + log.WithError(err).Print("chroot build failed") + return err + } } // Create a pipe to pass the configuration diff --git a/test/acceptance/helpers_test.go b/test/acceptance/helpers_test.go index b267f1a26..f2f630bee 100644 --- a/test/acceptance/helpers_test.go +++ b/test/acceptance/helpers_test.go @@ -245,7 +245,13 @@ func RunPagesProcessWithStubGitLabServer(t *testing.T, opts ...processOption) *L source := NewGitlabDomainsSourceStub(t, processCfg.gitlabStubOpts) gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t) - processCfg.extraArgs = append(processCfg.extraArgs, "-pages-root", wd, "-internal-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", "gitlab") + processCfg.extraArgs = append(processCfg.extraArgs, + "-pages-root", wd, + "-internal-gitlab-server", source.URL, + "-api-secret-key", gitLabAPISecretKey, + "-domain-config-source", "gitlab", + "-enable-disk", "false", + ) logBuf, cleanup := runPagesProcess(t, processCfg.wait, processCfg.pagesBinary, processCfg.listeners, "", processCfg.envs, processCfg.extraArgs...) -- GitLab From 7791f9795fb8626ea5635bb88ee65f90c47b4847 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Thu, 24 Jun 2021 12:22:59 +1000 Subject: [PATCH 02/11] enable some acceptance tests --- test/acceptance/serving_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/acceptance/serving_test.go b/test/acceptance/serving_test.go index 32291e529..ac29f1425 100644 --- a/test/acceptance/serving_test.go +++ b/test/acceptance/serving_test.go @@ -14,7 +14,7 @@ import ( ) func TestUnknownHostReturnsNotFound(t *testing.T) { - skipUnlessEnabled(t) + //skipUnlessEnabled(t) RunPagesProcessWithStubGitLabServer(t) @@ -28,7 +28,7 @@ func TestUnknownHostReturnsNotFound(t *testing.T) { } func TestUnknownProjectReturnsNotFound(t *testing.T) { - skipUnlessEnabled(t) + //skipUnlessEnabled(t) RunPagesProcessWithStubGitLabServer(t) @@ -39,7 +39,7 @@ func TestUnknownProjectReturnsNotFound(t *testing.T) { } func TestGroupDomainReturns200(t *testing.T) { - skipUnlessEnabled(t) + //skipUnlessEnabled(t) RunPagesProcessWithStubGitLabServer(t) @@ -55,7 +55,7 @@ func TestGroupDomainReturns200(t *testing.T) { } func TestKnownHostReturns200(t *testing.T) { - skipUnlessEnabled(t) + //skipUnlessEnabled(t) RunPagesProcessWithStubGitLabServer(t) @@ -162,7 +162,7 @@ func TestNestedSubgroups(t *testing.T) { } func TestCustom404(t *testing.T) { - skipUnlessEnabled(t) + //skipUnlessEnabled(t) RunPagesProcessWithStubGitLabServer(t) @@ -224,7 +224,7 @@ func TestCustom404(t *testing.T) { } func TestCORSWhenDisabled(t *testing.T) { - skipUnlessEnabled(t) + //skipUnlessEnabled(t) RunPagesProcessWithStubGitLabServer(t, withExtraArgument("disable-cross-origin-requests", "true")) -- GitLab From 908ef40c9a7b959245d3f8689c95b9bd6183a40b Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Thu, 24 Jun 2021 12:31:05 +1000 Subject: [PATCH 03/11] split inplace and tmpdir tests --- .gitlab/ci/test.yml | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/.gitlab/ci/test.yml b/.gitlab/ci/test.yml index f0e484b94..b2b7d21e0 100644 --- a/.gitlab/ci/test.yml +++ b/.gitlab/ci/test.yml @@ -15,13 +15,17 @@ - make test ARGS='-short' - make junit-report -.tests-acceptance-deamon: +.test-acceptance-inplace: extends: .tests-common script: - - echo "Running just the acceptance tests daemonized (tmpdir)...." - - TEST_DAEMONIZE=tmpdir make acceptance - echo "Running just the acceptance tests daemonized (inplace)...." - TEST_DAEMONIZE=inplace make acceptance + +.tests-acceptance-tmpdir: + extends: .tests-common + script: + - echo "Running just the acceptance tests daemonized (tmpdir)...." + - TEST_DAEMONIZE=tmpdir make acceptance - make junit-report test:1.14: @@ -29,7 +33,11 @@ test:1.14: image: golang:1.14 test-acceptance:1.14: - extends: .tests-acceptance-deamon + extends: .tests-acceptance-tmpdir + image: golang:1.14 + +test-acceptance-inplace:1.14: + extends: .tests-acceptance-inplace image: golang:1.14 test:1.15: @@ -37,7 +45,11 @@ test:1.15: image: golang:1.15 test-acceptance:1.15: - extends: .tests-acceptance-deamon + extends: .tests-acceptance-tmpdir + image: golang:1.15 + +test-acceptance-inplace:1.15: + extends: .tests-acceptance-inplace image: golang:1.15 test:1.16: @@ -45,7 +57,11 @@ test:1.16: image: golang:1.16 test-acceptance:1.16: - extends: .tests-acceptance-deamon + extends: .tests-acceptance-tmpdir + image: golang:1.16 + +test-acceptance-inplace:1.16: + extends: .tests-acceptance-inplace image: golang:1.16 race: -- GitLab From c767eae57800a841bbb03947c31cacdfc35b96fb Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Mon, 28 Jun 2021 16:59:37 +1000 Subject: [PATCH 04/11] fix typo --- .gitlab/ci/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/test.yml b/.gitlab/ci/test.yml index b2b7d21e0..e13219288 100644 --- a/.gitlab/ci/test.yml +++ b/.gitlab/ci/test.yml @@ -15,7 +15,7 @@ - make test ARGS='-short' - make junit-report -.test-acceptance-inplace: +.tests-acceptance-inplace: extends: .tests-common script: - echo "Running just the acceptance tests daemonized (inplace)...." -- GitLab From 00ac7bf1e1046baae2b4878b77f8c18c5311e5d1 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Mon, 28 Jun 2021 17:17:57 +1000 Subject: [PATCH 05/11] only run one test --- .gitlab/ci/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/ci/test.yml b/.gitlab/ci/test.yml index e13219288..5b116a123 100644 --- a/.gitlab/ci/test.yml +++ b/.gitlab/ci/test.yml @@ -19,13 +19,13 @@ extends: .tests-common script: - echo "Running just the acceptance tests daemonized (inplace)...." - - TEST_DAEMONIZE=inplace make acceptance + - TEST_DAEMONIZE=inplace make acceptance ARGS='-run TestCustom404' .tests-acceptance-tmpdir: extends: .tests-common script: - echo "Running just the acceptance tests daemonized (tmpdir)...." - - TEST_DAEMONIZE=tmpdir make acceptance + - TEST_DAEMONIZE=tmpdir make acceptance ARGS='-run TestCustom404' - make junit-report test:1.14: -- GitLab From c14f2cf5ab8cd8d8ac6aa819a5558a9f9faf17ef Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Mon, 28 Jun 2021 17:29:21 +1000 Subject: [PATCH 06/11] run a couple of tests --- .gitlab/ci/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/ci/test.yml b/.gitlab/ci/test.yml index 5b116a123..9ecc05d65 100644 --- a/.gitlab/ci/test.yml +++ b/.gitlab/ci/test.yml @@ -20,12 +20,15 @@ script: - echo "Running just the acceptance tests daemonized (inplace)...." - TEST_DAEMONIZE=inplace make acceptance ARGS='-run TestCustom404' + - TEST_DAEMONIZE=inplace make acceptance ARGS='-run TestGroupDomainReturns200' + - make junit-report .tests-acceptance-tmpdir: extends: .tests-common script: - echo "Running just the acceptance tests daemonized (tmpdir)...." - TEST_DAEMONIZE=tmpdir make acceptance ARGS='-run TestCustom404' + - TEST_DAEMONIZE=inplace make acceptance ARGS='-run TestGroupDomainReturns200' - make junit-report test:1.14: -- GitLab From 115aa71627d56151d5775250fda92be0f20aebd5 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Mon, 28 Jun 2021 17:40:07 +1000 Subject: [PATCH 07/11] Run all tests --- .gitlab/ci/test.yml | 6 ++---- test/acceptance/serving_test.go | 4 +--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.gitlab/ci/test.yml b/.gitlab/ci/test.yml index 9ecc05d65..dd759eddb 100644 --- a/.gitlab/ci/test.yml +++ b/.gitlab/ci/test.yml @@ -19,16 +19,14 @@ extends: .tests-common script: - echo "Running just the acceptance tests daemonized (inplace)...." - - TEST_DAEMONIZE=inplace make acceptance ARGS='-run TestCustom404' - - TEST_DAEMONIZE=inplace make acceptance ARGS='-run TestGroupDomainReturns200' + - TEST_DAEMONIZE=inplace make acceptance - make junit-report .tests-acceptance-tmpdir: extends: .tests-common script: - echo "Running just the acceptance tests daemonized (tmpdir)...." - - TEST_DAEMONIZE=tmpdir make acceptance ARGS='-run TestCustom404' - - TEST_DAEMONIZE=inplace make acceptance ARGS='-run TestGroupDomainReturns200' + - TEST_DAEMONIZE=tmpdir make acceptance - make junit-report test:1.14: diff --git a/test/acceptance/serving_test.go b/test/acceptance/serving_test.go index ac29f1425..a7bf24ac7 100644 --- a/test/acceptance/serving_test.go +++ b/test/acceptance/serving_test.go @@ -776,9 +776,7 @@ func TestServerRepliesWithHeaders(t *testing.T) { func TestDiskDisabledFailsToServeFileAndLocalContent(t *testing.T) { skipUnlessEnabled(t) - logBuf := RunPagesProcessWithStubGitLabServer(t, - withExtraArgument("enable-disk", "false"), - ) + logBuf := RunPagesProcessWithStubGitLabServer(t) //withExtraArgument("enable-disk", "false"), for host, suffix := range map[string]string{ // API serves "source": { "type": "local" } -- GitLab From 6f3e8c758e97004ea9cd734b964a4f17c304f166 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Mon, 28 Jun 2021 17:49:29 +1000 Subject: [PATCH 08/11] enable other test --- test/acceptance/serving_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/serving_test.go b/test/acceptance/serving_test.go index a7bf24ac7..2f6a39472 100644 --- a/test/acceptance/serving_test.go +++ b/test/acceptance/serving_test.go @@ -774,7 +774,7 @@ func TestServerRepliesWithHeaders(t *testing.T) { } func TestDiskDisabledFailsToServeFileAndLocalContent(t *testing.T) { - skipUnlessEnabled(t) + //skipUnlessEnabled(t) logBuf := RunPagesProcessWithStubGitLabServer(t) //withExtraArgument("enable-disk", "false"), -- GitLab From f0175f1a8a8ce80e8650aba250f7d7ed7688fb41 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Wed, 30 Jun 2021 12:00:35 +1000 Subject: [PATCH 09/11] Add skip again, fix enable-disk flag --- test/acceptance/artifacts_test.go | 4 ---- test/acceptance/auth_test.go | 1 + test/acceptance/helpers_test.go | 2 +- test/acceptance/serving_test.go | 14 +++++++------- test/acceptance/stub_test.go | 1 + 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/test/acceptance/artifacts_test.go b/test/acceptance/artifacts_test.go index 2f578a738..398b62a64 100644 --- a/test/acceptance/artifacts_test.go +++ b/test/acceptance/artifacts_test.go @@ -15,8 +15,6 @@ import ( ) func TestArtifactProxyRequest(t *testing.T) { - skipUnlessEnabled(t, "not-inplace-chroot") - transport := (TestHTTPSClient.Transport).(*http.Transport).Clone() transport.ResponseHeaderTimeout = 5 * time.Second @@ -161,8 +159,6 @@ func TestArtifactProxyRequest(t *testing.T) { } func TestPrivateArtifactProxyRequest(t *testing.T) { - skipUnlessEnabled(t, "not-inplace-chroot") - setupTransport(t) testServer := makeGitLabPagesAccessStub(t) diff --git a/test/acceptance/auth_test.go b/test/acceptance/auth_test.go index 980fe377e..aa1f82fe3 100644 --- a/test/acceptance/auth_test.go +++ b/test/acceptance/auth_test.go @@ -264,6 +264,7 @@ func TestAccessControlUnderCustomDomain(t *testing.T) { func TestCustomErrorPageWithAuth(t *testing.T) { skipUnlessEnabled(t, "not-inplace-chroot") + testServer := makeGitLabPagesAccessStub(t) testServer.Start() defer testServer.Close() diff --git a/test/acceptance/helpers_test.go b/test/acceptance/helpers_test.go index f2f630bee..4b6af8676 100644 --- a/test/acceptance/helpers_test.go +++ b/test/acceptance/helpers_test.go @@ -250,7 +250,7 @@ func RunPagesProcessWithStubGitLabServer(t *testing.T, opts ...processOption) *L "-internal-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", "gitlab", - "-enable-disk", "false", + "-enable-disk=false", ) logBuf, cleanup := runPagesProcess(t, processCfg.wait, processCfg.pagesBinary, processCfg.listeners, "", processCfg.envs, processCfg.extraArgs...) diff --git a/test/acceptance/serving_test.go b/test/acceptance/serving_test.go index 2f6a39472..81a9cba09 100644 --- a/test/acceptance/serving_test.go +++ b/test/acceptance/serving_test.go @@ -14,7 +14,7 @@ import ( ) func TestUnknownHostReturnsNotFound(t *testing.T) { - //skipUnlessEnabled(t) + skipUnlessEnabled(t) RunPagesProcessWithStubGitLabServer(t) @@ -28,7 +28,7 @@ func TestUnknownHostReturnsNotFound(t *testing.T) { } func TestUnknownProjectReturnsNotFound(t *testing.T) { - //skipUnlessEnabled(t) + skipUnlessEnabled(t) RunPagesProcessWithStubGitLabServer(t) @@ -39,7 +39,7 @@ func TestUnknownProjectReturnsNotFound(t *testing.T) { } func TestGroupDomainReturns200(t *testing.T) { - //skipUnlessEnabled(t) + skipUnlessEnabled(t) RunPagesProcessWithStubGitLabServer(t) @@ -55,7 +55,7 @@ func TestGroupDomainReturns200(t *testing.T) { } func TestKnownHostReturns200(t *testing.T) { - //skipUnlessEnabled(t) + skipUnlessEnabled(t) RunPagesProcessWithStubGitLabServer(t) @@ -162,7 +162,7 @@ func TestNestedSubgroups(t *testing.T) { } func TestCustom404(t *testing.T) { - //skipUnlessEnabled(t) + skipUnlessEnabled(t) RunPagesProcessWithStubGitLabServer(t) @@ -224,7 +224,7 @@ func TestCustom404(t *testing.T) { } func TestCORSWhenDisabled(t *testing.T) { - //skipUnlessEnabled(t) + skipUnlessEnabled(t) RunPagesProcessWithStubGitLabServer(t, withExtraArgument("disable-cross-origin-requests", "true")) @@ -774,7 +774,7 @@ func TestServerRepliesWithHeaders(t *testing.T) { } func TestDiskDisabledFailsToServeFileAndLocalContent(t *testing.T) { - //skipUnlessEnabled(t) + skipUnlessEnabled(t) logBuf := RunPagesProcessWithStubGitLabServer(t) //withExtraArgument("enable-disk", "false"), diff --git a/test/acceptance/stub_test.go b/test/acceptance/stub_test.go index 6346fe4ed..91ad3a491 100644 --- a/test/acceptance/stub_test.go +++ b/test/acceptance/stub_test.go @@ -50,6 +50,7 @@ func withEnv(envs []string) processOption { func withExtraArgument(key, value string) processOption { return func(config *processConfig) { + // note: boolean arguments need to be defined using the `=` sign config.extraArgs = append(config.extraArgs, fmt.Sprintf("-%s=%s", key, value)) } } -- GitLab From d19f3821ad80cbb7482357e281af2a12e5fd0037 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Wed, 30 Jun 2021 17:08:26 +1000 Subject: [PATCH 10/11] WIP changing acceptance tests --- test/acceptance/testdata/api_responses.go | 64 +++++++++++++++++++---- 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/test/acceptance/testdata/api_responses.go b/test/acceptance/testdata/api_responses.go index 07349466f..9ec8f4046 100644 --- a/test/acceptance/testdata/api_responses.go +++ b/test/acceptance/testdata/api_responses.go @@ -2,8 +2,11 @@ package testdata import ( "fmt" + "net/http" + "net/http/httptest" "os" "path/filepath" + "reflect" "strings" "testing" @@ -21,10 +24,10 @@ var DomainResponses = map[string]responseFn{ "zip-from-disk.gitlab.io": customDomain(projectConfig{ projectID: 123, pathOnDisk: "@hashed/zip-from-disk.gitlab.io", - }), - "zip-from-disk-not-found.gitlab.io": customDomain(projectConfig{}), + }, true), + "zip-from-disk-not-found.gitlab.io": customDomain(projectConfig{}, true), // outside of working dir - "zip-not-allowed-path.gitlab.io": customDomain(projectConfig{pathOnDisk: "../../../../"}), + "zip-not-allowed-path.gitlab.io": customDomain(projectConfig{pathOnDisk: "../../../../"}, true), "group.gitlab-example.com": generateVirtualDomainFromDir("group", "group.gitlab-example.com", nil), "CapitalGroup.gitlab-example.com": generateVirtualDomainFromDir("CapitalGroup", "CapitalGroup.gitlab-example.com", nil), "group.404.gitlab-example.com": generateVirtualDomainFromDir("group.404", "group.404.gitlab-example.com", nil), @@ -41,11 +44,11 @@ var DomainResponses = map[string]responseFn{ "domain.404.com": customDomain(projectConfig{ projectID: 1000, pathOnDisk: "group.404/domain.404", - }), + }, false), "withacmechallenge.domain.com": customDomain(projectConfig{ projectID: 1234, pathOnDisk: "group.acme/with.acme.challenge", - }), + }, false), // NOTE: before adding more domains here, generate the zip archive by running (per project) // make zip PROJECT_SUBDIR=group/serving // make zip PROJECT_SUBDIR=group/project2 @@ -58,7 +61,9 @@ func generateVirtualDomainFromDir(dir, rootDomain string, perPrefixConfig map[st t.Helper() var foundZips []string - + handlerPaths := map[string]string{} + //path "group.https-only.gitlab-example.com/project2/public.zip" + //handler"group.https-only.gitlab-example.com/project2/public.zip" // walk over dir and save any paths containing a `.zip` file // $(GITLAB_PAGES_DIR)/shared/pages + "/" + group @@ -73,11 +78,14 @@ func generateVirtualDomainFromDir(dir, rootDomain string, perPrefixConfig map[st if strings.HasSuffix(info.Name(), ".zip") { project := strings.TrimPrefix(path, wd+"/"+dir) foundZips = append(foundZips, project) + handlerPaths[strings.ToLower(project)] = path } return nil }) + testServerURL := newZipFileServer(t, handlerPaths) + lookupPaths := make([]api.LookupPath, 0, len(foundZips)) // generate lookup paths for _, project := range foundZips { @@ -87,10 +95,13 @@ func generateVirtualDomainFromDir(dir, rootDomain string, perPrefixConfig map[st prefix := strings.TrimPrefix(project, dir) prefix = strings.TrimSuffix(prefix, "/"+filepath.Base(project)) + //urlPath := // use / as prefix when the current prefix matches the rootDomain, e.g. // if request is group.gitlab-example.com/ and group/group.gitlab-example.com/public.zip exists if prefix == "/"+rootDomain { prefix = "/" + // so it can be found by the testServerURL handler + //urlPath = project + "/public.zip" } cfg, ok := perPrefixConfig[prefix] @@ -105,7 +116,7 @@ func generateVirtualDomainFromDir(dir, rootDomain string, perPrefixConfig map[st Prefix: prefix, Source: api.Source{ Type: "zip", - Path: fmt.Sprintf("file://%s", wd+"/"+dir+project), + Path: fmt.Sprintf("%s%s", testServerURL, strings.ToLower(project)), }, } @@ -127,10 +138,19 @@ type projectConfig struct { } // customDomain with per project config -func customDomain(config projectConfig) responseFn { +func customDomain(config projectConfig, serveFromDisk bool) responseFn { return func(t *testing.T, wd string) api.VirtualDomain { t.Helper() + path := fmt.Sprintf("file://%s/%s/public.zip", wd, config.pathOnDisk) + if !serveFromDisk { + cleanPath := filepath.Clean(wd + "/" + config.pathOnDisk + "/public.zip") + testServerURL := newZipFileServer(t, map[string]string{ + "/" + config.pathOnDisk + "/public.zip": cleanPath, + }) + path = fmt.Sprintf("%s/%s/public.zip", testServerURL, config.pathOnDisk) + } + return api.VirtualDomain{ Certificate: "", Key: "", @@ -145,10 +165,36 @@ func customDomain(config projectConfig) responseFn { Prefix: "/", Source: api.Source{ Type: "zip", - Path: fmt.Sprintf("file://%s/%s/public.zip", wd, config.pathOnDisk), + Path: path, }, }, }, } } } + +func newZipFileServer(t *testing.T, projectPaths map[string]string) string { + t.Helper() + + mux := http.NewServeMux() + for urlPath, diskPath := range projectPaths { + fmt.Printf("creating handler for: %q in - %q\n\n", urlPath, diskPath) + mux.HandleFunc(urlPath, func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("FULL REQ URL: %q\n", r.URL.Path) + fmt.Printf("INSIDE THE HANDLER FOR: %q - opening:%q\n\n", urlPath, diskPath) + fi, err := os.Lstat(diskPath) + require.NoError(t, err) + require.False(t, fi.IsDir()) + http.ServeFile(w, r, diskPath) + }) + } + v := reflect.ValueOf(mux).Elem() + fmt.Printf("routes: %v\n", v.FieldByName("m")) + testServer := httptest.NewServer(mux) + + t.Cleanup(func() { + testServer.Close() + }) + + return testServer.URL +} -- GitLab From cc887ef3fb5700007591d56c4bc078f07979dfa6 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Thu, 1 Jul 2021 17:51:43 +1000 Subject: [PATCH 11/11] WIP: use filename from query, need to figure out issue with test --- test/acceptance/serving_test.go | 23 +++++------ test/acceptance/testdata/api_responses.go | 48 +++++++++++------------ 2 files changed, 34 insertions(+), 37 deletions(-) diff --git a/test/acceptance/serving_test.go b/test/acceptance/serving_test.go index 81a9cba09..bf75ed5e6 100644 --- a/test/acceptance/serving_test.go +++ b/test/acceptance/serving_test.go @@ -57,7 +57,9 @@ func TestGroupDomainReturns200(t *testing.T) { func TestKnownHostReturns200(t *testing.T) { skipUnlessEnabled(t) - RunPagesProcessWithStubGitLabServer(t) + RunPagesProcessWithStubGitLabServer(t, + withListeners([]ListenSpec{httpListener}), + ) tests := []struct { name string @@ -96,20 +98,19 @@ func TestKnownHostReturns200(t *testing.T) { content: "A subgroup project\n", }, } - for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - for _, spec := range supportedListeners() { - rsp, err := GetPageFromListener(t, spec, tt.host, tt.path) - require.NoError(t, err) - require.Equal(t, http.StatusOK, rsp.StatusCode) + //for _, spec := range supportedListeners() { + rsp, err := GetPageFromListener(t, httpListener, tt.host, tt.path) + require.NoError(t, err) + require.Equal(t, http.StatusOK, rsp.StatusCode) - body, err := ioutil.ReadAll(rsp.Body) - require.NoError(t, err) - require.Equal(t, tt.content, string(body)) + body, err := ioutil.ReadAll(rsp.Body) + require.NoError(t, err) + rsp.Body.Close() - rsp.Body.Close() - } + require.Equal(t, tt.content, string(body)) + //} }) } } diff --git a/test/acceptance/testdata/api_responses.go b/test/acceptance/testdata/api_responses.go index 9ec8f4046..39cefd90c 100644 --- a/test/acceptance/testdata/api_responses.go +++ b/test/acceptance/testdata/api_responses.go @@ -4,9 +4,9 @@ import ( "fmt" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" - "reflect" "strings" "testing" @@ -61,7 +61,6 @@ func generateVirtualDomainFromDir(dir, rootDomain string, perPrefixConfig map[st t.Helper() var foundZips []string - handlerPaths := map[string]string{} //path "group.https-only.gitlab-example.com/project2/public.zip" //handler"group.https-only.gitlab-example.com/project2/public.zip" // walk over dir and save any paths containing a `.zip` file @@ -78,13 +77,12 @@ func generateVirtualDomainFromDir(dir, rootDomain string, perPrefixConfig map[st if strings.HasSuffix(info.Name(), ".zip") { project := strings.TrimPrefix(path, wd+"/"+dir) foundZips = append(foundZips, project) - handlerPaths[strings.ToLower(project)] = path } return nil }) - testServerURL := newZipFileServer(t, handlerPaths) + testServerURL := newZipFileServer(t) lookupPaths := make([]api.LookupPath, 0, len(foundZips)) // generate lookup paths @@ -95,13 +93,10 @@ func generateVirtualDomainFromDir(dir, rootDomain string, perPrefixConfig map[st prefix := strings.TrimPrefix(project, dir) prefix = strings.TrimSuffix(prefix, "/"+filepath.Base(project)) - //urlPath := // use / as prefix when the current prefix matches the rootDomain, e.g. // if request is group.gitlab-example.com/ and group/group.gitlab-example.com/public.zip exists if prefix == "/"+rootDomain { prefix = "/" - // so it can be found by the testServerURL handler - //urlPath = project + "/public.zip" } cfg, ok := perPrefixConfig[prefix] @@ -116,7 +111,7 @@ func generateVirtualDomainFromDir(dir, rootDomain string, perPrefixConfig map[st Prefix: prefix, Source: api.Source{ Type: "zip", - Path: fmt.Sprintf("%s%s", testServerURL, strings.ToLower(project)), + Path: fmt.Sprintf("%s/zip?file=%s", testServerURL, url.QueryEscape(wd+"/"+dir+project)), }, } @@ -145,10 +140,8 @@ func customDomain(config projectConfig, serveFromDisk bool) responseFn { path := fmt.Sprintf("file://%s/%s/public.zip", wd, config.pathOnDisk) if !serveFromDisk { cleanPath := filepath.Clean(wd + "/" + config.pathOnDisk + "/public.zip") - testServerURL := newZipFileServer(t, map[string]string{ - "/" + config.pathOnDisk + "/public.zip": cleanPath, - }) - path = fmt.Sprintf("%s/%s/public.zip", testServerURL, config.pathOnDisk) + testServerURL := newZipFileServer(t) + path = fmt.Sprintf("%s/zip?file=%s", testServerURL, url.QueryEscape(cleanPath)) } return api.VirtualDomain{ @@ -173,23 +166,26 @@ func customDomain(config projectConfig, serveFromDisk bool) responseFn { } } -func newZipFileServer(t *testing.T, projectPaths map[string]string) string { +func newZipFileServer(t *testing.T) string { t.Helper() mux := http.NewServeMux() - for urlPath, diskPath := range projectPaths { - fmt.Printf("creating handler for: %q in - %q\n\n", urlPath, diskPath) - mux.HandleFunc(urlPath, func(w http.ResponseWriter, r *http.Request) { - fmt.Printf("FULL REQ URL: %q\n", r.URL.Path) - fmt.Printf("INSIDE THE HANDLER FOR: %q - opening:%q\n\n", urlPath, diskPath) - fi, err := os.Lstat(diskPath) - require.NoError(t, err) - require.False(t, fi.IsDir()) - http.ServeFile(w, r, diskPath) - }) - } - v := reflect.ValueOf(mux).Elem() - fmt.Printf("routes: %v\n", v.FieldByName("m")) + + mux.HandleFunc("/zip", func(w http.ResponseWriter, r *http.Request) { + file := r.URL.Query().Get("file") + + fi, err := os.Lstat(file) + require.NoError(t, err) + require.False(t, fi.IsDir()) + fmt.Printf("file: %q\nREQ HEADERS?: %+v\n\n", file, r.Header) + http.ServeFile(w, r, file) + + if f, ok := w.(http.Flusher); ok { + fmt.Printf("ARE WE FLUSHING?\n\n") + f.Flush() + } + }) + testServer := httptest.NewServer(mux) t.Cleanup(func() { -- GitLab