From 2bcd6e358fe7aa2c1b65572f0616f83beded7a72 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 14 Nov 2017 19:49:58 +0100 Subject: [PATCH 01/10] Move all sources to `cmd/gitlab-pages` --- acceptance_test.go => cmd/gitlab-pages/acceptance_test.go | 0 app.go => cmd/gitlab-pages/app.go | 0 app_config.go => cmd/gitlab-pages/app_config.go | 0 daemon.go => cmd/gitlab-pages/daemon.go | 0 domain.go => cmd/gitlab-pages/domain.go | 0 domain_config.go => cmd/gitlab-pages/domain_config.go | 0 domain_config_test.go => cmd/gitlab-pages/domain_config_test.go | 0 domain_test.go => cmd/gitlab-pages/domain_test.go | 0 domains.go => cmd/gitlab-pages/domains.go | 0 domains_test.go => cmd/gitlab-pages/domains_test.go | 0 helpers.go => cmd/gitlab-pages/helpers.go | 0 helpers_test.go => cmd/gitlab-pages/helpers_test.go | 0 logging.go => cmd/gitlab-pages/logging.go | 0 logging_test.go => cmd/gitlab-pages/logging_test.go | 0 main.go => cmd/gitlab-pages/main.go | 0 multi_string_flag.go => cmd/gitlab-pages/multi_string_flag.go | 0 .../gitlab-pages/multi_string_flag_test.go | 0 server.go => cmd/gitlab-pages/server.go | 0 18 files changed, 0 insertions(+), 0 deletions(-) rename acceptance_test.go => cmd/gitlab-pages/acceptance_test.go (100%) rename app.go => cmd/gitlab-pages/app.go (100%) rename app_config.go => cmd/gitlab-pages/app_config.go (100%) rename daemon.go => cmd/gitlab-pages/daemon.go (100%) rename domain.go => cmd/gitlab-pages/domain.go (100%) rename domain_config.go => cmd/gitlab-pages/domain_config.go (100%) rename domain_config_test.go => cmd/gitlab-pages/domain_config_test.go (100%) rename domain_test.go => cmd/gitlab-pages/domain_test.go (100%) rename domains.go => cmd/gitlab-pages/domains.go (100%) rename domains_test.go => cmd/gitlab-pages/domains_test.go (100%) rename helpers.go => cmd/gitlab-pages/helpers.go (100%) rename helpers_test.go => cmd/gitlab-pages/helpers_test.go (100%) rename logging.go => cmd/gitlab-pages/logging.go (100%) rename logging_test.go => cmd/gitlab-pages/logging_test.go (100%) rename main.go => cmd/gitlab-pages/main.go (100%) rename multi_string_flag.go => cmd/gitlab-pages/multi_string_flag.go (100%) rename multi_string_flag_test.go => cmd/gitlab-pages/multi_string_flag_test.go (100%) rename server.go => cmd/gitlab-pages/server.go (100%) diff --git a/acceptance_test.go b/cmd/gitlab-pages/acceptance_test.go similarity index 100% rename from acceptance_test.go rename to cmd/gitlab-pages/acceptance_test.go diff --git a/app.go b/cmd/gitlab-pages/app.go similarity index 100% rename from app.go rename to cmd/gitlab-pages/app.go diff --git a/app_config.go b/cmd/gitlab-pages/app_config.go similarity index 100% rename from app_config.go rename to cmd/gitlab-pages/app_config.go diff --git a/daemon.go b/cmd/gitlab-pages/daemon.go similarity index 100% rename from daemon.go rename to cmd/gitlab-pages/daemon.go diff --git a/domain.go b/cmd/gitlab-pages/domain.go similarity index 100% rename from domain.go rename to cmd/gitlab-pages/domain.go diff --git a/domain_config.go b/cmd/gitlab-pages/domain_config.go similarity index 100% rename from domain_config.go rename to cmd/gitlab-pages/domain_config.go diff --git a/domain_config_test.go b/cmd/gitlab-pages/domain_config_test.go similarity index 100% rename from domain_config_test.go rename to cmd/gitlab-pages/domain_config_test.go diff --git a/domain_test.go b/cmd/gitlab-pages/domain_test.go similarity index 100% rename from domain_test.go rename to cmd/gitlab-pages/domain_test.go diff --git a/domains.go b/cmd/gitlab-pages/domains.go similarity index 100% rename from domains.go rename to cmd/gitlab-pages/domains.go diff --git a/domains_test.go b/cmd/gitlab-pages/domains_test.go similarity index 100% rename from domains_test.go rename to cmd/gitlab-pages/domains_test.go diff --git a/helpers.go b/cmd/gitlab-pages/helpers.go similarity index 100% rename from helpers.go rename to cmd/gitlab-pages/helpers.go diff --git a/helpers_test.go b/cmd/gitlab-pages/helpers_test.go similarity index 100% rename from helpers_test.go rename to cmd/gitlab-pages/helpers_test.go diff --git a/logging.go b/cmd/gitlab-pages/logging.go similarity index 100% rename from logging.go rename to cmd/gitlab-pages/logging.go diff --git a/logging_test.go b/cmd/gitlab-pages/logging_test.go similarity index 100% rename from logging_test.go rename to cmd/gitlab-pages/logging_test.go diff --git a/main.go b/cmd/gitlab-pages/main.go similarity index 100% rename from main.go rename to cmd/gitlab-pages/main.go diff --git a/multi_string_flag.go b/cmd/gitlab-pages/multi_string_flag.go similarity index 100% rename from multi_string_flag.go rename to cmd/gitlab-pages/multi_string_flag.go diff --git a/multi_string_flag_test.go b/cmd/gitlab-pages/multi_string_flag_test.go similarity index 100% rename from multi_string_flag_test.go rename to cmd/gitlab-pages/multi_string_flag_test.go diff --git a/server.go b/cmd/gitlab-pages/server.go similarity index 100% rename from server.go rename to cmd/gitlab-pages/server.go -- GitLab From 232375da28333e327433eb5f770edf7d50cf3f2f Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 14 Nov 2017 20:07:57 +0100 Subject: [PATCH 02/10] Add skeleton of Pages Deployer --- cmd/gitlab-pages-deployer/job.go | 7 ++ cmd/gitlab-pages-deployer/main.go | 89 +++++++++++++++++++++++++ cmd/gitlab-pages-deployer/middleware.go | 12 ++++ 3 files changed, 108 insertions(+) create mode 100644 cmd/gitlab-pages-deployer/job.go create mode 100644 cmd/gitlab-pages-deployer/main.go create mode 100644 cmd/gitlab-pages-deployer/middleware.go diff --git a/cmd/gitlab-pages-deployer/job.go b/cmd/gitlab-pages-deployer/job.go new file mode 100644 index 000000000..38650164e --- /dev/null +++ b/cmd/gitlab-pages-deployer/job.go @@ -0,0 +1,7 @@ +package main + +import workers "github.com/jrallison/go-workers" + +func pagesJob(message *workers.Msg) { + println(message.ToJson()) +} diff --git a/cmd/gitlab-pages-deployer/main.go b/cmd/gitlab-pages-deployer/main.go new file mode 100644 index 000000000..d5dad8aaa --- /dev/null +++ b/cmd/gitlab-pages-deployer/main.go @@ -0,0 +1,89 @@ +package main + +import ( + "flag" + "fmt" + "log" + "net/http" + "os" + "strconv" + + workers "github.com/jrallison/go-workers" +) + +// VERSION stores the information about the semantic version of application +var VERSION = "dev" + +// REVISION stores the information about the git revision of application +var REVISION = "HEAD" + +var ( + showVersion = flag.Bool("version", false, "Show version") + + apiURL = flag.String("api-url", "https://gitlab.com/api/v4", "The API URL to GitLab") + apiAccessToken = flag.String("api-access-token", "", "API Access Token to post back the statuses of Pages") + + statsServer = flag.String("stats-server", "localhost:9818", "Address to statistics server") + + deployerRoot = flag.String("deployer-root", "shared/pages", "The directory where pages are stored") + deployerMaximumSize = flag.Int("deployer-maximum-size", 1024, "The maximum size of artifacts extracted (in Bytes)") + deployerConcurrency = flag.Int("deployer-concurrency", 10, "The maximum concurrency") + deployerID = flag.String("deployer-id", "1", "Unique ID of Deployer") + + redisServer = flag.String("redis-server", "localhost:6379", "The address of Redis Server") + redisDatabase = flag.String("redis-database", "0", "The name of Redis Database") + redisPassword = flag.String("redis-password", "", "The password to Redis Database") + redisNamespace = flag.String("redis-namespace", "", "The namespace to use") + redisPool = flag.Int("redis-password", 10, "The connection pool to Redis Database") +) + +func runStatsServer() { + http.HandleFunc("/stats", workers.Stats) + + if *statsServer != "" { + log.Println("Stats are available at", fmt.Sprint(*statsServer, "/stats")) + + go func() { + if err := http.ListenAndServe(*statsServer, nil); err != nil { + log.Println(err) + } + }() + } +} + +func appMain() { + flag.Parse() + + printVersion(*showVersion, VERSION) + + log.Printf("GitLab Pages Deployer %s (%s)", VERSION, REVISION) + log.Printf("URL: https://gitlab.com/gitlab-org/gitlab-pages\n") + + runStatsServer() + + workers.Configure(map[string]string{ + "server": *redisServer, + "database": *redisDatabase, + "password": *redisPassword, + "pool": strconv.Itoa(*redisPool), + "process": *deployerID, + }) + + workers.Middleware.Append(&myMiddleware{}) + workers.Process("pages", pagesJob, *deployerConcurrency) + workers.Run() +} + +func printVersion(showVersion bool, version string) { + if showVersion { + log.SetFlags(0) + log.Printf(version) + os.Exit(0) + } +} + +func main() { + log.SetOutput(os.Stderr) + + appMain() +} diff --git a/cmd/gitlab-pages-deployer/middleware.go b/cmd/gitlab-pages-deployer/middleware.go new file mode 100644 index 000000000..6e41d3ca1 --- /dev/null +++ b/cmd/gitlab-pages-deployer/middleware.go @@ -0,0 +1,12 @@ +package main + +import workers "github.com/jrallison/go-workers" + +type myMiddleware struct{} + +func (r *myMiddleware) Call(queue string, message *workers.Msg, next func() bool) (acknowledge bool) { + // do something before each message is processed + acknowledge = next() + // do something after each message is processed + return +} -- GitLab From a8ffc26791f5b9d3ebf9a956c3885120156f328b Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 14 Nov 2017 20:23:27 +0100 Subject: [PATCH 03/10] Start implementing PagesWorker --- cmd/gitlab-pages-deployer/job.go | 9 +++++++-- cmd/gitlab-pages-deployer/main.go | 15 ++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/cmd/gitlab-pages-deployer/job.go b/cmd/gitlab-pages-deployer/job.go index 38650164e..98f992988 100644 --- a/cmd/gitlab-pages-deployer/job.go +++ b/cmd/gitlab-pages-deployer/job.go @@ -1,7 +1,12 @@ package main -import workers "github.com/jrallison/go-workers" +import ( + "time" + + workers "github.com/jrallison/go-workers" +) func pagesJob(message *workers.Msg) { - println(message.ToJson()) + time.Sleep(time.Minute) + println(message.Args()) } diff --git a/cmd/gitlab-pages-deployer/main.go b/cmd/gitlab-pages-deployer/main.go index d5dad8aaa..329b172b8 100644 --- a/cmd/gitlab-pages-deployer/main.go +++ b/cmd/gitlab-pages-deployer/main.go @@ -33,8 +33,8 @@ var ( redisServer = flag.String("redis-server", "localhost:6379", "The address of Redis Server") redisDatabase = flag.String("redis-database", "0", "The name of Redis Database") redisPassword = flag.String("redis-password", "", "The password to Redis Database") - redisNamespace = flag.String("redis-namespace", "", "The namespace to use") - redisPool = flag.Int("redis-password", 10, "The connection pool to Redis Database") + redisNamespace = flag.String("redis-namespace", "resque:gitlab", "The namespace to use") + redisPool = flag.Int("redis-pool", 10, "The connection pool to Redis Database") ) func runStatsServer() { @@ -62,11 +62,12 @@ func appMain() { runStatsServer() workers.Configure(map[string]string{ - "server": *redisServer, - "database": *redisDatabase, - "password": *redisPassword, - "pool": strconv.Itoa(*redisPool), - "process": *deployerID, + "server": *redisServer, + "database": *redisDatabase, + "password": *redisPassword, + "namespace": *redisNamespace, + "pool": strconv.Itoa(*redisPool), + "process": *deployerID, }) workers.Middleware.Append(&myMiddleware{}) -- GitLab From 551084d8b995450aaf41bf9288376b982575b9f6 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 14 Nov 2017 20:39:34 +0100 Subject: [PATCH 04/10] Define all commands --- .gitignore | 1 + cmd/gitlab-pages-deployer/deploy.go | 11 +++++ cmd/gitlab-pages-deployer/job.go | 12 ----- cmd/gitlab-pages-deployer/transfer.go | 19 ++++++++ cmd/gitlab-pages-deployer/worker.go | 69 +++++++++++++++++++++++++++ 5 files changed, 100 insertions(+), 12 deletions(-) create mode 100644 cmd/gitlab-pages-deployer/deploy.go delete mode 100644 cmd/gitlab-pages-deployer/job.go create mode 100644 cmd/gitlab-pages-deployer/transfer.go create mode 100644 cmd/gitlab-pages-deployer/worker.go diff --git a/.gitignore b/.gitignore index 1e0ddeed2..41c6bf517 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Created by .ignore support plugin (hsz.mobi) shared/pages/.update /gitlab-pages +/gitlab-pages-deployer # Used by the makefile /.GOPATH diff --git a/cmd/gitlab-pages-deployer/deploy.go b/cmd/gitlab-pages-deployer/deploy.go new file mode 100644 index 000000000..8b2fb2ff8 --- /dev/null +++ b/cmd/gitlab-pages-deployer/deploy.go @@ -0,0 +1,11 @@ +package main + +import workers "github.com/jrallison/go-workers" + +func deployJob(message *workers.Msg, projectId int64, projectPath string, jobId int64, config map[string]interface{}) { + +} + +func configJob(message *workers.Msg, projectId int64, projectPath string, config map[string]interface{}) { + +} diff --git a/cmd/gitlab-pages-deployer/job.go b/cmd/gitlab-pages-deployer/job.go deleted file mode 100644 index 98f992988..000000000 --- a/cmd/gitlab-pages-deployer/job.go +++ /dev/null @@ -1,12 +0,0 @@ -package main - -import ( - "time" - - workers "github.com/jrallison/go-workers" -) - -func pagesJob(message *workers.Msg) { - time.Sleep(time.Minute) - println(message.Args()) -} diff --git a/cmd/gitlab-pages-deployer/transfer.go b/cmd/gitlab-pages-deployer/transfer.go new file mode 100644 index 000000000..b379d7567 --- /dev/null +++ b/cmd/gitlab-pages-deployer/transfer.go @@ -0,0 +1,19 @@ +package main + +import workers "github.com/jrallison/go-workers" + +func renameNamespaceJob(message *workers.Msg, projectId int64, fullPathWas, fullPath string) { + +} + +func renameProjectJob(message *workers.Msg, projectId int64, pathWas, path, fullPath string) { + +} + +func moveProjectJob(message *workers.Msg, projectId int64, path, fullPathWas, fullPath string) { + +} + +func removeJob(message *workers.Msg, projectId int64, namespacePath, path string) { + +} diff --git a/cmd/gitlab-pages-deployer/worker.go b/cmd/gitlab-pages-deployer/worker.go new file mode 100644 index 000000000..41140e2b2 --- /dev/null +++ b/cmd/gitlab-pages-deployer/worker.go @@ -0,0 +1,69 @@ +package main + +import ( + workers "github.com/jrallison/go-workers" +) + +func pagesJob(message *workers.Msg) { + if message.Get("class").MustString() != "PagesWorker" { + panic("Expected PagesWorker class: " + message.Get("class").MustString()) + } + + args := message.Args() + + switch args.GetIndex(0).MustString() { + case "deploy": + deployJob( + message, + args.GetIndex(1).MustInt64(), + args.GetIndex(2).MustString(), + args.GetIndex(3).MustInt64(), + args.GetIndex(4).MustMap(), + ) + + case "remove": + removeJob( + message, + args.GetIndex(1).MustInt64(), + args.GetIndex(2).MustString(), + args.GetIndex(3).MustString(), + ) + + case "config": + configJob( + message, + args.GetIndex(1).MustInt64(), + args.GetIndex(2).MustString(), + args.GetIndex(3).MustMap(), + ) + + case "rename_namespace": + renameNamespaceJob( + message, + args.GetIndex(1).MustInt64(), + args.GetIndex(2).MustString(), + args.GetIndex(3).MustString(), + ) + + case "rename_project": + renameProjectJob( + message, + args.GetIndex(1).MustInt64(), + args.GetIndex(2).MustString(), + args.GetIndex(3).MustString(), + args.GetIndex(4).MustString(), + ) + + case "move_project": + moveProjectJob( + message, + args.GetIndex(1).MustInt64(), + args.GetIndex(2).MustString(), + args.GetIndex(3).MustString(), + args.GetIndex(4).MustString(), + ) + + default: + panic("Unknown method: " + args.GetIndex(0).MustString()) + } +} -- GitLab From 43895d6497eb49c5f7c206f01786e42acf56b48a Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 14 Nov 2017 20:45:09 +0100 Subject: [PATCH 05/10] Implement transfer methods --- cmd/gitlab-pages-deployer/transfer.go | 30 +++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/cmd/gitlab-pages-deployer/transfer.go b/cmd/gitlab-pages-deployer/transfer.go index b379d7567..13dd08f35 100644 --- a/cmd/gitlab-pages-deployer/transfer.go +++ b/cmd/gitlab-pages-deployer/transfer.go @@ -1,19 +1,37 @@ package main -import workers "github.com/jrallison/go-workers" +import ( + "os" + "path/filepath" -func renameNamespaceJob(message *workers.Msg, projectId int64, fullPathWas, fullPath string) { + workers "github.com/jrallison/go-workers" +) +func panicOnFileSystemError(err error) { + if err == nil && !os.IsNotExist(err) { + panic(err) + } } -func renameProjectJob(message *workers.Msg, projectId int64, pathWas, path, fullPath string) { +func renameNamespaceJob(message *workers.Msg, projectID int64, fullPathWas, fullPath string) { + err := os.Rename(filepath.Join(*deployerRoot, fullPathWas), filepath.Join(*deployerRoot, fullPath)) + panicOnFileSystemError(err) +} +func renameProjectJob(message *workers.Msg, projectID int64, pathWas, path, fullPath string) { + err := os.Rename(filepath.Join(*deployerRoot, fullPath, pathWas), filepath.Join(*deployerRoot, fullPath, path)) + panicOnFileSystemError(err) } -func moveProjectJob(message *workers.Msg, projectId int64, path, fullPathWas, fullPath string) { +func moveProjectJob(message *workers.Msg, projectID int64, path, fullPathWas, fullPath string) { + err := os.MkdirAll(filepath.Join(*deployerRoot, fullPath), 0750) + panicOnFileSystemError(err) + err = os.Rename(filepath.Join(*deployerRoot, fullPathWas, path), filepath.Join(*deployerRoot, fullPath, path)) + panicOnFileSystemError(err) } -func removeJob(message *workers.Msg, projectId int64, namespacePath, path string) { - +func removeJob(message *workers.Msg, projectID int64, namespacePath, path string) { + err := os.RemoveAll(filepath.Join(*deployerRoot, namespacePath, path)) + panicOnFileSystemError(err) } -- GitLab From 4c24b5b6d95898080793698f91bc8596a1586b17 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 14 Nov 2017 20:55:43 +0100 Subject: [PATCH 06/10] Implement config saving --- cmd/gitlab-pages-deployer/config.go | 48 +++++++++++++++++++++++++++++ cmd/gitlab-pages-deployer/deploy.go | 12 +++++--- 2 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 cmd/gitlab-pages-deployer/config.go diff --git a/cmd/gitlab-pages-deployer/config.go b/cmd/gitlab-pages-deployer/config.go new file mode 100644 index 000000000..a5be3ed85 --- /dev/null +++ b/cmd/gitlab-pages-deployer/config.go @@ -0,0 +1,48 @@ +package main + +import ( + "crypto/rand" + "encoding/json" + "io/ioutil" + "os" + "path/filepath" +) + +func panicOnError(err error) { + if err != nil { + panic(err) + } +} + +func saveConfig(projectID int64, projectPath string, config map[string]interface{}) { + data, err := json.MarshalIndent(config, "", "\t") + panicOnError(err) + + if replaceFile(filepath.Join(*deployerRoot, projectPath), "config.json", data) { + touchDaemon() + } +} + +func replaceFile(path, filename string, data []byte) bool { + err := os.MkdirAll(path, 0750) + panicOnFileSystemError(err) + + f, err := ioutil.TempFile(path, "config") + panicOnError(err) + defer f.Close() + defer os.Remove(f.Name()) + + println(f.Name()) + + err = os.Rename(f.Name(), filepath.Join(path, filename)) + panicOnFileSystemError(err) + return true +} + +func touchDaemon() { + randomData := make([]byte, 32) + _, err := rand.Read(randomData) + panicOnError(err) + + replaceFile(*deployerRoot, ".update", randomData) +} diff --git a/cmd/gitlab-pages-deployer/deploy.go b/cmd/gitlab-pages-deployer/deploy.go index 8b2fb2ff8..0fd810ddd 100644 --- a/cmd/gitlab-pages-deployer/deploy.go +++ b/cmd/gitlab-pages-deployer/deploy.go @@ -1,11 +1,13 @@ package main -import workers "github.com/jrallison/go-workers" - -func deployJob(message *workers.Msg, projectId int64, projectPath string, jobId int64, config map[string]interface{}) { +import ( + workers "github.com/jrallison/go-workers" +) +func deployJob(message *workers.Msg, projectID int64, projectPath string, jobID int64, config map[string]interface{}) { + saveConfig(projectID, projectPath, config) } -func configJob(message *workers.Msg, projectId int64, projectPath string, config map[string]interface{}) { - +func configJob(message *workers.Msg, projectID int64, projectPath string, config map[string]interface{}) { + saveConfig(projectID, projectPath, config) } -- GitLab From fea3435903a77312da64df6d312bee1c2a8e2d51 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 14 Nov 2017 21:00:13 +0100 Subject: [PATCH 07/10] Save only on change --- cmd/gitlab-pages-deployer/config.go | 13 +++++++++++-- cmd/gitlab-pages-deployer/transfer.go | 2 +- shared/pages/h5bp/test/test/config.json | 3 +++ 3 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 shared/pages/h5bp/test/test/config.json diff --git a/cmd/gitlab-pages-deployer/config.go b/cmd/gitlab-pages-deployer/config.go index a5be3ed85..940526bcb 100644 --- a/cmd/gitlab-pages-deployer/config.go +++ b/cmd/gitlab-pages-deployer/config.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "crypto/rand" "encoding/json" "io/ioutil" @@ -24,17 +25,25 @@ func saveConfig(projectID int64, projectPath string, config map[string]interface } func replaceFile(path, filename string, data []byte) bool { + targetFile := filepath.Join(path, filename) + err := os.MkdirAll(path, 0750) panicOnFileSystemError(err) + dataWas, err := ioutil.ReadFile(targetFile) + if err == nil && bytes.Equal(data, dataWas) { + return false + } + f, err := ioutil.TempFile(path, "config") panicOnError(err) defer f.Close() defer os.Remove(f.Name()) - println(f.Name()) + _, err = f.Write(data) + panicOnError(err) - err = os.Rename(f.Name(), filepath.Join(path, filename)) + err = os.Rename(f.Name(), targetFile) panicOnFileSystemError(err) return true } diff --git a/cmd/gitlab-pages-deployer/transfer.go b/cmd/gitlab-pages-deployer/transfer.go index 13dd08f35..7ee2f1d4b 100644 --- a/cmd/gitlab-pages-deployer/transfer.go +++ b/cmd/gitlab-pages-deployer/transfer.go @@ -8,7 +8,7 @@ import ( ) func panicOnFileSystemError(err error) { - if err == nil && !os.IsNotExist(err) { + if err != nil && !os.IsNotExist(err) { panic(err) } } diff --git a/shared/pages/h5bp/test/test/config.json b/shared/pages/h5bp/test/test/config.json new file mode 100644 index 000000000..ad702b32c --- /dev/null +++ b/shared/pages/h5bp/test/test/config.json @@ -0,0 +1,3 @@ +{ + "domains": [] +} \ No newline at end of file -- GitLab From ffa7957c9990bdc84d85d2a244fc695bb1c1a59e Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 14 Nov 2017 21:00:44 +0100 Subject: [PATCH 08/10] Force to reload daemon on project transfer --- cmd/gitlab-pages-deployer/transfer.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/gitlab-pages-deployer/transfer.go b/cmd/gitlab-pages-deployer/transfer.go index 7ee2f1d4b..b2722f097 100644 --- a/cmd/gitlab-pages-deployer/transfer.go +++ b/cmd/gitlab-pages-deployer/transfer.go @@ -16,11 +16,15 @@ func panicOnFileSystemError(err error) { func renameNamespaceJob(message *workers.Msg, projectID int64, fullPathWas, fullPath string) { err := os.Rename(filepath.Join(*deployerRoot, fullPathWas), filepath.Join(*deployerRoot, fullPath)) panicOnFileSystemError(err) + + touchDaemon() } func renameProjectJob(message *workers.Msg, projectID int64, pathWas, path, fullPath string) { err := os.Rename(filepath.Join(*deployerRoot, fullPath, pathWas), filepath.Join(*deployerRoot, fullPath, path)) panicOnFileSystemError(err) + + touchDaemon() } func moveProjectJob(message *workers.Msg, projectID int64, path, fullPathWas, fullPath string) { @@ -29,9 +33,13 @@ func moveProjectJob(message *workers.Msg, projectID int64, path, fullPathWas, fu err = os.Rename(filepath.Join(*deployerRoot, fullPathWas, path), filepath.Join(*deployerRoot, fullPath, path)) panicOnFileSystemError(err) + + touchDaemon() } func removeJob(message *workers.Msg, projectID int64, namespacePath, path string) { err := os.RemoveAll(filepath.Join(*deployerRoot, namespacePath, path)) panicOnFileSystemError(err) + + touchDaemon() } -- GitLab From 8a4001aaecb6772998f3321c59bf5b349d1972ff Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 14 Nov 2017 21:02:47 +0100 Subject: [PATCH 09/10] Pass to :deploy also pipeline ID --- cmd/gitlab-pages-deployer/deploy.go | 6 +++++- cmd/gitlab-pages-deployer/worker.go | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/gitlab-pages-deployer/deploy.go b/cmd/gitlab-pages-deployer/deploy.go index 0fd810ddd..f863f7308 100644 --- a/cmd/gitlab-pages-deployer/deploy.go +++ b/cmd/gitlab-pages-deployer/deploy.go @@ -4,7 +4,11 @@ import ( workers "github.com/jrallison/go-workers" ) -func deployJob(message *workers.Msg, projectID int64, projectPath string, jobID int64, config map[string]interface{}) { +func deployJob(message *workers.Msg, projectID int64, projectPath string, pipelineID int64, jobID int64, config map[string]interface{}) { + // TODO: Implement extracting the archive + // that is under projectID+jobID + // Send CommitStatus to Pipeline for given JobID + saveConfig(projectID, projectPath, config) } diff --git a/cmd/gitlab-pages-deployer/worker.go b/cmd/gitlab-pages-deployer/worker.go index 41140e2b2..a50c1a11e 100644 --- a/cmd/gitlab-pages-deployer/worker.go +++ b/cmd/gitlab-pages-deployer/worker.go @@ -18,7 +18,8 @@ func pagesJob(message *workers.Msg) { args.GetIndex(1).MustInt64(), args.GetIndex(2).MustString(), args.GetIndex(3).MustInt64(), - args.GetIndex(4).MustMap(), + args.GetIndex(4).MustInt64(), + args.GetIndex(5).MustMap(), ) case "remove": -- GitLab From bba79f32d263531b83a98c7216066ee6bde019ea Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Wed, 15 Nov 2017 18:32:04 +0100 Subject: [PATCH 10/10] Implement deployment procedure --- cmd/gitlab-pages-deployer/api.go | 33 +++++ cmd/gitlab-pages-deployer/config.go | 2 +- cmd/gitlab-pages-deployer/deploy.go | 188 +++++++++++++++++++++++++- cmd/gitlab-pages-deployer/main.go | 6 + cmd/gitlab-pages-deployer/transfer.go | 8 +- cmd/gitlab-pages-deployer/worker.go | 16 +-- 6 files changed, 235 insertions(+), 18 deletions(-) create mode 100644 cmd/gitlab-pages-deployer/api.go diff --git a/cmd/gitlab-pages-deployer/api.go b/cmd/gitlab-pages-deployer/api.go new file mode 100644 index 000000000..532d1fb2c --- /dev/null +++ b/cmd/gitlab-pages-deployer/api.go @@ -0,0 +1,33 @@ +package main + +import ( + "net/http" + + gitlab "github.com/xanzy/go-gitlab" +) + +func panicOnAPIError(resp *gitlab.Response, err error) { + if resp != nil { + panicOnHTTPError(resp.Response, err) + } else { + panicOnHTTPError(nil, err) + } +} + +func panicOnHTTPError(resp *http.Response, err error) { + if err == nil { + return + } + if resp != nil { + if resp.StatusCode/100 == 2 { + return + } + if resp.StatusCode == http.StatusNotFound { + return + } + if resp.StatusCode == http.StatusBadRequest { + return + } + } + panic(err) +} diff --git a/cmd/gitlab-pages-deployer/config.go b/cmd/gitlab-pages-deployer/config.go index 940526bcb..bec253d03 100644 --- a/cmd/gitlab-pages-deployer/config.go +++ b/cmd/gitlab-pages-deployer/config.go @@ -15,7 +15,7 @@ func panicOnError(err error) { } } -func saveConfig(projectID int64, projectPath string, config map[string]interface{}) { +func saveConfig(projectID int, projectPath string, config map[string]interface{}) { data, err := json.MarshalIndent(config, "", "\t") panicOnError(err) diff --git a/cmd/gitlab-pages-deployer/deploy.go b/cmd/gitlab-pages-deployer/deploy.go index f863f7308..7ead3a35e 100644 --- a/cmd/gitlab-pages-deployer/deploy.go +++ b/cmd/gitlab-pages-deployer/deploy.go @@ -1,17 +1,195 @@ package main import ( + "archive/zip" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + + "github.com/jfbus/httprs" + "github.com/xanzy/go-gitlab" + workers "github.com/jrallison/go-workers" ) -func deployJob(message *workers.Msg, projectID int64, projectPath string, pipelineID int64, jobID int64, config map[string]interface{}) { - // TODO: Implement extracting the archive - // that is under projectID+jobID - // Send CommitStatus to Pipeline for given JobID +func writeFile(fullPath string, r io.Reader, size int64) error { + f, err := os.Create(fullPath) + if err != nil { + return err + } + defer f.Close() + + n, err := io.Copy(f, r) + if err != nil { + return err + } + + if n != size { + return fmt.Errorf("%s: readed only %d instead of %d", fullPath, n, size) + } + return nil +} + +func extractFile(file *zip.File, dir string) error { + fullPath := filepath.Join(dir, file.Name) + println(fullPath, "size=", file.UncompressedSize64) + + rc, err := file.Open() + if err != nil { + return err + } + defer rc.Close() + + dirPath := filepath.Dir(fullPath) + "/" + os.MkdirAll(dirPath, 0750) + + if file.Mode().IsRegular() { + return writeFile(fullPath, rc, int64(file.UncompressedSize64)) + } + + return nil +} + +func extractArtifactsArchive(tempDir string, artifactsURL string) error { + req, err := http.NewRequest("GET", artifactsURL, nil) + if err != nil { + return err + } + + req.Header.Set("PRIVATE-TOKEN", *apiAccessToken) + + resp, err := http.DefaultClient.Do(req) + panicOnHTTPError(resp, err) + + archiveStream := httprs.NewHttpReadSeeker(resp) + defer archiveStream.Close() + + reader, err := zip.NewReader(archiveStream, resp.ContentLength) + + for _, file := range reader.File { + // if !strings.HasPrefix(file.Name, "public/") { + // continue + // } + + err := extractFile(file, tempDir) + if err != nil { + return err + } + } + + return nil +} + +func extractJob(message *workers.Msg, projectID int, projectPath string, artifactsURL string) error { + tempDir := filepath.Join(*deployerRoot, "tmp") + os.MkdirAll(tempDir, 0750) + + targetPath := filepath.Join(*deployerRoot, projectPath) + os.MkdirAll(targetPath, 0750) + + // Create temp directory where we will store artifacts + deployTempDir, err := ioutil.TempDir(tempDir, "new_deploy") + if err != nil { + return err + } + defer os.RemoveAll(deployTempDir) + + // Extract archive to deployTempDir + err = extractArtifactsArchive(filepath.Join(deployTempDir, "public"), artifactsURL) + if err != nil { + return err + } + + oldTargetDir, err := ioutil.TempDir(targetPath, ".deleted") + if err != nil { + return err + } + defer os.RemoveAll(oldTargetDir) + + // Shuffle directories doing save "atomic" move + publicTempDir := filepath.Join(deployTempDir, "public") + publicTargetDir := filepath.Join(targetPath, "public") + publicOldTargetDir := filepath.Join(oldTargetDir, "public") + + fi, err := os.Stat(publicTempDir) + if err != nil { + if os.IsNotExist(err) { + return errors.New("public/ has to be present") + } + return err + } + if !fi.IsDir() { + return errors.New("public/ has to be directory") + } + + // Move old target public to a temporary public + err = os.Rename(publicTargetDir, publicOldTargetDir) + if err != nil && !os.IsNotExist(err) { + return err + } + + // Move new public to target public + err = os.Rename(publicTempDir, publicTargetDir) + if err != nil { + // If this fails, try to bring back old target dir + os.Rename(publicOldTargetDir, publicTargetDir) + return err + } + + // Use defer to remove: publicOldTarget and deployTempDir + return nil +} + +func deployJob(message *workers.Msg, projectID int, projectPath string, pipelineID int, jobID int, config map[string]interface{}) { + pipeline, resp, err := api.Pipelines.GetPipeline(projectID, pipelineID) + panicOnAPIError(resp, err) + + if pipeline == nil { + return + } + + println("Pipeline", projectPath, pipeline.ID, pipeline.Sha, pipeline.Ref) + + _, resp, err = api.Commits.SetCommitStatus(projectID, pipeline.Sha, &gitlab.SetCommitStatusOptions{ + State: gitlab.Running, + Ref: &pipeline.Ref, + Name: gitlab.String("pages:deploy"), + Description: gitlab.String("started deploying"), + }) + panicOnAPIError(resp, err) + + err = extractJob(message, projectID, projectPath, + fmt.Sprint(*apiURL, "/projects/", projectID, "/jobs/", jobID, "/artifacts")) + + if err != nil { + println("Pipeline", pipeline.ID, pipeline.Sha, pipeline.Ref, err.Error()) + _, resp, err = api.Commits.SetCommitStatus(projectID, pipeline.Sha, &gitlab.SetCommitStatusOptions{ + State: gitlab.Failed, + Ref: &pipeline.Ref, + Name: gitlab.String("pages:deploy"), + Description: gitlab.String(err.Error()), + }) + panicOnAPIError(resp, err) + return + } + + println("Pipeline", pipeline.ID, pipeline.Sha, pipeline.Ref, "SUCCESS") saveConfig(projectID, projectPath, config) + + _, resp, err = api.Commits.SetCommitStatus(projectID, pipeline.Sha, &gitlab.SetCommitStatusOptions{ + State: gitlab.Success, + Ref: &pipeline.Ref, + Name: gitlab.String("pages:deploy"), + Description: gitlab.String("deployed"), + }) + panicOnAPIError(resp, err) } -func configJob(message *workers.Msg, projectID int64, projectPath string, config map[string]interface{}) { +func configJob(message *workers.Msg, projectID int, projectPath string, config map[string]interface{}) { saveConfig(projectID, projectPath, config) } diff --git a/cmd/gitlab-pages-deployer/main.go b/cmd/gitlab-pages-deployer/main.go index 329b172b8..871083b6c 100644 --- a/cmd/gitlab-pages-deployer/main.go +++ b/cmd/gitlab-pages-deployer/main.go @@ -9,6 +9,7 @@ import ( "strconv" workers "github.com/jrallison/go-workers" + gitlab "github.com/xanzy/go-gitlab" ) // VERSION stores the information about the semantic version of application @@ -17,6 +18,8 @@ var VERSION = "dev" // REVISION stores the information about the git revision of application var REVISION = "HEAD" +var api *gitlab.Client + var ( showVersion = flag.Bool("version", false, "Show version") @@ -56,6 +59,9 @@ func appMain() { printVersion(*showVersion, VERSION) + api = gitlab.NewClient(nil, *apiAccessToken) + api.SetBaseURL(*apiURL) + log.Printf("GitLab Pages Deployer %s (%s)", VERSION, REVISION) log.Printf("URL: https://gitlab.com/gitlab-org/gitlab-pages\n") diff --git a/cmd/gitlab-pages-deployer/transfer.go b/cmd/gitlab-pages-deployer/transfer.go index b2722f097..279127f69 100644 --- a/cmd/gitlab-pages-deployer/transfer.go +++ b/cmd/gitlab-pages-deployer/transfer.go @@ -13,21 +13,21 @@ func panicOnFileSystemError(err error) { } } -func renameNamespaceJob(message *workers.Msg, projectID int64, fullPathWas, fullPath string) { +func renameNamespaceJob(message *workers.Msg, projectID int, fullPathWas, fullPath string) { err := os.Rename(filepath.Join(*deployerRoot, fullPathWas), filepath.Join(*deployerRoot, fullPath)) panicOnFileSystemError(err) touchDaemon() } -func renameProjectJob(message *workers.Msg, projectID int64, pathWas, path, fullPath string) { +func renameProjectJob(message *workers.Msg, projectID int, pathWas, path, fullPath string) { err := os.Rename(filepath.Join(*deployerRoot, fullPath, pathWas), filepath.Join(*deployerRoot, fullPath, path)) panicOnFileSystemError(err) touchDaemon() } -func moveProjectJob(message *workers.Msg, projectID int64, path, fullPathWas, fullPath string) { +func moveProjectJob(message *workers.Msg, projectID int, path, fullPathWas, fullPath string) { err := os.MkdirAll(filepath.Join(*deployerRoot, fullPath), 0750) panicOnFileSystemError(err) @@ -37,7 +37,7 @@ func moveProjectJob(message *workers.Msg, projectID int64, path, fullPathWas, fu touchDaemon() } -func removeJob(message *workers.Msg, projectID int64, namespacePath, path string) { +func removeJob(message *workers.Msg, projectID int, namespacePath, path string) { err := os.RemoveAll(filepath.Join(*deployerRoot, namespacePath, path)) panicOnFileSystemError(err) diff --git a/cmd/gitlab-pages-deployer/worker.go b/cmd/gitlab-pages-deployer/worker.go index a50c1a11e..e8c1af758 100644 --- a/cmd/gitlab-pages-deployer/worker.go +++ b/cmd/gitlab-pages-deployer/worker.go @@ -15,17 +15,17 @@ func pagesJob(message *workers.Msg) { case "deploy": deployJob( message, - args.GetIndex(1).MustInt64(), + args.GetIndex(1).MustInt(), args.GetIndex(2).MustString(), - args.GetIndex(3).MustInt64(), - args.GetIndex(4).MustInt64(), + args.GetIndex(3).MustInt(), + args.GetIndex(4).MustInt(), args.GetIndex(5).MustMap(), ) case "remove": removeJob( message, - args.GetIndex(1).MustInt64(), + args.GetIndex(1).MustInt(), args.GetIndex(2).MustString(), args.GetIndex(3).MustString(), ) @@ -33,7 +33,7 @@ func pagesJob(message *workers.Msg) { case "config": configJob( message, - args.GetIndex(1).MustInt64(), + args.GetIndex(1).MustInt(), args.GetIndex(2).MustString(), args.GetIndex(3).MustMap(), ) @@ -41,7 +41,7 @@ func pagesJob(message *workers.Msg) { case "rename_namespace": renameNamespaceJob( message, - args.GetIndex(1).MustInt64(), + args.GetIndex(1).MustInt(), args.GetIndex(2).MustString(), args.GetIndex(3).MustString(), ) @@ -49,7 +49,7 @@ func pagesJob(message *workers.Msg) { case "rename_project": renameProjectJob( message, - args.GetIndex(1).MustInt64(), + args.GetIndex(1).MustInt(), args.GetIndex(2).MustString(), args.GetIndex(3).MustString(), args.GetIndex(4).MustString(), @@ -58,7 +58,7 @@ func pagesJob(message *workers.Msg) { case "move_project": moveProjectJob( message, - args.GetIndex(1).MustInt64(), + args.GetIndex(1).MustInt(), args.GetIndex(2).MustString(), args.GetIndex(3).MustString(), args.GetIndex(4).MustString(), -- GitLab