diff --git a/.gitignore b/.gitignore index 1e0ddeed281576740ba3cdbbecb942517fa69911..41c6bf51747256e6b3794d3b6932abc8a3e34ff9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Created by .ignore support plugin (hsz.mobi) shared/pages/.update /gitlab-pages +/gitlab-pages-deployer # Used by the makefile /.GOPATH diff --git a/cmd/gitlab-pages-deployer/api.go b/cmd/gitlab-pages-deployer/api.go new file mode 100644 index 0000000000000000000000000000000000000000..532d1fb2c137d9e0c439bff7544dee346c2a0baf --- /dev/null +++ b/cmd/gitlab-pages-deployer/api.go @@ -0,0 +1,33 @@ +package main + +import ( + "net/http" + + gitlab "github.com/xanzy/go-gitlab" +) + +func panicOnAPIError(resp *gitlab.Response, err error) { + if resp != nil { + panicOnHTTPError(resp.Response, err) + } else { + panicOnHTTPError(nil, err) + } +} + +func panicOnHTTPError(resp *http.Response, err error) { + if err == nil { + return + } + if resp != nil { + if resp.StatusCode/100 == 2 { + return + } + if resp.StatusCode == http.StatusNotFound { + return + } + if resp.StatusCode == http.StatusBadRequest { + return + } + } + panic(err) +} diff --git a/cmd/gitlab-pages-deployer/config.go b/cmd/gitlab-pages-deployer/config.go new file mode 100644 index 0000000000000000000000000000000000000000..bec253d03721c1ca45883ca5b31e24e4ee7b544f --- /dev/null +++ b/cmd/gitlab-pages-deployer/config.go @@ -0,0 +1,57 @@ +package main + +import ( + "bytes" + "crypto/rand" + "encoding/json" + "io/ioutil" + "os" + "path/filepath" +) + +func panicOnError(err error) { + if err != nil { + panic(err) + } +} + +func saveConfig(projectID int, projectPath string, config map[string]interface{}) { + data, err := json.MarshalIndent(config, "", "\t") + panicOnError(err) + + if replaceFile(filepath.Join(*deployerRoot, projectPath), "config.json", data) { + touchDaemon() + } +} + +func replaceFile(path, filename string, data []byte) bool { + targetFile := filepath.Join(path, filename) + + err := os.MkdirAll(path, 0750) + panicOnFileSystemError(err) + + dataWas, err := ioutil.ReadFile(targetFile) + if err == nil && bytes.Equal(data, dataWas) { + return false + } + + f, err := ioutil.TempFile(path, "config") + panicOnError(err) + defer f.Close() + defer os.Remove(f.Name()) + + _, err = f.Write(data) + panicOnError(err) + + err = os.Rename(f.Name(), targetFile) + panicOnFileSystemError(err) + return true +} + +func touchDaemon() { + randomData := make([]byte, 32) + _, err := rand.Read(randomData) + panicOnError(err) + + replaceFile(*deployerRoot, ".update", randomData) +} diff --git a/cmd/gitlab-pages-deployer/deploy.go b/cmd/gitlab-pages-deployer/deploy.go new file mode 100644 index 0000000000000000000000000000000000000000..7ead3a35e0b1749db7a79a727ace0fff1abad4af --- /dev/null +++ b/cmd/gitlab-pages-deployer/deploy.go @@ -0,0 +1,195 @@ +package main + +import ( + "archive/zip" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + + "github.com/jfbus/httprs" + "github.com/xanzy/go-gitlab" + + workers "github.com/jrallison/go-workers" +) + +func writeFile(fullPath string, r io.Reader, size int64) error { + f, err := os.Create(fullPath) + if err != nil { + return err + } + defer f.Close() + + n, err := io.Copy(f, r) + if err != nil { + return err + } + + if n != size { + return fmt.Errorf("%s: readed only %d instead of %d", fullPath, n, size) + } + return nil +} + +func extractFile(file *zip.File, dir string) error { + fullPath := filepath.Join(dir, file.Name) + println(fullPath, "size=", file.UncompressedSize64) + + rc, err := file.Open() + if err != nil { + return err + } + defer rc.Close() + + dirPath := filepath.Dir(fullPath) + "/" + os.MkdirAll(dirPath, 0750) + + if file.Mode().IsRegular() { + return writeFile(fullPath, rc, int64(file.UncompressedSize64)) + } + + return nil +} + +func extractArtifactsArchive(tempDir string, artifactsURL string) error { + req, err := http.NewRequest("GET", artifactsURL, nil) + if err != nil { + return err + } + + req.Header.Set("PRIVATE-TOKEN", *apiAccessToken) + + resp, err := http.DefaultClient.Do(req) + panicOnHTTPError(resp, err) + + archiveStream := httprs.NewHttpReadSeeker(resp) + defer archiveStream.Close() + + reader, err := zip.NewReader(archiveStream, resp.ContentLength) + + for _, file := range reader.File { + // if !strings.HasPrefix(file.Name, "public/") { + // continue + // } + + err := extractFile(file, tempDir) + if err != nil { + return err + } + } + + return nil +} + +func extractJob(message *workers.Msg, projectID int, projectPath string, artifactsURL string) error { + tempDir := filepath.Join(*deployerRoot, "tmp") + os.MkdirAll(tempDir, 0750) + + targetPath := filepath.Join(*deployerRoot, projectPath) + os.MkdirAll(targetPath, 0750) + + // Create temp directory where we will store artifacts + deployTempDir, err := ioutil.TempDir(tempDir, "new_deploy") + if err != nil { + return err + } + defer os.RemoveAll(deployTempDir) + + // Extract archive to deployTempDir + err = extractArtifactsArchive(filepath.Join(deployTempDir, "public"), artifactsURL) + if err != nil { + return err + } + + oldTargetDir, err := ioutil.TempDir(targetPath, ".deleted") + if err != nil { + return err + } + defer os.RemoveAll(oldTargetDir) + + // Shuffle directories doing save "atomic" move + publicTempDir := filepath.Join(deployTempDir, "public") + publicTargetDir := filepath.Join(targetPath, "public") + publicOldTargetDir := filepath.Join(oldTargetDir, "public") + + fi, err := os.Stat(publicTempDir) + if err != nil { + if os.IsNotExist(err) { + return errors.New("public/ has to be present") + } + return err + } + if !fi.IsDir() { + return errors.New("public/ has to be directory") + } + + // Move old target public to a temporary public + err = os.Rename(publicTargetDir, publicOldTargetDir) + if err != nil && !os.IsNotExist(err) { + return err + } + + // Move new public to target public + err = os.Rename(publicTempDir, publicTargetDir) + if err != nil { + // If this fails, try to bring back old target dir + os.Rename(publicOldTargetDir, publicTargetDir) + return err + } + + // Use defer to remove: publicOldTarget and deployTempDir + return nil +} + +func deployJob(message *workers.Msg, projectID int, projectPath string, pipelineID int, jobID int, config map[string]interface{}) { + pipeline, resp, err := api.Pipelines.GetPipeline(projectID, pipelineID) + panicOnAPIError(resp, err) + + if pipeline == nil { + return + } + + println("Pipeline", projectPath, pipeline.ID, pipeline.Sha, pipeline.Ref) + + _, resp, err = api.Commits.SetCommitStatus(projectID, pipeline.Sha, &gitlab.SetCommitStatusOptions{ + State: gitlab.Running, + Ref: &pipeline.Ref, + Name: gitlab.String("pages:deploy"), + Description: gitlab.String("started deploying"), + }) + panicOnAPIError(resp, err) + + err = extractJob(message, projectID, projectPath, + fmt.Sprint(*apiURL, "/projects/", projectID, "/jobs/", jobID, "/artifacts")) + + if err != nil { + println("Pipeline", pipeline.ID, pipeline.Sha, pipeline.Ref, err.Error()) + _, resp, err = api.Commits.SetCommitStatus(projectID, pipeline.Sha, &gitlab.SetCommitStatusOptions{ + State: gitlab.Failed, + Ref: &pipeline.Ref, + Name: gitlab.String("pages:deploy"), + Description: gitlab.String(err.Error()), + }) + panicOnAPIError(resp, err) + return + } + + println("Pipeline", pipeline.ID, pipeline.Sha, pipeline.Ref, "SUCCESS") + + saveConfig(projectID, projectPath, config) + + _, resp, err = api.Commits.SetCommitStatus(projectID, pipeline.Sha, &gitlab.SetCommitStatusOptions{ + State: gitlab.Success, + Ref: &pipeline.Ref, + Name: gitlab.String("pages:deploy"), + Description: gitlab.String("deployed"), + }) + panicOnAPIError(resp, err) +} + +func configJob(message *workers.Msg, projectID int, projectPath string, config map[string]interface{}) { + saveConfig(projectID, projectPath, config) +} diff --git a/cmd/gitlab-pages-deployer/main.go b/cmd/gitlab-pages-deployer/main.go new file mode 100644 index 0000000000000000000000000000000000000000..871083b6c616e2ee541a7af1c529ddf421460b96 --- /dev/null +++ b/cmd/gitlab-pages-deployer/main.go @@ -0,0 +1,96 @@ +package main + +import ( + "flag" + "fmt" + "log" + "net/http" + "os" + "strconv" + + workers "github.com/jrallison/go-workers" + gitlab "github.com/xanzy/go-gitlab" +) + +// VERSION stores the information about the semantic version of application +var VERSION = "dev" + +// REVISION stores the information about the git revision of application +var REVISION = "HEAD" + +var api *gitlab.Client + +var ( + showVersion = flag.Bool("version", false, "Show version") + + apiURL = flag.String("api-url", "https://gitlab.com/api/v4", "The API URL to GitLab") + apiAccessToken = flag.String("api-access-token", "", "API Access Token to post back the statuses of Pages") + + statsServer = flag.String("stats-server", "localhost:9818", "Address to statistics server") + + deployerRoot = flag.String("deployer-root", "shared/pages", "The directory where pages are stored") + deployerMaximumSize = flag.Int("deployer-maximum-size", 1024, "The maximum size of artifacts extracted (in Bytes)") + deployerConcurrency = flag.Int("deployer-concurrency", 10, "The maximum concurrency") + deployerID = flag.String("deployer-id", "1", "Unique ID of Deployer") + + redisServer = flag.String("redis-server", "localhost:6379", "The address of Redis Server") + redisDatabase = flag.String("redis-database", "0", "The name of Redis Database") + redisPassword = flag.String("redis-password", "", "The password to Redis Database") + redisNamespace = flag.String("redis-namespace", "resque:gitlab", "The namespace to use") + redisPool = flag.Int("redis-pool", 10, "The connection pool to Redis Database") +) + +func runStatsServer() { + http.HandleFunc("/stats", workers.Stats) + + if *statsServer != "" { + log.Println("Stats are available at", fmt.Sprint(*statsServer, "/stats")) + + go func() { + if err := http.ListenAndServe(*statsServer, nil); err != nil { + log.Println(err) + } + }() + } +} + +func appMain() { + flag.Parse() + + printVersion(*showVersion, VERSION) + + api = gitlab.NewClient(nil, *apiAccessToken) + api.SetBaseURL(*apiURL) + + log.Printf("GitLab Pages Deployer %s (%s)", VERSION, REVISION) + log.Printf("URL: https://gitlab.com/gitlab-org/gitlab-pages\n") + + runStatsServer() + + workers.Configure(map[string]string{ + "server": *redisServer, + "database": *redisDatabase, + "password": *redisPassword, + "namespace": *redisNamespace, + "pool": strconv.Itoa(*redisPool), + "process": *deployerID, + }) + + workers.Middleware.Append(&myMiddleware{}) + workers.Process("pages", pagesJob, *deployerConcurrency) + workers.Run() +} + +func printVersion(showVersion bool, version string) { + if showVersion { + log.SetFlags(0) + log.Printf(version) + os.Exit(0) + } +} + +func main() { + log.SetOutput(os.Stderr) + + appMain() +} diff --git a/cmd/gitlab-pages-deployer/middleware.go b/cmd/gitlab-pages-deployer/middleware.go new file mode 100644 index 0000000000000000000000000000000000000000..6e41d3ca1516fed11f73ab7993ccdfd9d964c37f --- /dev/null +++ b/cmd/gitlab-pages-deployer/middleware.go @@ -0,0 +1,12 @@ +package main + +import workers "github.com/jrallison/go-workers" + +type myMiddleware struct{} + +func (r *myMiddleware) Call(queue string, message *workers.Msg, next func() bool) (acknowledge bool) { + // do something before each message is processed + acknowledge = next() + // do something after each message is processed + return +} diff --git a/cmd/gitlab-pages-deployer/transfer.go b/cmd/gitlab-pages-deployer/transfer.go new file mode 100644 index 0000000000000000000000000000000000000000..279127f691497b0e7ef19684cf36891dad3da8b8 --- /dev/null +++ b/cmd/gitlab-pages-deployer/transfer.go @@ -0,0 +1,45 @@ +package main + +import ( + "os" + "path/filepath" + + workers "github.com/jrallison/go-workers" +) + +func panicOnFileSystemError(err error) { + if err != nil && !os.IsNotExist(err) { + panic(err) + } +} + +func renameNamespaceJob(message *workers.Msg, projectID int, fullPathWas, fullPath string) { + err := os.Rename(filepath.Join(*deployerRoot, fullPathWas), filepath.Join(*deployerRoot, fullPath)) + panicOnFileSystemError(err) + + touchDaemon() +} + +func renameProjectJob(message *workers.Msg, projectID int, pathWas, path, fullPath string) { + err := os.Rename(filepath.Join(*deployerRoot, fullPath, pathWas), filepath.Join(*deployerRoot, fullPath, path)) + panicOnFileSystemError(err) + + touchDaemon() +} + +func moveProjectJob(message *workers.Msg, projectID int, path, fullPathWas, fullPath string) { + err := os.MkdirAll(filepath.Join(*deployerRoot, fullPath), 0750) + panicOnFileSystemError(err) + + err = os.Rename(filepath.Join(*deployerRoot, fullPathWas, path), filepath.Join(*deployerRoot, fullPath, path)) + panicOnFileSystemError(err) + + touchDaemon() +} + +func removeJob(message *workers.Msg, projectID int, namespacePath, path string) { + err := os.RemoveAll(filepath.Join(*deployerRoot, namespacePath, path)) + panicOnFileSystemError(err) + + touchDaemon() +} diff --git a/cmd/gitlab-pages-deployer/worker.go b/cmd/gitlab-pages-deployer/worker.go new file mode 100644 index 0000000000000000000000000000000000000000..e8c1af758e27a542d3e7ccb4cb139bb4729ab250 --- /dev/null +++ b/cmd/gitlab-pages-deployer/worker.go @@ -0,0 +1,70 @@ +package main + +import ( + workers "github.com/jrallison/go-workers" +) + +func pagesJob(message *workers.Msg) { + if message.Get("class").MustString() != "PagesWorker" { + panic("Expected PagesWorker class: " + message.Get("class").MustString()) + } + + args := message.Args() + + switch args.GetIndex(0).MustString() { + case "deploy": + deployJob( + message, + args.GetIndex(1).MustInt(), + args.GetIndex(2).MustString(), + args.GetIndex(3).MustInt(), + args.GetIndex(4).MustInt(), + args.GetIndex(5).MustMap(), + ) + + case "remove": + removeJob( + message, + args.GetIndex(1).MustInt(), + args.GetIndex(2).MustString(), + args.GetIndex(3).MustString(), + ) + + case "config": + configJob( + message, + args.GetIndex(1).MustInt(), + args.GetIndex(2).MustString(), + args.GetIndex(3).MustMap(), + ) + + case "rename_namespace": + renameNamespaceJob( + message, + args.GetIndex(1).MustInt(), + args.GetIndex(2).MustString(), + args.GetIndex(3).MustString(), + ) + + case "rename_project": + renameProjectJob( + message, + args.GetIndex(1).MustInt(), + args.GetIndex(2).MustString(), + args.GetIndex(3).MustString(), + args.GetIndex(4).MustString(), + ) + + case "move_project": + moveProjectJob( + message, + args.GetIndex(1).MustInt(), + args.GetIndex(2).MustString(), + args.GetIndex(3).MustString(), + args.GetIndex(4).MustString(), + ) + + default: + panic("Unknown method: " + args.GetIndex(0).MustString()) + } +} diff --git a/acceptance_test.go b/cmd/gitlab-pages/acceptance_test.go similarity index 100% rename from acceptance_test.go rename to cmd/gitlab-pages/acceptance_test.go diff --git a/app.go b/cmd/gitlab-pages/app.go similarity index 100% rename from app.go rename to cmd/gitlab-pages/app.go diff --git a/app_config.go b/cmd/gitlab-pages/app_config.go similarity index 100% rename from app_config.go rename to cmd/gitlab-pages/app_config.go diff --git a/daemon.go b/cmd/gitlab-pages/daemon.go similarity index 100% rename from daemon.go rename to cmd/gitlab-pages/daemon.go diff --git a/domain.go b/cmd/gitlab-pages/domain.go similarity index 100% rename from domain.go rename to cmd/gitlab-pages/domain.go diff --git a/domain_config.go b/cmd/gitlab-pages/domain_config.go similarity index 100% rename from domain_config.go rename to cmd/gitlab-pages/domain_config.go diff --git a/domain_config_test.go b/cmd/gitlab-pages/domain_config_test.go similarity index 100% rename from domain_config_test.go rename to cmd/gitlab-pages/domain_config_test.go diff --git a/domain_test.go b/cmd/gitlab-pages/domain_test.go similarity index 100% rename from domain_test.go rename to cmd/gitlab-pages/domain_test.go diff --git a/domains.go b/cmd/gitlab-pages/domains.go similarity index 100% rename from domains.go rename to cmd/gitlab-pages/domains.go diff --git a/domains_test.go b/cmd/gitlab-pages/domains_test.go similarity index 100% rename from domains_test.go rename to cmd/gitlab-pages/domains_test.go diff --git a/helpers.go b/cmd/gitlab-pages/helpers.go similarity index 100% rename from helpers.go rename to cmd/gitlab-pages/helpers.go diff --git a/helpers_test.go b/cmd/gitlab-pages/helpers_test.go similarity index 100% rename from helpers_test.go rename to cmd/gitlab-pages/helpers_test.go diff --git a/logging.go b/cmd/gitlab-pages/logging.go similarity index 100% rename from logging.go rename to cmd/gitlab-pages/logging.go diff --git a/logging_test.go b/cmd/gitlab-pages/logging_test.go similarity index 100% rename from logging_test.go rename to cmd/gitlab-pages/logging_test.go diff --git a/main.go b/cmd/gitlab-pages/main.go similarity index 100% rename from main.go rename to cmd/gitlab-pages/main.go diff --git a/multi_string_flag.go b/cmd/gitlab-pages/multi_string_flag.go similarity index 100% rename from multi_string_flag.go rename to cmd/gitlab-pages/multi_string_flag.go diff --git a/multi_string_flag_test.go b/cmd/gitlab-pages/multi_string_flag_test.go similarity index 100% rename from multi_string_flag_test.go rename to cmd/gitlab-pages/multi_string_flag_test.go diff --git a/server.go b/cmd/gitlab-pages/server.go similarity index 100% rename from server.go rename to cmd/gitlab-pages/server.go diff --git a/shared/pages/h5bp/test/test/config.json b/shared/pages/h5bp/test/test/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ad702b32c0d94b29e08cb955c5f0cf3735639c3f --- /dev/null +++ b/shared/pages/h5bp/test/test/config.json @@ -0,0 +1,3 @@ +{ + "domains": [] +} \ No newline at end of file