From 34a8f4960340472ee906c640c503db5a48f012e8 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 9 May 2019 21:16:54 -0500 Subject: [PATCH 1/9] Sketch gitaly bundle snapshots --- cmd/gitaly-bundle-apply/main.go | 80 +++++++++++++++++++++++++++++++++ gitaly-bundle-create | 13 ++++++ 2 files changed, 93 insertions(+) create mode 100644 cmd/gitaly-bundle-apply/main.go create mode 100755 gitaly-bundle-create diff --git a/cmd/gitaly-bundle-apply/main.go b/cmd/gitaly-bundle-apply/main.go new file mode 100644 index 00000000000..67064ac8fd8 --- /dev/null +++ b/cmd/gitaly-bundle-apply/main.go @@ -0,0 +1,80 @@ +package main + +import ( + "bufio" + "bytes" + "fmt" + "log" + "os" + "os/exec" + "strings" +) + +func main() { + if err := _main(); err != nil { + log.Fatal(err) + } +} + +func _main() error { + r := bufio.NewReader(os.Stdin) + + const header = "#gitaly bundle v1" + line, err := readLine(r) + if err != nil { + return err + } + if line != header { + return fmt.Errorf("invalid header %q", line) + } + + refUpdate := &bytes.Buffer{} + for { + line, err := readLine(r) + if err != nil { + return err + } + if line == "" { + break + } + + split := strings.SplitN(line, " ", 2) + if len(split) != 2 { + return fmt.Errorf("invalid ref line %q", line) + } + if _, err := fmt.Fprintf(refUpdate, "update %s %s\n", split[1], split[0]); err != nil { + return err + } + } + + indexPack := exec.Command("git", "index-pack", "--stdin", "--fix-thin") + indexPack.Stdin = r + indexPack.Stdout = os.Stderr + indexPack.Stderr = os.Stderr + + if err := indexPack.Start(); err != nil { + return err + } + if err := indexPack.Wait(); err != nil { + return err + } + + updateRef := exec.Command("git", "update-ref", "--stdin") + updateRef.Stdin = refUpdate + updateRef.Stdout = os.Stderr + updateRef.Stderr = os.Stderr + + if err := updateRef.Start(); err != nil { + return err + } + if err := updateRef.Wait(); err != nil { + return err + } + + return nil +} + +func readLine(r *bufio.Reader) (string, error) { + b, err := r.ReadBytes('\n') + return string(b)[:len(b)-1], err +} diff --git a/gitaly-bundle-create b/gitaly-bundle-create new file mode 100755 index 00000000000..467a1326715 --- /dev/null +++ b/gitaly-bundle-create @@ -0,0 +1,13 @@ +#!/bin/sh + +set -ex + +ref_snapshot=$(mktemp -t gitaly-create-bundle.XXXXXX) +echo '# gitaly bundle v1' +git for-each-ref --format='%(objectname) %(refname)' | tee "${ref_snapshot}" +echo +( + awk '{print $1}' "${ref_snapshot}" + rm "${ref_snapshot}" + awk '/^$/ { exit } /^[^#]/ { print "^" $1 }' +) | git pack-objects --thin --stdout --non-empty --delta-base-offset -- GitLab From 364afafabb5fbf6104995d6686f0fca94e12d948 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 9 May 2019 21:19:56 -0500 Subject: [PATCH 2/9] space --- cmd/gitaly-bundle-apply/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/gitaly-bundle-apply/main.go b/cmd/gitaly-bundle-apply/main.go index 67064ac8fd8..5d43c671336 100644 --- a/cmd/gitaly-bundle-apply/main.go +++ b/cmd/gitaly-bundle-apply/main.go @@ -19,7 +19,7 @@ func main() { func _main() error { r := bufio.NewReader(os.Stdin) - const header = "#gitaly bundle v1" + const header = "# gitaly bundle v1" line, err := readLine(r) if err != nil { return err -- GitLab From 4fdef01163df2c67240c34045a7780cce9b46e26 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 9 May 2019 21:20:30 -0500 Subject: [PATCH 3/9] gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 24aefa7e671..4fe95707856 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ git-env /gitaly-debug /praefect gitaly.pid +/gitaly-bundle-apply -- GitLab From 407eebb2e2cee5959ef7ea5a951f6960b5a47312 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 9 May 2019 21:27:35 -0500 Subject: [PATCH 4/9] Use Run() --- cmd/gitaly-bundle-apply/main.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cmd/gitaly-bundle-apply/main.go b/cmd/gitaly-bundle-apply/main.go index 5d43c671336..c37104fc37f 100644 --- a/cmd/gitaly-bundle-apply/main.go +++ b/cmd/gitaly-bundle-apply/main.go @@ -52,10 +52,7 @@ func _main() error { indexPack.Stdout = os.Stderr indexPack.Stderr = os.Stderr - if err := indexPack.Start(); err != nil { - return err - } - if err := indexPack.Wait(); err != nil { + if err := indexPack.Run(); err != nil { return err } @@ -64,10 +61,7 @@ func _main() error { updateRef.Stdout = os.Stderr updateRef.Stderr = os.Stderr - if err := updateRef.Start(); err != nil { - return err - } - if err := updateRef.Wait(); err != nil { + if err := updateRef.Run(); err != nil { return err } -- GitLab From 1ad6c45a2575b5d1f70551437f9e97abd638db63 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 10 May 2019 10:00:40 -0500 Subject: [PATCH 5/9] Handle deletions, add comments --- cmd/gitaly-bundle-apply/main.go | 92 +++++++++++++++++++++++++-------- gitaly-bundle-create | 10 ++++ 2 files changed, 81 insertions(+), 21 deletions(-) diff --git a/cmd/gitaly-bundle-apply/main.go b/cmd/gitaly-bundle-apply/main.go index c37104fc37f..6b38066db0a 100644 --- a/cmd/gitaly-bundle-apply/main.go +++ b/cmd/gitaly-bundle-apply/main.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "fmt" + "io" "log" "os" "os/exec" @@ -19,43 +20,52 @@ func main() { func _main() error { r := bufio.NewReader(os.Stdin) - const header = "# gitaly bundle v1" line, err := readLine(r) if err != nil { return err } + + const header = "# gitaly bundle v1" if line != header { return fmt.Errorf("invalid header %q", line) } - refUpdate := &bytes.Buffer{} - for { - line, err := readLine(r) - if err != nil { - return err - } - if line == "" { - break - } + // Ref dump is at the head of the bundle so we must consume it first + newRefs, err := refMapFromReader(r) + if err != nil { + return err + } - split := strings.SplitN(line, " ", 2) - if len(split) != 2 { - return fmt.Errorf("invalid ref line %q", line) - } - if _, err := fmt.Fprintf(refUpdate, "update %s %s\n", split[1], split[0]); err != nil { + // There may not be a pack file, for instance if the only change is a ref deletion. + const magic = "PACK" + if b, err := r.Peek(len(magic)); err == nil && string(b) == magic { + indexPack := exec.Command("git", "index-pack", "--stdin", "--fix-thin") + indexPack.Stdin = r + indexPack.Stdout = os.Stderr + indexPack.Stderr = os.Stderr + + if err := indexPack.Run(); err != nil { return err } } - indexPack := exec.Command("git", "index-pack", "--stdin", "--fix-thin") - indexPack.Stdin = r - indexPack.Stdout = os.Stderr - indexPack.Stderr = os.Stderr - - if err := indexPack.Run(); err != nil { + oldRefs, err := currentRefs() + if err != nil { return err } + refUpdate := &bytes.Buffer{} + for ref, oid := range newRefs { + fmt.Fprintf(refUpdate, "update %s %s\n", ref, oid) + } + + for ref, _ := range oldRefs { + if _, ok := newRefs[ref]; ok { + continue + } + fmt.Fprintf(refUpdate, "delete %s\n", ref) + } + updateRef := exec.Command("git", "update-ref", "--stdin") updateRef.Stdin = refUpdate updateRef.Stdout = os.Stderr @@ -70,5 +80,45 @@ func _main() error { func readLine(r *bufio.Reader) (string, error) { b, err := r.ReadBytes('\n') + if err != nil { + return "", err + } + return string(b)[:len(b)-1], err } + +func currentRefs() (map[string]string, error) { + cmd := exec.Command("git", "for-each-ref", "--format=%(objectname) %(refname)") + cmd.Stderr = os.Stderr + out, err := cmd.Output() + if err != nil { + return nil, err + } + + refs, err := refMapFromReader(bufio.NewReader(bytes.NewReader(out))) + if err == io.EOF { + err = nil + } + + return refs, err +} + +func refMapFromReader(r *bufio.Reader) (map[string]string, error) { + result := make(map[string]string) + var err error + + for { + line, err := readLine(r) + if err != nil || line == "" { + break + } + + split := strings.SplitN(line, " ", 2) + if len(split) != 2 { + return nil, fmt.Errorf("invalid ref line %q", line) + } + result[split[1]] = split[0] + } + + return result, err +} diff --git a/gitaly-bundle-create b/gitaly-bundle-create index 467a1326715..875b231f7e2 100755 --- a/gitaly-bundle-create +++ b/gitaly-bundle-create @@ -3,11 +3,21 @@ set -ex ref_snapshot=$(mktemp -t gitaly-create-bundle.XXXXXX) + +# Header echo '# gitaly bundle v1' + +# Ref dump git for-each-ref --format='%(objectname) %(refname)' | tee "${ref_snapshot}" echo + ( + # Tell pack-objects the objects we want to include awk '{print $1}' "${ref_snapshot}" rm "${ref_snapshot}" + + # Use the ref dump from the previous snapshot to tell pack-objects which + # objects we already have. The previous snapshot may be empty + # (/dev/null). awk '/^$/ { exit } /^[^#]/ { print "^" $1 }' ) | git pack-objects --thin --stdout --non-empty --delta-base-offset -- GitLab From 224e8ec521cd7f3bde430833ad5842fb273bb22e Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 10 May 2019 10:06:25 -0500 Subject: [PATCH 6/9] More comments --- cmd/gitaly-bundle-apply/main.go | 2 ++ gitaly-bundle-create | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/gitaly-bundle-apply/main.go b/cmd/gitaly-bundle-apply/main.go index 6b38066db0a..31a61a43e9d 100644 --- a/cmd/gitaly-bundle-apply/main.go +++ b/cmd/gitaly-bundle-apply/main.go @@ -39,6 +39,8 @@ func _main() error { // There may not be a pack file, for instance if the only change is a ref deletion. const magic = "PACK" if b, err := r.Peek(len(magic)); err == nil && string(b) == magic { + // TODO use --keep to prevent concurrent repack/gc from deleting this + // packfile before we apply the ref update below? indexPack := exec.Command("git", "index-pack", "--stdin", "--fix-thin") indexPack.Stdin = r indexPack.Stdout = os.Stderr diff --git a/gitaly-bundle-create b/gitaly-bundle-create index 875b231f7e2..f48466f62db 100755 --- a/gitaly-bundle-create +++ b/gitaly-bundle-create @@ -2,7 +2,7 @@ set -ex -ref_snapshot=$(mktemp -t gitaly-create-bundle.XXXXXX) +ref_snapshot=$(mktemp -t gitaly-bundle-create.XXXXXX) # Header echo '# gitaly bundle v1' @@ -16,8 +16,9 @@ echo awk '{print $1}' "${ref_snapshot}" rm "${ref_snapshot}" - # Use the ref dump from the previous snapshot to tell pack-objects which - # objects we already have. The previous snapshot may be empty - # (/dev/null). + # The previous snapshot should be on stdin. Use it to tell pack-objects + # which objects we already have. The previous snapshot may be empty + # (/dev/null). This awk command will only consume the ref dump part of + # stdin; pack data is not read. awk '/^$/ { exit } /^[^#]/ { print "^" $1 }' ) | git pack-objects --thin --stdout --non-empty --delta-base-offset -- GitLab From 7fe0cadec42f41cdaf2c8a84fde1b4ae522582c7 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 10 May 2019 15:37:13 -0500 Subject: [PATCH 7/9] Remove unnecessary _ --- cmd/gitaly-bundle-apply/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/gitaly-bundle-apply/main.go b/cmd/gitaly-bundle-apply/main.go index 31a61a43e9d..132caa0d7ac 100644 --- a/cmd/gitaly-bundle-apply/main.go +++ b/cmd/gitaly-bundle-apply/main.go @@ -61,7 +61,7 @@ func _main() error { fmt.Fprintf(refUpdate, "update %s %s\n", ref, oid) } - for ref, _ := range oldRefs { + for ref := range oldRefs { if _, ok := newRefs[ref]; ok { continue } -- GitLab From 79e3e78fddc3a4f4e23bd3ba26a8d779d5fd8053 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Sun, 12 May 2019 10:07:14 -0500 Subject: [PATCH 8/9] Handle missing prerequisite objects --- gitaly-bundle-create | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gitaly-bundle-create b/gitaly-bundle-create index f48466f62db..6e7ea0c53c8 100755 --- a/gitaly-bundle-create +++ b/gitaly-bundle-create @@ -20,5 +20,8 @@ echo # which objects we already have. The previous snapshot may be empty # (/dev/null). This awk command will only consume the ref dump part of # stdin; pack data is not read. - awk '/^$/ { exit } /^[^#]/ { print "^" $1 }' + awk '/^$/ { exit } /^[^#]/ { print $1 }' |\ + git cat-file --batch-check='%(objectname)' |\ + grep -v ' missing$' |\ + sed 's/^/^/' ) | git pack-objects --thin --stdout --non-empty --delta-base-offset -- GitLab From c617df3ceaaa05292d0907c42ce1cdfb894a71f4 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Sun, 12 May 2019 20:34:02 -0500 Subject: [PATCH 9/9] Replace grep/sed with awk --- gitaly-bundle-create | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gitaly-bundle-create b/gitaly-bundle-create index 6e7ea0c53c8..e95b0d1cdae 100755 --- a/gitaly-bundle-create +++ b/gitaly-bundle-create @@ -22,6 +22,5 @@ echo # stdin; pack data is not read. awk '/^$/ { exit } /^[^#]/ { print $1 }' |\ git cat-file --batch-check='%(objectname)' |\ - grep -v ' missing$' |\ - sed 's/^/^/' + awk '!/ missing$/ { print "^" $1 }' ) | git pack-objects --thin --stdout --non-empty --delta-base-offset -- GitLab