From 863c22f0d35135f3d56c72e7529e9f598b55bf61 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sat, 21 Nov 2020 12:08:53 +0100
Subject: [PATCH 01/22] Pack-objects hook kind of works

---
 cmd/gitaly-hooks/hooks.go                       | 14 ++++++++++++++
 internal/git/hooks.go                           | 17 +++++++++++++++++
 internal/gitaly/service/register.go             |  1 +
 internal/gitaly/service/smarthttp/server.go     |  8 ++++++++
 .../gitaly/service/smarthttp/testhelper_test.go |  1 +
 .../gitaly/service/smarthttp/upload_pack.go     |  9 ++++++++-
 .../service/smarthttp/upload_pack_test.go       |  2 ++
 7 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/cmd/gitaly-hooks/hooks.go b/cmd/gitaly-hooks/hooks.go
index 2b30cf1425a..406af12f9cd 100644
--- a/cmd/gitaly-hooks/hooks.go
+++ b/cmd/gitaly-hooks/hooks.go
@@ -7,6 +7,7 @@ import (
 	"io"
 	"log"
 	"os"
+	"os/exec"
 	"strconv"
 	"strings"
 
@@ -34,6 +35,9 @@ func main() {
 		logger.Fatalf("requires hook name. args: %v", os.Args)
 	}
 
+	f, _ := os.OpenFile("/tmp/hello", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
+	hello := log.New(f, "gitaly-hooks: ", log.LstdFlags)
+
 	subCmd := os.Args[1]
 
 	if subCmd == "check" {
@@ -224,6 +228,16 @@ func main() {
 		}, f, os.Stdout, os.Stderr); err != nil {
 			logger.Fatalf("error when receiving data for %q: %v", subCmd, err)
 		}
+	case "git":
+		cmd := exec.Command("git", os.Args[2:]...)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		cmd.Stdin = os.Stdin
+		err := cmd.Run()
+		hello.Print(err)
+		if err == nil {
+			hookStatus = 0
+		}
 	default:
 		logger.Fatalf("subcommand name invalid: %q", subCmd)
 	}
diff --git a/internal/git/hooks.go b/internal/git/hooks.go
index 8716ffc7b55..41d7ef22b49 100644
--- a/internal/git/hooks.go
+++ b/internal/git/hooks.go
@@ -36,6 +36,23 @@ func WithRefTxHook(ctx context.Context, repo *gitalypb.Repository, cfg config.Cf
 	}
 }
 
+func WithPackObjectsHook(ctx context.Context, repo *gitalypb.Repository, cfg config.Cfg) CmdOpt {
+	return func(cc *cmdCfg) error {
+		if repo == nil {
+			return fmt.Errorf("missing repo: %w", ErrInvalidArg)
+		}
+
+		rfEnvs, err := refHookEnv(ctx, repo, cfg)
+		if err != nil {
+			return fmt.Errorf("ref hook env var: %w", err)
+		}
+
+		cc.env = append(cc.env, rfEnvs...)
+
+		return nil
+	}
+}
+
 // refHookEnv returns all env vars required by the reference transaction hook
 func refHookEnv(ctx context.Context, repo *gitalypb.Repository, cfg config.Cfg) ([]string, error) {
 	repoJSON, err := jsonpbMarshaller.MarshalToString(repo)
diff --git a/internal/gitaly/service/register.go b/internal/gitaly/service/register.go
index ed6aed76a17..d02aaf73ea1 100644
--- a/internal/gitaly/service/register.go
+++ b/internal/gitaly/service/register.go
@@ -84,6 +84,7 @@ func RegisterAll(grpcServer *grpc.Server, cfg config.Cfg, rubyServer *rubyserver
 	gitalypb.RegisterSmartHTTPServiceServer(grpcServer, smarthttp.NewServer(
 		locator,
 		smarthttp.WithPackfileNegotiationMetrics(smarthttpPackfileNegotiationMetrics),
+		smarthttp.WithConfig(cfg),
 	))
 	gitalypb.RegisterWikiServiceServer(grpcServer, wiki.NewServer(rubyServer, locator))
 	gitalypb.RegisterConflictsServiceServer(grpcServer, conflicts.NewServer(rubyServer, cfg, locator))
diff --git a/internal/gitaly/service/smarthttp/server.go b/internal/gitaly/service/smarthttp/server.go
index 68f96a238fc..81eec997986 100644
--- a/internal/gitaly/service/smarthttp/server.go
+++ b/internal/gitaly/service/smarthttp/server.go
@@ -2,6 +2,7 @@ package smarthttp
 
 import (
 	"github.com/prometheus/client_golang/prometheus"
+	"gitlab.com/gitlab-org/gitaly/internal/gitaly/config"
 	"gitlab.com/gitlab-org/gitaly/internal/storage"
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 )
@@ -9,6 +10,7 @@ import (
 type server struct {
 	locator                    storage.Locator
 	packfileNegotiationMetrics *prometheus.CounterVec
+	cfg                        config.Cfg
 }
 
 // NewServer creates a new instance of a grpc SmartHTTPServer
@@ -36,3 +38,9 @@ func WithPackfileNegotiationMetrics(c *prometheus.CounterVec) ServerOpt {
 		s.packfileNegotiationMetrics = c
 	}
 }
+
+func WithConfig(cfg config.Cfg) ServerOpt {
+	return func(s *server) {
+		s.cfg = cfg
+	}
+}
\ No newline at end of file
diff --git a/internal/gitaly/service/smarthttp/testhelper_test.go b/internal/gitaly/service/smarthttp/testhelper_test.go
index bb8cbe4f5db..b45bb5bf316 100644
--- a/internal/gitaly/service/smarthttp/testhelper_test.go
+++ b/internal/gitaly/service/smarthttp/testhelper_test.go
@@ -60,6 +60,7 @@ func runSmartHTTPServer(t *testing.T, serverOpts ...ServerOpt) (string, func())
 		},
 	)
 
+	serverOpts = append([]ServerOpt{WithConfig(config.Config)}, serverOpts...)
 	gitalypb.RegisterSmartHTTPServiceServer(srv.GrpcServer(), NewServer(config.NewLocator(config.Config), serverOpts...))
 	reflection.Register(srv.GrpcServer())
 
diff --git a/internal/gitaly/service/smarthttp/upload_pack.go b/internal/gitaly/service/smarthttp/upload_pack.go
index f501ad3977d..5d0dd913f48 100644
--- a/internal/gitaly/service/smarthttp/upload_pack.go
+++ b/internal/gitaly/service/smarthttp/upload_pack.go
@@ -4,6 +4,7 @@ import (
 	"crypto/sha1"
 	"fmt"
 	"io"
+	"path/filepath"
 
 	"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
 	"gitlab.com/gitlab-org/gitaly/internal/command"
@@ -77,11 +78,17 @@ func (s *server) PostUploadPack(stream gitalypb.SmartHTTPService_PostUploadPackS
 		globalOpts = append(globalOpts, git.ValueFlag{"-c", o})
 	}
 
+	hookBin := filepath.Join(s.cfg.BinDir, "gitaly-hooks")
+	globalOpts = append(globalOpts, git.ValueFlag{"-c", "uploadpack.packobjectshook=" + hookBin})
+
 	cmd, err := git.SafeBareCmd(ctx, git.CmdStream{In: stdin, Out: stdout}, nil, globalOpts, git.SubCmd{
 		Name:  "upload-pack",
 		Flags: []git.Option{git.Flag{Name: "--stateless-rpc"}},
 		Args:  []string{repoPath},
-	}, git.WithGitProtocol(ctx, req))
+	},
+		git.WithGitProtocol(ctx, req),
+		git.WithPackObjectsHook(ctx, req.Repository, s.cfg),
+	)
 
 	if err != nil {
 		return status.Errorf(codes.Unavailable, "PostUploadPack: cmd: %v", err)
diff --git a/internal/gitaly/service/smarthttp/upload_pack_test.go b/internal/gitaly/service/smarthttp/upload_pack_test.go
index ebbc6d34148..caebc6ef937 100644
--- a/internal/gitaly/service/smarthttp/upload_pack_test.go
+++ b/internal/gitaly/service/smarthttp/upload_pack_test.go
@@ -324,6 +324,8 @@ func extractPackDataFromResponse(t *testing.T, buf *bytes.Buffer) ([]byte, int,
 }
 
 func TestUploadPackRequestForPartialCloneSuccess(t *testing.T) {
+	t.Skip("skipping because of quoting issue in pack-objects hook")
+
 	negotiationMetrics := prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"feature"})
 
 	serverSocketPath, stop := runSmartHTTPServer(
-- 
GitLab


From a142c30764752c76f8e4cc57262a7eff28114437 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sat, 21 Nov 2020 12:45:36 +0100
Subject: [PATCH 02/22] Clone via pack-objects hook

---
 cmd/gitaly-hooks/hooks.go                     |  29 +-
 internal/gitaly/service/hook/pack_objects.go  |  52 ++++
 .../service/smarthttp/testhelper_test.go      |  10 +
 proto/go/gitalypb/hook.pb.go                  | 259 +++++++++++++++---
 proto/hook.proto                              |  17 ++
 ruby/proto/gitaly/hook_pb.rb                  |  12 +
 ruby/proto/gitaly/hook_services_pb.rb         |   1 +
 7 files changed, 333 insertions(+), 47 deletions(-)
 create mode 100644 internal/gitaly/service/hook/pack_objects.go

diff --git a/cmd/gitaly-hooks/hooks.go b/cmd/gitaly-hooks/hooks.go
index 406af12f9cd..956ad00326f 100644
--- a/cmd/gitaly-hooks/hooks.go
+++ b/cmd/gitaly-hooks/hooks.go
@@ -7,7 +7,6 @@ import (
 	"io"
 	"log"
 	"os"
-	"os/exec"
 	"strconv"
 	"strings"
 
@@ -229,14 +228,26 @@ func main() {
 			logger.Fatalf("error when receiving data for %q: %v", subCmd, err)
 		}
 	case "git":
-		cmd := exec.Command("git", os.Args[2:]...)
-		cmd.Stdout = os.Stdout
-		cmd.Stderr = os.Stderr
-		cmd.Stdin = os.Stdin
-		err := cmd.Run()
-		hello.Print(err)
-		if err == nil {
-			hookStatus = 0
+		packObjectsHookStream, err := hookClient.PackObjectsHook(ctx)
+		if err != nil {
+			hello.Fatalf("error when getting stream client for %q: %v", subCmd, err)
+		}
+
+		if err := packObjectsHookStream.Send(&gitalypb.PackObjectsHookRequest{
+			Repository: repository,
+			Args:       os.Args[2:],
+		}); err != nil {
+			hello.Fatalf("error when sending request for %q: %v", subCmd, err)
+		}
+
+		f := sendFunc(streamio.NewWriter(func(p []byte) error {
+			return packObjectsHookStream.Send(&gitalypb.PackObjectsHookRequest{Stdin: p})
+		}), packObjectsHookStream, os.Stdin)
+
+		if hookStatus, err = stream.Handler(func() (stream.StdoutStderrResponse, error) {
+			return packObjectsHookStream.Recv()
+		}, f, os.Stdout, os.Stderr); err != nil {
+			hello.Fatalf("error when receiving data for %q: %v", subCmd, err)
 		}
 	default:
 		logger.Fatalf("subcommand name invalid: %q", subCmd)
diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
new file mode 100644
index 00000000000..bc338d950ba
--- /dev/null
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -0,0 +1,52 @@
+package hook
+
+import (
+	"os/exec"
+	"sync"
+
+	"gitlab.com/gitlab-org/gitaly/internal/command"
+	"gitlab.com/gitlab-org/gitaly/internal/helper"
+	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+	"gitlab.com/gitlab-org/gitaly/streamio"
+)
+
+func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServer) error {
+	ctx := stream.Context()
+
+	firstRequest, err := stream.Recv()
+	if err != nil {
+		return helper.ErrInternal(err)
+	}
+
+	repoPath, err := helper.GetRepoPath(firstRequest.Repository)
+	if err != nil {
+		return err
+	}
+
+	stdin := streamio.NewReader(func() ([]byte, error) {
+		req, err := stream.Recv()
+		return req.GetStdin(), err
+	})
+
+	var m sync.Mutex
+	stdout := streamio.NewSyncWriter(&m, func(p []byte) error {
+		return stream.Send(&gitalypb.PackObjectsHookResponse{Stdout: p})
+	})
+	stderr := streamio.NewSyncWriter(&m, func(p []byte) error {
+		return stream.Send(&gitalypb.PackObjectsHookResponse{Stderr: p})
+	})
+
+	cmd, err := command.New(ctx, exec.Command("git", append([]string{"-C", repoPath}, firstRequest.Args...)...), stdin, stdout, stderr)
+	if err != nil {
+		return err
+	}
+	lastResponse := &gitalypb.PackObjectsHookResponse{
+		ExitStatus: &gitalypb.ExitStatus{Value: 0},
+	}
+	if err := cmd.Wait(); err != nil {
+		lastResponse.ExitStatus.Value = 1
+	}
+
+	return stream.Send(lastResponse)
+
+}
diff --git a/internal/gitaly/service/smarthttp/testhelper_test.go b/internal/gitaly/service/smarthttp/testhelper_test.go
index b45bb5bf316..13714e99cff 100644
--- a/internal/gitaly/service/smarthttp/testhelper_test.go
+++ b/internal/gitaly/service/smarthttp/testhelper_test.go
@@ -1,6 +1,7 @@
 package smarthttp
 
 import (
+	"net"
 	"os"
 	"path/filepath"
 	"testing"
@@ -11,6 +12,8 @@ import (
 	diskcache "gitlab.com/gitlab-org/gitaly/internal/cache"
 	"gitlab.com/gitlab-org/gitaly/internal/git/hooks"
 	"gitlab.com/gitlab-org/gitaly/internal/gitaly/config"
+	gitalyhook "gitlab.com/gitlab-org/gitaly/internal/gitaly/hook"
+	"gitlab.com/gitlab-org/gitaly/internal/gitaly/service/hook"
 	"gitlab.com/gitlab-org/gitaly/internal/middleware/cache"
 	"gitlab.com/gitlab-org/gitaly/internal/praefect/protoregistry"
 	"gitlab.com/gitlab-org/gitaly/internal/testhelper"
@@ -62,9 +65,16 @@ func runSmartHTTPServer(t *testing.T, serverOpts ...ServerOpt) (string, func())
 
 	serverOpts = append([]ServerOpt{WithConfig(config.Config)}, serverOpts...)
 	gitalypb.RegisterSmartHTTPServiceServer(srv.GrpcServer(), NewServer(config.NewLocator(config.Config), serverOpts...))
+	gitalypb.RegisterHookServiceServer(srv.GrpcServer(), hook.NewServer(gitalyhook.NewManager(gitalyhook.GitlabAPIStub, config.Config)))
 	reflection.Register(srv.GrpcServer())
 
+	internalListener, err := net.Listen("unix", config.Config.GitalyInternalSocketPath())
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	require.NoError(t, srv.Start())
+	go srv.GrpcServer().Serve(internalListener)
 
 	return "unix://" + srv.Socket(), srv.Stop
 }
diff --git a/proto/go/gitalypb/hook.pb.go b/proto/go/gitalypb/hook.pb.go
index 5adab06d02e..93b9c82d028 100644
--- a/proto/go/gitalypb/hook.pb.go
+++ b/proto/go/gitalypb/hook.pb.go
@@ -532,6 +532,116 @@ func (m *ReferenceTransactionHookResponse) GetExitStatus() *ExitStatus {
 	return nil
 }
 
+type PackObjectsHookRequest struct {
+	Repository           *Repository `protobuf:"bytes,1,opt,name=repository,proto3" json:"repository,omitempty"`
+	Args                 []string    `protobuf:"bytes,2,rep,name=args,proto3" json:"args,omitempty"`
+	Stdin                []byte      `protobuf:"bytes,3,opt,name=stdin,proto3" json:"stdin,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}    `json:"-"`
+	XXX_unrecognized     []byte      `json:"-"`
+	XXX_sizecache        int32       `json:"-"`
+}
+
+func (m *PackObjectsHookRequest) Reset()         { *m = PackObjectsHookRequest{} }
+func (m *PackObjectsHookRequest) String() string { return proto.CompactTextString(m) }
+func (*PackObjectsHookRequest) ProtoMessage()    {}
+func (*PackObjectsHookRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_3eef30da1c11ee1b, []int{8}
+}
+
+func (m *PackObjectsHookRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_PackObjectsHookRequest.Unmarshal(m, b)
+}
+func (m *PackObjectsHookRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_PackObjectsHookRequest.Marshal(b, m, deterministic)
+}
+func (m *PackObjectsHookRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_PackObjectsHookRequest.Merge(m, src)
+}
+func (m *PackObjectsHookRequest) XXX_Size() int {
+	return xxx_messageInfo_PackObjectsHookRequest.Size(m)
+}
+func (m *PackObjectsHookRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_PackObjectsHookRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PackObjectsHookRequest proto.InternalMessageInfo
+
+func (m *PackObjectsHookRequest) GetRepository() *Repository {
+	if m != nil {
+		return m.Repository
+	}
+	return nil
+}
+
+func (m *PackObjectsHookRequest) GetArgs() []string {
+	if m != nil {
+		return m.Args
+	}
+	return nil
+}
+
+func (m *PackObjectsHookRequest) GetStdin() []byte {
+	if m != nil {
+		return m.Stdin
+	}
+	return nil
+}
+
+type PackObjectsHookResponse struct {
+	Stdout               []byte      `protobuf:"bytes,1,opt,name=stdout,proto3" json:"stdout,omitempty"`
+	Stderr               []byte      `protobuf:"bytes,2,opt,name=stderr,proto3" json:"stderr,omitempty"`
+	ExitStatus           *ExitStatus `protobuf:"bytes,3,opt,name=exit_status,json=exitStatus,proto3" json:"exit_status,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}    `json:"-"`
+	XXX_unrecognized     []byte      `json:"-"`
+	XXX_sizecache        int32       `json:"-"`
+}
+
+func (m *PackObjectsHookResponse) Reset()         { *m = PackObjectsHookResponse{} }
+func (m *PackObjectsHookResponse) String() string { return proto.CompactTextString(m) }
+func (*PackObjectsHookResponse) ProtoMessage()    {}
+func (*PackObjectsHookResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_3eef30da1c11ee1b, []int{9}
+}
+
+func (m *PackObjectsHookResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_PackObjectsHookResponse.Unmarshal(m, b)
+}
+func (m *PackObjectsHookResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_PackObjectsHookResponse.Marshal(b, m, deterministic)
+}
+func (m *PackObjectsHookResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_PackObjectsHookResponse.Merge(m, src)
+}
+func (m *PackObjectsHookResponse) XXX_Size() int {
+	return xxx_messageInfo_PackObjectsHookResponse.Size(m)
+}
+func (m *PackObjectsHookResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_PackObjectsHookResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PackObjectsHookResponse proto.InternalMessageInfo
+
+func (m *PackObjectsHookResponse) GetStdout() []byte {
+	if m != nil {
+		return m.Stdout
+	}
+	return nil
+}
+
+func (m *PackObjectsHookResponse) GetStderr() []byte {
+	if m != nil {
+		return m.Stderr
+	}
+	return nil
+}
+
+func (m *PackObjectsHookResponse) GetExitStatus() *ExitStatus {
+	if m != nil {
+		return m.ExitStatus
+	}
+	return nil
+}
+
 func init() {
 	proto.RegisterEnum("gitaly.ReferenceTransactionHookRequest_State", ReferenceTransactionHookRequest_State_name, ReferenceTransactionHookRequest_State_value)
 	proto.RegisterType((*PreReceiveHookRequest)(nil), "gitaly.PreReceiveHookRequest")
@@ -542,49 +652,54 @@ func init() {
 	proto.RegisterType((*UpdateHookResponse)(nil), "gitaly.UpdateHookResponse")
 	proto.RegisterType((*ReferenceTransactionHookRequest)(nil), "gitaly.ReferenceTransactionHookRequest")
 	proto.RegisterType((*ReferenceTransactionHookResponse)(nil), "gitaly.ReferenceTransactionHookResponse")
+	proto.RegisterType((*PackObjectsHookRequest)(nil), "gitaly.PackObjectsHookRequest")
+	proto.RegisterType((*PackObjectsHookResponse)(nil), "gitaly.PackObjectsHookResponse")
 }
 
 func init() { proto.RegisterFile("hook.proto", fileDescriptor_3eef30da1c11ee1b) }
 
 var fileDescriptor_3eef30da1c11ee1b = []byte{
-	// 591 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x55, 0x4f, 0x6f, 0xd3, 0x4e,
-	0x10, 0xfd, 0xad, 0xd3, 0xe4, 0xd7, 0x4c, 0x42, 0x09, 0xab, 0xb6, 0x18, 0x23, 0x68, 0xe4, 0x0b,
-	0x3e, 0xd0, 0xa4, 0xb4, 0x17, 0xae, 0xfd, 0x13, 0x09, 0x0e, 0x55, 0xa3, 0x6d, 0xe9, 0x01, 0x24,
-	0x22, 0x27, 0x9e, 0x26, 0xab, 0xba, 0x5e, 0xb3, 0xbb, 0x4e, 0x9b, 0x03, 0x5c, 0xf9, 0x02, 0x48,
-	0x70, 0xe2, 0xe3, 0x94, 0x1b, 0x1f, 0x88, 0x13, 0xb2, 0x9d, 0xbf, 0x4d, 0x42, 0x39, 0x85, 0xde,
-	0xf6, 0xcd, 0xdb, 0x99, 0xf5, 0x7b, 0x3b, 0x3b, 0x06, 0xe8, 0x08, 0x71, 0x5e, 0x09, 0xa5, 0xd0,
-	0x82, 0xe6, 0xda, 0x5c, 0xbb, 0x7e, 0xcf, 0x02, 0x9f, 0x07, 0x3a, 0x8d, 0x59, 0x45, 0xd5, 0x71,
-	0x25, 0x7a, 0x29, 0xb2, 0xaf, 0x09, 0xac, 0xd5, 0x25, 0x32, 0x6c, 0x21, 0xef, 0xe2, 0x2b, 0x21,
-	0xce, 0x19, 0x7e, 0x88, 0x50, 0x69, 0xfa, 0x12, 0x40, 0x62, 0x28, 0x14, 0xd7, 0x42, 0xf6, 0x4c,
-	0x52, 0x26, 0x4e, 0x61, 0x9b, 0x56, 0xd2, 0x82, 0x15, 0x36, 0x64, 0xf6, 0x96, 0xbe, 0x5d, 0x3f,
-	0x27, 0x6c, 0x6c, 0x2f, 0xdd, 0x81, 0x35, 0x0c, 0xba, 0x5c, 0x8a, 0xe0, 0x02, 0x03, 0xdd, 0xe8,
-	0xba, 0x92, 0xbb, 0x4d, 0x1f, 0x95, 0x69, 0x94, 0x33, 0x4e, 0x9e, 0xad, 0x8e, 0x91, 0xa7, 0x03,
-	0x8e, 0xae, 0x42, 0x56, 0x69, 0x8f, 0x07, 0xe6, 0x52, 0x99, 0x38, 0x45, 0x96, 0x02, 0xea, 0x40,
-	0xa9, 0xcd, 0x75, 0x23, 0x8c, 0x54, 0xa7, 0x21, 0x42, 0xcd, 0x45, 0xa0, 0xcc, 0x6c, 0x52, 0x65,
-	0xa5, 0xcd, 0x75, 0x3d, 0x52, 0x9d, 0xa3, 0x34, 0x6a, 0x7f, 0x84, 0xf5, 0x9b, 0x3a, 0x54, 0x28,
-	0x02, 0x85, 0x74, 0x1d, 0x72, 0x4a, 0x7b, 0x22, 0xd2, 0x89, 0x88, 0x22, 0xeb, 0xa3, 0x7e, 0x1c,
-	0xa5, 0x34, 0x8d, 0x61, 0x1c, 0xa5, 0xa4, 0x3b, 0x50, 0xc0, 0x2b, 0xae, 0x1b, 0x4a, 0xbb, 0x3a,
-	0x52, 0x66, 0x66, 0x52, 0x79, 0xed, 0x8a, 0xeb, 0xe3, 0x84, 0x61, 0x80, 0xc3, 0xb5, 0xfd, 0x83,
-	0xc0, 0x7a, 0x5d, 0x28, 0x7d, 0x87, 0x8c, 0xcc, 0xdc, 0x66, 0xe4, 0xd2, 0x4c, 0x23, 0x3f, 0xc1,
-	0xc3, 0x29, 0x21, 0x8b, 0x74, 0xf2, 0x27, 0x81, 0x07, 0x6f, 0x42, 0xcf, 0xd5, 0xff, 0xd2, 0xc4,
-	0x12, 0x64, 0x24, 0x9e, 0xf5, 0x2d, 0x8c, 0x97, 0xf4, 0x31, 0xe4, 0x85, 0xef, 0x35, 0xba, 0xae,
-	0x1f, 0x61, 0xd2, 0xa3, 0x79, 0xb6, 0x2c, 0x7c, 0xef, 0x34, 0xc6, 0x31, 0x19, 0xe0, 0x65, 0x9f,
-	0xcc, 0xa6, 0x64, 0x80, 0x97, 0x09, 0x69, 0xf7, 0x80, 0x8e, 0xeb, 0x59, 0xa4, 0x97, 0xdf, 0x0d,
-	0xd8, 0x60, 0x78, 0x86, 0x12, 0x83, 0x16, 0x9e, 0x48, 0x37, 0x50, 0x6e, 0x2b, 0xbe, 0xe5, 0x3b,
-	0xd7, 0x9e, 0xfb, 0x71, 0xd4, 0xd5, 0xa9, 0xb3, 0x2b, 0xdb, 0x9b, 0xa3, 0xf3, 0xff, 0xf8, 0xf1,
-	0x95, 0x58, 0x27, 0xb2, 0x34, 0xd7, 0x7e, 0x01, 0xd9, 0x04, 0xd3, 0x22, 0x2c, 0xd7, 0x59, 0xad,
-	0xbe, 0xcb, 0x6a, 0x07, 0xa5, 0xff, 0xe8, 0x3d, 0xc8, 0xef, 0x1f, 0x1d, 0x1e, 0xbe, 0x3e, 0x39,
-	0xa9, 0x1d, 0x94, 0x08, 0x2d, 0xc0, 0xff, 0xbb, 0x7b, 0x47, 0x2c, 0x06, 0x86, 0xfd, 0x99, 0x40,
-	0x79, 0xfe, 0x19, 0x0b, 0xbc, 0xaa, 0xed, 0x2f, 0x19, 0x28, 0xc4, 0xa7, 0x1e, 0xa3, 0xec, 0xf2,
-	0x16, 0xd2, 0x77, 0xb0, 0x32, 0x39, 0xcf, 0xe8, 0x93, 0x41, 0x85, 0x99, 0xf3, 0xda, 0x7a, 0x3a,
-	0x8f, 0x4e, 0x55, 0xd8, 0xb9, 0x5f, 0x5f, 0x1d, 0x63, 0xd9, 0x70, 0xc8, 0x16, 0xa1, 0xef, 0xe1,
-	0xfe, 0x8d, 0x37, 0x4e, 0x47, 0xe9, 0x33, 0xa7, 0x98, 0xb5, 0x31, 0x97, 0x9f, 0x51, 0xff, 0x10,
-	0x60, 0xd4, 0xf2, 0xf4, 0xd1, 0x20, 0x75, 0xea, 0x59, 0x5b, 0xd6, 0x2c, 0x6a, 0xb2, 0xe0, 0x16,
-	0xa1, 0x3d, 0x30, 0xe7, 0x5d, 0x12, 0x7d, 0xf6, 0x97, 0xad, 0x62, 0x39, 0xb7, 0x6f, 0x9c, 0x56,
-	0xb2, 0xb7, 0xf5, 0x36, 0x4e, 0xf3, 0xdd, 0x66, 0xa5, 0x25, 0x2e, 0xaa, 0xe9, 0x72, 0x53, 0xc8,
-	0x76, 0x35, 0x2d, 0x56, 0x4d, 0xfe, 0xa2, 0xd5, 0xb6, 0xe8, 0xe3, 0xb0, 0xd9, 0xcc, 0x25, 0xa1,
-	0x9d, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x1e, 0xa9, 0xde, 0x88, 0x88, 0x07, 0x00, 0x00,
+	// 638 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x56, 0xcd, 0x72, 0x12, 0x41,
+	0x10, 0x76, 0xf9, 0x33, 0x34, 0x18, 0x71, 0x2a, 0xc1, 0x75, 0x2d, 0x0d, 0xb5, 0x17, 0xf7, 0x60,
+	0x20, 0x26, 0x17, 0xaf, 0xf9, 0xa1, 0x4a, 0x0f, 0x29, 0xa8, 0x49, 0xcc, 0x41, 0xab, 0xa4, 0x86,
+	0xa5, 0x03, 0x63, 0x36, 0x3b, 0xeb, 0xcc, 0x40, 0x42, 0x95, 0x7a, 0xf5, 0x11, 0xf4, 0xe4, 0xe3,
+	0xc4, 0x9b, 0x4f, 0xe3, 0xc9, 0x93, 0xb5, 0xbb, 0x90, 0x40, 0xb2, 0x18, 0x0f, 0x16, 0xe6, 0xd6,
+	0xdd, 0xdf, 0x4c, 0x37, 0xdf, 0xd7, 0x3d, 0xcd, 0x02, 0xf4, 0x84, 0x38, 0xaa, 0x06, 0x52, 0x68,
+	0x41, 0x72, 0x5d, 0xae, 0x99, 0x37, 0xb4, 0xc0, 0xe3, 0xbe, 0x8e, 0x63, 0x56, 0x51, 0xf5, 0x98,
+	0xc4, 0x4e, 0xec, 0xd9, 0x67, 0x06, 0x2c, 0x37, 0x25, 0x52, 0x74, 0x91, 0x0f, 0xf0, 0x85, 0x10,
+	0x47, 0x14, 0xdf, 0xf7, 0x51, 0x69, 0xf2, 0x1c, 0x40, 0x62, 0x20, 0x14, 0xd7, 0x42, 0x0e, 0x4d,
+	0xa3, 0x62, 0x38, 0x85, 0x75, 0x52, 0x8d, 0x13, 0x56, 0xe9, 0x39, 0xb2, 0x95, 0xf9, 0x7a, 0xf6,
+	0xd4, 0xa0, 0x13, 0x67, 0xc9, 0x06, 0x2c, 0xa3, 0x3f, 0xe0, 0x52, 0xf8, 0xc7, 0xe8, 0xeb, 0xd6,
+	0x80, 0x49, 0xce, 0xda, 0x1e, 0x2a, 0x33, 0x55, 0x49, 0x3b, 0x79, 0xba, 0x34, 0x01, 0x1e, 0x8c,
+	0x31, 0xb2, 0x04, 0x59, 0xa5, 0x3b, 0xdc, 0x37, 0x33, 0x15, 0xc3, 0x29, 0xd2, 0xd8, 0x21, 0x0e,
+	0x94, 0xba, 0x5c, 0xb7, 0x82, 0xbe, 0xea, 0xb5, 0x44, 0xa0, 0xb9, 0xf0, 0x95, 0x99, 0x8d, 0xb2,
+	0x2c, 0x76, 0xb9, 0x6e, 0xf6, 0x55, 0xaf, 0x11, 0x47, 0xed, 0x8f, 0x50, 0xbe, 0xcc, 0x43, 0x05,
+	0xc2, 0x57, 0x48, 0xca, 0x90, 0x53, 0xba, 0x23, 0xfa, 0x3a, 0x22, 0x51, 0xa4, 0x23, 0x6f, 0x14,
+	0x47, 0x29, 0xcd, 0xd4, 0x79, 0x1c, 0xa5, 0x24, 0x1b, 0x50, 0xc0, 0x53, 0xae, 0x5b, 0x4a, 0x33,
+	0xdd, 0x57, 0x66, 0x7a, 0x9a, 0x79, 0xfd, 0x94, 0xeb, 0xbd, 0x08, 0xa1, 0x80, 0xe7, 0xb6, 0xfd,
+	0xdd, 0x80, 0x72, 0x53, 0x28, 0x7d, 0x83, 0x84, 0x4c, 0x5f, 0x27, 0x64, 0x26, 0x51, 0xc8, 0x4f,
+	0x70, 0xff, 0x0a, 0x91, 0x79, 0x2a, 0xf9, 0xc3, 0x80, 0x7b, 0xaf, 0x82, 0x0e, 0xd3, 0xff, 0x53,
+	0xc4, 0x12, 0xa4, 0x25, 0x1e, 0x8e, 0x24, 0x0c, 0x4d, 0xf2, 0x10, 0xf2, 0xc2, 0xeb, 0xb4, 0x06,
+	0xcc, 0xeb, 0x63, 0x34, 0xa3, 0x79, 0xba, 0x20, 0xbc, 0xce, 0x41, 0xe8, 0x87, 0xa0, 0x8f, 0x27,
+	0x23, 0x30, 0x1b, 0x83, 0x3e, 0x9e, 0x44, 0xa0, 0x3d, 0x04, 0x32, 0xc9, 0x67, 0x9e, 0x5a, 0x7e,
+	0x4b, 0xc1, 0x0a, 0xc5, 0x43, 0x94, 0xe8, 0xbb, 0xb8, 0x2f, 0x99, 0xaf, 0x98, 0x1b, 0x76, 0xf9,
+	0xc6, 0x8d, 0xe7, 0x76, 0x18, 0x65, 0x3a, 0x56, 0x76, 0x71, 0x7d, 0xf5, 0xa2, 0xfe, 0x1f, 0x7f,
+	0x7c, 0x35, 0xe4, 0x89, 0x34, 0xbe, 0x6b, 0x3f, 0x83, 0x6c, 0xe4, 0x93, 0x22, 0x2c, 0x34, 0x69,
+	0xbd, 0xb9, 0x49, 0xeb, 0x3b, 0xa5, 0x5b, 0xe4, 0x0e, 0xe4, 0xb7, 0x1b, 0xbb, 0xbb, 0x2f, 0xf7,
+	0xf7, 0xeb, 0x3b, 0x25, 0x83, 0x14, 0xe0, 0xf6, 0xe6, 0x56, 0x83, 0x86, 0x4e, 0xca, 0xfe, 0x6c,
+	0x40, 0x65, 0x76, 0x8d, 0x79, 0xb6, 0xea, 0x03, 0x94, 0x9b, 0xcc, 0x3d, 0x6a, 0xb4, 0xdf, 0xa1,
+	0xab, 0xd5, 0xbf, 0x69, 0x10, 0x81, 0x0c, 0x93, 0xdd, 0x71, 0x3f, 0x22, 0x3b, 0x59, 0xff, 0xe8,
+	0xd1, 0x5f, 0xae, 0x3e, 0x47, 0xf6, 0xeb, 0x3f, 0xd3, 0x50, 0x08, 0xab, 0xee, 0xa1, 0x1c, 0x70,
+	0x17, 0xc9, 0x1b, 0x58, 0x9c, 0xde, 0xe6, 0xe4, 0xd1, 0x38, 0x43, 0xe2, 0xbf, 0x95, 0xf5, 0x78,
+	0x16, 0x1c, 0xb3, 0xb0, 0x73, 0xbf, 0xbe, 0x38, 0xa9, 0x85, 0x94, 0x63, 0xac, 0x19, 0xe4, 0x2d,
+	0xdc, 0xbd, 0xb4, 0xe1, 0xc8, 0xc5, 0xf5, 0xc4, 0x1d, 0x6e, 0xad, 0xcc, 0xc4, 0x13, 0xf2, 0xef,
+	0x02, 0x5c, 0x3c, 0x78, 0xf2, 0x60, 0x7c, 0xf5, 0xca, 0x52, 0xb3, 0xac, 0x24, 0x68, 0x3a, 0xe1,
+	0x9a, 0x41, 0x86, 0x60, 0xce, 0x1a, 0x51, 0xf2, 0xe4, 0x2f, 0x1f, 0x8a, 0xe5, 0x5c, 0x7f, 0x70,
+	0x86, 0x52, 0xd3, 0x63, 0x31, 0xa1, 0x54, 0xe2, 0xb4, 0x4e, 0x28, 0x95, 0x3c, 0x4f, 0x93, 0xf9,
+	0xb7, 0xd6, 0x5e, 0x87, 0xa7, 0x3d, 0xd6, 0xae, 0xba, 0xe2, 0xb8, 0x16, 0x9b, 0xab, 0x42, 0x76,
+	0x6b, 0x71, 0x8e, 0x5a, 0xf4, 0x8d, 0x52, 0xeb, 0x8a, 0x91, 0x1f, 0xb4, 0xdb, 0xb9, 0x28, 0xb4,
+	0xf1, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x5f, 0x23, 0xe8, 0xd8, 0xe6, 0x08, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -603,6 +718,7 @@ type HookServiceClient interface {
 	PostReceiveHook(ctx context.Context, opts ...grpc.CallOption) (HookService_PostReceiveHookClient, error)
 	UpdateHook(ctx context.Context, in *UpdateHookRequest, opts ...grpc.CallOption) (HookService_UpdateHookClient, error)
 	ReferenceTransactionHook(ctx context.Context, opts ...grpc.CallOption) (HookService_ReferenceTransactionHookClient, error)
+	PackObjectsHook(ctx context.Context, opts ...grpc.CallOption) (HookService_PackObjectsHookClient, error)
 }
 
 type hookServiceClient struct {
@@ -738,12 +854,44 @@ func (x *hookServiceReferenceTransactionHookClient) Recv() (*ReferenceTransactio
 	return m, nil
 }
 
+func (c *hookServiceClient) PackObjectsHook(ctx context.Context, opts ...grpc.CallOption) (HookService_PackObjectsHookClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_HookService_serviceDesc.Streams[4], "/gitaly.HookService/PackObjectsHook", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &hookServicePackObjectsHookClient{stream}
+	return x, nil
+}
+
+type HookService_PackObjectsHookClient interface {
+	Send(*PackObjectsHookRequest) error
+	Recv() (*PackObjectsHookResponse, error)
+	grpc.ClientStream
+}
+
+type hookServicePackObjectsHookClient struct {
+	grpc.ClientStream
+}
+
+func (x *hookServicePackObjectsHookClient) Send(m *PackObjectsHookRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *hookServicePackObjectsHookClient) Recv() (*PackObjectsHookResponse, error) {
+	m := new(PackObjectsHookResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
 // HookServiceServer is the server API for HookService service.
 type HookServiceServer interface {
 	PreReceiveHook(HookService_PreReceiveHookServer) error
 	PostReceiveHook(HookService_PostReceiveHookServer) error
 	UpdateHook(*UpdateHookRequest, HookService_UpdateHookServer) error
 	ReferenceTransactionHook(HookService_ReferenceTransactionHookServer) error
+	PackObjectsHook(HookService_PackObjectsHookServer) error
 }
 
 // UnimplementedHookServiceServer can be embedded to have forward compatible implementations.
@@ -762,6 +910,9 @@ func (*UnimplementedHookServiceServer) UpdateHook(req *UpdateHookRequest, srv Ho
 func (*UnimplementedHookServiceServer) ReferenceTransactionHook(srv HookService_ReferenceTransactionHookServer) error {
 	return status.Errorf(codes.Unimplemented, "method ReferenceTransactionHook not implemented")
 }
+func (*UnimplementedHookServiceServer) PackObjectsHook(srv HookService_PackObjectsHookServer) error {
+	return status.Errorf(codes.Unimplemented, "method PackObjectsHook not implemented")
+}
 
 func RegisterHookServiceServer(s *grpc.Server, srv HookServiceServer) {
 	s.RegisterService(&_HookService_serviceDesc, srv)
@@ -866,6 +1017,32 @@ func (x *hookServiceReferenceTransactionHookServer) Recv() (*ReferenceTransactio
 	return m, nil
 }
 
+func _HookService_PackObjectsHook_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(HookServiceServer).PackObjectsHook(&hookServicePackObjectsHookServer{stream})
+}
+
+type HookService_PackObjectsHookServer interface {
+	Send(*PackObjectsHookResponse) error
+	Recv() (*PackObjectsHookRequest, error)
+	grpc.ServerStream
+}
+
+type hookServicePackObjectsHookServer struct {
+	grpc.ServerStream
+}
+
+func (x *hookServicePackObjectsHookServer) Send(m *PackObjectsHookResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *hookServicePackObjectsHookServer) Recv() (*PackObjectsHookRequest, error) {
+	m := new(PackObjectsHookRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
 var _HookService_serviceDesc = grpc.ServiceDesc{
 	ServiceName: "gitaly.HookService",
 	HandlerType: (*HookServiceServer)(nil),
@@ -894,6 +1071,12 @@ var _HookService_serviceDesc = grpc.ServiceDesc{
 			ServerStreams: true,
 			ClientStreams: true,
 		},
+		{
+			StreamName:    "PackObjectsHook",
+			Handler:       _HookService_PackObjectsHook_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
 	},
 	Metadata: "hook.proto",
 }
diff --git a/proto/hook.proto b/proto/hook.proto
index 0a67247fb55..6cb5f427f4e 100644
--- a/proto/hook.proto
+++ b/proto/hook.proto
@@ -28,6 +28,11 @@ service HookService {
       op: ACCESSOR
     };
   }
+  rpc PackObjectsHook(stream PackObjectsHookRequest) returns  (stream PackObjectsHookResponse) {
+    option (op_type) = {
+      op: ACCESSOR
+    };
+  }
 }
 
 message PreReceiveHookRequest {
@@ -87,3 +92,15 @@ message ReferenceTransactionHookResponse {
   bytes stderr = 2;
   ExitStatus exit_status = 3;
 }
+
+message PackObjectsHookRequest {
+  Repository repository = 1 [(target_repository)=true];
+  repeated string args = 2;
+  bytes stdin = 3;
+}
+
+message PackObjectsHookResponse{
+  bytes stdout = 1;
+  bytes stderr = 2;
+  ExitStatus exit_status = 3;
+}
diff --git a/ruby/proto/gitaly/hook_pb.rb b/ruby/proto/gitaly/hook_pb.rb
index bf32ab7ec8c..60caeb87231 100644
--- a/ruby/proto/gitaly/hook_pb.rb
+++ b/ruby/proto/gitaly/hook_pb.rb
@@ -57,6 +57,16 @@ Google::Protobuf::DescriptorPool.generated_pool.build do
       optional :stderr, :bytes, 2
       optional :exit_status, :message, 3, "gitaly.ExitStatus"
     end
+    add_message "gitaly.PackObjectsHookRequest" do
+      optional :repository, :message, 1, "gitaly.Repository"
+      repeated :args, :string, 2
+      optional :stdin, :bytes, 3
+    end
+    add_message "gitaly.PackObjectsHookResponse" do
+      optional :stdout, :bytes, 1
+      optional :stderr, :bytes, 2
+      optional :exit_status, :message, 3, "gitaly.ExitStatus"
+    end
   end
 end
 
@@ -70,4 +80,6 @@ module Gitaly
   ReferenceTransactionHookRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.ReferenceTransactionHookRequest").msgclass
   ReferenceTransactionHookRequest::State = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.ReferenceTransactionHookRequest.State").enummodule
   ReferenceTransactionHookResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.ReferenceTransactionHookResponse").msgclass
+  PackObjectsHookRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.PackObjectsHookRequest").msgclass
+  PackObjectsHookResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.PackObjectsHookResponse").msgclass
 end
diff --git a/ruby/proto/gitaly/hook_services_pb.rb b/ruby/proto/gitaly/hook_services_pb.rb
index 02e3419e550..07632a2f5ca 100644
--- a/ruby/proto/gitaly/hook_services_pb.rb
+++ b/ruby/proto/gitaly/hook_services_pb.rb
@@ -18,6 +18,7 @@ module Gitaly
       rpc :PostReceiveHook, stream(Gitaly::PostReceiveHookRequest), stream(Gitaly::PostReceiveHookResponse)
       rpc :UpdateHook, Gitaly::UpdateHookRequest, stream(Gitaly::UpdateHookResponse)
       rpc :ReferenceTransactionHook, stream(Gitaly::ReferenceTransactionHookRequest), stream(Gitaly::ReferenceTransactionHookResponse)
+      rpc :PackObjectsHook, stream(Gitaly::PackObjectsHookRequest), stream(Gitaly::PackObjectsHookResponse)
     end
 
     Stub = Service.rpc_stub_class
-- 
GitLab


From 20dd176020c9b2d1e3b310020cccf84c78b676f3 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sat, 21 Nov 2020 13:39:09 +0100
Subject: [PATCH 03/22] Simplify

---
 internal/gitaly/service/hook/pack_objects.go | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index bc338d950ba..622d010da93 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -40,13 +40,6 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 	if err != nil {
 		return err
 	}
-	lastResponse := &gitalypb.PackObjectsHookResponse{
-		ExitStatus: &gitalypb.ExitStatus{Value: 0},
-	}
-	if err := cmd.Wait(); err != nil {
-		lastResponse.ExitStatus.Value = 1
-	}
-
-	return stream.Send(lastResponse)
 
+	return cmd.Wait()
 }
-- 
GitLab


From c72363bf28280a0b72ab01348576c67b92b31e2b Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sat, 21 Nov 2020 13:46:40 +0100
Subject: [PATCH 04/22] Quoting bug workaround

---
 cmd/gitaly-hooks/hooks.go                             | 8 +++++++-
 internal/gitaly/service/smarthttp/upload_pack_test.go | 2 --
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/cmd/gitaly-hooks/hooks.go b/cmd/gitaly-hooks/hooks.go
index 956ad00326f..c078b59b4a7 100644
--- a/cmd/gitaly-hooks/hooks.go
+++ b/cmd/gitaly-hooks/hooks.go
@@ -228,6 +228,12 @@ func main() {
 			logger.Fatalf("error when receiving data for %q: %v", subCmd, err)
 		}
 	case "git":
+		args := os.Args[2:]
+		// Silly workaround for Git quoting bug: --filter='limit=200' should not contain single quotes.
+		for i, a := range args {
+			args[i] = strings.Replace(a, "'", "", -1)
+		}
+
 		packObjectsHookStream, err := hookClient.PackObjectsHook(ctx)
 		if err != nil {
 			hello.Fatalf("error when getting stream client for %q: %v", subCmd, err)
@@ -235,7 +241,7 @@ func main() {
 
 		if err := packObjectsHookStream.Send(&gitalypb.PackObjectsHookRequest{
 			Repository: repository,
-			Args:       os.Args[2:],
+			Args:       args,
 		}); err != nil {
 			hello.Fatalf("error when sending request for %q: %v", subCmd, err)
 		}
diff --git a/internal/gitaly/service/smarthttp/upload_pack_test.go b/internal/gitaly/service/smarthttp/upload_pack_test.go
index caebc6ef937..ebbc6d34148 100644
--- a/internal/gitaly/service/smarthttp/upload_pack_test.go
+++ b/internal/gitaly/service/smarthttp/upload_pack_test.go
@@ -324,8 +324,6 @@ func extractPackDataFromResponse(t *testing.T, buf *bytes.Buffer) ([]byte, int,
 }
 
 func TestUploadPackRequestForPartialCloneSuccess(t *testing.T) {
-	t.Skip("skipping because of quoting issue in pack-objects hook")
-
 	negotiationMetrics := prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"feature"})
 
 	serverSocketPath, stop := runSmartHTTPServer(
-- 
GitLab


From 5c7f3ca9a797e3653b62348c3e4a62f76ca112d8 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sat, 21 Nov 2020 15:02:58 +0100
Subject: [PATCH 05/22] it works in memory

---
 go.mod                                        |   5 +-
 go.sum                                        | 245 ++++++++++++++++++
 .../gitaly/rubyserver/balancer/balancer.go    |   4 +-
 internal/gitaly/service/hook/pack_objects.go  | 196 +++++++++++++-
 4 files changed, 439 insertions(+), 11 deletions(-)

diff --git a/go.mod b/go.mod
index 28d02380ad5..ecb3003f2ec 100644
--- a/go.mod
+++ b/go.mod
@@ -6,7 +6,7 @@ require (
 	github.com/getsentry/sentry-go v0.7.0
 	github.com/git-lfs/git-lfs v1.5.1-0.20200916154635-9ea4eed5b112
 	github.com/gogo/protobuf v1.2.1 // indirect
-	github.com/golang/protobuf v1.3.2
+	github.com/golang/protobuf v1.4.2
 	github.com/google/uuid v1.1.1
 	github.com/grpc-ecosystem/go-grpc-middleware v1.0.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
@@ -28,12 +28,13 @@ require (
 	gitlab.com/gitlab-org/gitlab-shell v1.9.8-0.20201117050822-3f9890ef73dc
 	gitlab.com/gitlab-org/labkit v0.0.0-20201113080642-40dcf811328c
 	go.uber.org/atomic v1.4.0 // indirect
+	gocloud.dev v0.20.0
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
 	golang.org/x/net v0.0.0-20200904194848-62affa334b73 // indirect
 	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
 	golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211
 	golang.org/x/text v0.3.3 // indirect
-	google.golang.org/grpc v1.24.0
+	google.golang.org/grpc v1.29.1
 	gopkg.in/yaml.v2 v2.3.0
 )
 
diff --git a/go.sum b/go.sum
index 4bfb00523ec..53d83ef1d77 100644
--- a/go.sum
+++ b/go.sum
@@ -1,30 +1,85 @@
+bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
 bou.ke/monkey v1.0.1 h1:zEMLInw9xvNakzUUPjfS4Ds6jYPqCFx3m7bRmG5NH2U=
 bou.ke/monkey v1.0.1/go.mod h1:FgHuK96Rv2Nlf+0u1OOVDpCMdsWyOFmeeketDHE7LIg=
 cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts=
 cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
 cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
 cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
 cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
 cloud.google.com/go v0.50.0 h1:0E3eE8MX426vUOs7aHfI7aN1BrIzzzf4ccKCSfSjGmc=
 cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.55.0/go.mod h1:ZHmoY+/lIMNkN2+fBmuTiqZ4inFhvQad8ft7MT8IV5Y=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.58.0 h1:vtAfVc723K3xKq1BQydk/FyCldnaNFhGhpJxaJzgRMQ=
+cloud.google.com/go v0.58.0/go.mod h1:W+9FnSUw6nhVwXlFcp1eL+krq5+HQUJeUogSeJZZiWg=
 cloud.google.com/go/bigquery v1.0.1 h1:hL+ycaJpVE9M7nLoiXb/Pn10ENE2u+oddxbD8uu0ZVU=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
 cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/firestore v1.2.0/go.mod h1:iISCjWnTpnoJT1R287xRdjvQHJrxQOpeah4phb5D3h0=
 cloud.google.com/go/pubsub v1.0.1 h1:W9tAK3E57P75u0XLLR82LZyw8VpAnhmyTOxW9qzmyj8=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
 cloud.google.com/go/storage v1.0.0 h1:VV2nUM3wwLLGh9lSABFgZMjInyUbJeaRSE64WuAIQ+4=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.9.0/go.mod h1:m+/etGaqZbylxaNT876QGXqEHp4PR2Rq5GMqICWb9bU=
+contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA=
+contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw=
+contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE=
+contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
+github.com/Azure/azure-amqp-common-go/v3 v3.0.0/go.mod h1:SY08giD/XbhTz07tJdpw1SoxQXHPN30+DI3Z04SYqyg=
+github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
+github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc=
+github.com/Azure/azure-sdk-for-go v37.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/azure-service-bus-go v0.10.1/go.mod h1:E/FOceuKAFUfpbIJDKWz/May6guE+eGibfGT6q+n1to=
+github.com/Azure/azure-storage-blob-go v0.9.0/go.mod h1:8UBPbiOhrMQ4pLPi3gA1tXnpjrS76UYE/fo5A40vf4g=
+github.com/Azure/go-amqp v0.12.6/go.mod h1:qApuH6OFTSKZFmCOxccvAv5rLizBQf4v8pRmG138DPo=
+github.com/Azure/go-amqp v0.12.7/go.mod h1:qApuH6OFTSKZFmCOxccvAv5rLizBQf4v8pRmG138DPo=
+github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
+github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0=
+github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
+github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
+github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
+github.com/Azure/go-autorest/autorest/adal v0.8.3/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
+github.com/Azure/go-autorest/autorest/azure/auth v0.4.2/go.mod h1:90gmfKdlmKgfjUpnCEpOJzsUEjrWDSLwHIG73tSXddM=
+github.com/Azure/go-autorest/autorest/azure/cli v0.3.1/go.mod h1:ZG5p860J94/0kI9mNJVoIoLgXcirM2gF5i2kWloofxw=
+github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
+github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
+github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
+github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA=
+github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI=
+github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
+github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw=
 github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w=
+github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20191009163259-e802c2cb94ae/go.mod h1:mjwGPas4yKduTyubHvD1Atl9r1rUq8DfVy+gkVvZ+oo=
 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
 github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM=
 github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
@@ -37,6 +92,9 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/avast/retry-go v2.4.2+incompatible h1:+ZjCypQT/CyP0kyJO2EcU4d/ZEJWSbP8NENI578cPmA=
 github.com/avast/retry-go v2.4.2+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
+github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
+github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.31.13/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -45,6 +103,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2 h1:MmeatFT1pTPSVb4nkPmBFN/LRZ97vPjsFKsZrU3KKTs=
 github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -56,6 +116,7 @@ github.com/client9/reopen v1.0.0/go.mod h1:caXVCEr+lUtoN1FlsRiOWdfQtdRHIYfcb0ai8
 github.com/cloudflare/tableflip v0.0.0-20190329062924-8392f1641731/go.mod h1:erh4dYezoMVbIa52pi7i1Du7+cXOgqNuTamt10qvMoA=
 github.com/cloudflare/tableflip v1.2.1-0.20200514155827-4baec9811f2b h1:PF1TsqplD1wyV3nBSzRHP3teTQFtaWl3jaENrGqZ+lI=
 github.com/cloudflare/tableflip v1.2.1-0.20200514155827-4baec9811f2b/go.mod h1:vhhSlJqV8uUnxGkRSgyvGthfGlkAwJ4UuSV51fSrCQY=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
@@ -67,18 +128,25 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
+github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY=
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
 github.com/dpotapov/go-spnego v0.0.0-20190506202455-c2c609116ad0 h1:Hhh7nu7CfFVlnBJqmDDUh+j1H5fqjLMzM4czZzNNJGM=
 github.com/dpotapov/go-spnego v0.0.0-20190506202455-c2c609116ad0/go.mod h1:P4f4MSk7h52F2PK0lCapn5+fu47Uf8aRdxDSqgezxZE=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
 github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
@@ -104,12 +172,16 @@ github.com/git-lfs/wildmatch v1.0.4/go.mod h1:SdHAGnApDpnFYQ0vAxbniWR0sn7yLJ3QXo
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
 github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
 github.com/gobuffalo/envy v1.7.1 h1:OQl5ys5MBea7OGCdvPbBJWRgnhC/fGona6QKfvFeau8=
@@ -133,33 +205,64 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE=
+github.com/google/go-replayers/httpreplay v0.1.0/go.mod h1:YKZViNhiGgqdBlUbI2MwGpq4pXxNmhJLPHQ7cv2b5no=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc h1:DLpL8pWq0v4JYoRpEhDfsJhhJyGKCcQM2WPW2TJs31c=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c h1:lIC98ZUNah83ky7d9EXktLFe4H7Nwus59dTOLXr8xAI=
+github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
+github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww=
+github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
@@ -192,6 +295,9 @@ github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrO
 github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
 github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
 github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
+github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -230,6 +336,7 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
+github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/libgit2/git2go v0.0.0-20190104134018-ecaeb7a21d47 h1:HDt7WT3kpXSHq4mlOuLzgXH9LeOK1qlhyFdKIAzxxeM=
@@ -241,6 +348,9 @@ github.com/lightstep/lightstep-tracer-go v0.15.6/go.mod h1:6AMpwZpsyCFwSovxzM78e
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
+github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
+github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
@@ -315,6 +425,8 @@ github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
@@ -378,6 +490,7 @@ github.com/ssgelm/cookiejarparser v1.0.1/go.mod h1:DUfC0mpjIzlDN7DzKjXpHj0qMI5m9
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
@@ -418,6 +531,8 @@ github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmv
 github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
 github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
 github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs=
 github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
 gitlab.com/gitlab-org/gitaly v1.68.0/go.mod h1:/pCsB918Zu5wFchZ9hLYin9WkJ2yQqdVNz0zlv5HbXg=
@@ -429,14 +544,19 @@ gitlab.com/gitlab-org/labkit v0.0.0-20200908084045-45895e129029 h1:L7b9YLsU3zBfT
 gitlab.com/gitlab-org/labkit v0.0.0-20200908084045-45895e129029/go.mod h1:SNfxkfUwVNECgtmluVayv0GWFgEjjBs5AzgsowPQuo0=
 gitlab.com/gitlab-org/labkit v0.0.0-20201113080642-40dcf811328c h1:ohBeINHT6jMF4U8DjnqyhUvENU3jK19JnOXN51Oa4o4=
 gitlab.com/gitlab-org/labkit v0.0.0-20201113080642-40dcf811328c/go.mod h1:nohrYTSLDnZix0ebXZrbZJjymRar8HeV2roWL5/jw2U=
+go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+gocloud.dev v0.20.0 h1:mbEKMfnyPV7W1Rj35R1xXfjszs9dXkwSOq2KoFr25g8=
+gocloud.dev v0.20.0/go.mod h1:+Y/RpSXrJthIOM8uFNzWp6MRu9pFPNFEEZrQMxpkfIc=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
@@ -451,6 +571,7 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49N
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -458,9 +579,13 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
 golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
 golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20191227195350-da58074b4299 h1:zQpM52jfKHG6II1ISZY1ZcpygvuSFZpLwfluuF89XOg=
 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -470,13 +595,19 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -492,18 +623,34 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
 golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -513,6 +660,7 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -534,16 +682,32 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1 h1:gZpLHxUX5BdYLA08Lj4YCJNN/jk7KtquiArPoeX0WvA=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f h1:6Sc1XOXTulBN6imkqo6XoAXDEzoQ4/ro6xy7Vn8+rOM=
 golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211 h1:9UQO31fZ+0aKQOFldThf7BKPMJTiBfWycGh/u3UoO88=
 golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
@@ -552,6 +716,7 @@ golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -563,6 +728,7 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
@@ -573,45 +739,120 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200117161641-43d50277825c h1:2EA2K0k9bcvvEDlqD8xdlOhCOqq+O/p9Voqi4x9W1YU=
 golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200317043434-63da46f3035e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200601175630-2caf76543d99/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200606014950-c42cb6316fb6/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200608174601-1b747fd94509/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.15.0 h1:yzlyyDW/J0w8yNFJIhiAJy4kq74S+1DOLdawELNxFMA=
 google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.26.0 h1:VJZ8h6E8ip82FRpQl848c5vAadxlTXrUh8RzQzSRm08=
+google.golang.org/api v0.26.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
 google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba h1:pRj9OXZbwNtbtZtOB4dLwfK4u+EVRMvP+e9zKkg2grM=
 google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200317114155-1f3552e48f24/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200325114520-5b2d0af7952b/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200603110839-e855014d5736/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482 h1:i+Aiej6cta/Frzp13/swvwz5O00kYcSe0A/C5Wd7zX8=
+google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
 google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 gopkg.in/DataDog/dd-trace-go.v1 v1.7.0 h1:7wbMayb6JXcbAS95RN7MI42W3o1BCxCcdIzZfVWBAiE=
 gopkg.in/DataDog/dd-trace-go.v1 v1.7.0/go.mod h1:DVp8HmDh8PuTu2Z0fVVlBsyWaC++fzwVCaGWylTe3tg=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
@@ -655,4 +896,8 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
diff --git a/internal/gitaly/rubyserver/balancer/balancer.go b/internal/gitaly/rubyserver/balancer/balancer.go
index c2f1a8ae775..e59778b008f 100644
--- a/internal/gitaly/rubyserver/balancer/balancer.go
+++ b/internal/gitaly/rubyserver/balancer/balancer.go
@@ -120,7 +120,7 @@ func (*builder) Scheme() string { return Scheme }
 // Build ignores its resolver.Target argument. That means it does not
 // care what "address" the caller wants to resolve. We always resolve to
 // the current list of address for local gitaly-ruby processes.
-func (b *builder) Build(_ resolver.Target, cc resolver.ClientConn, _ resolver.BuildOption) (resolver.Resolver, error) {
+func (b *builder) Build(_ resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) {
 	//nolint:staticcheck // There is no obvious way to use UpdateState() without completely replacing the current configuration
 	cc.NewServiceConfig(`{"LoadBalancingPolicy":"round_robin"}`)
 	return newGitalyResolver(cc, b.addressUpdates), nil
@@ -218,7 +218,7 @@ func newGitalyResolver(cc resolver.ClientConn, auCh chan addressUpdate) *gitalyR
 	return r
 }
 
-func (r *gitalyResolver) ResolveNow(resolver.ResolveNowOption) {
+func (r *gitalyResolver) ResolveNow(resolver.ResolveNowOptions) {
 	r.resolveNow <- struct{}{}
 }
 
diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index 622d010da93..9a94b2c9c07 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -1,6 +1,16 @@
 package hook
 
 import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
 	"os/exec"
 	"sync"
 
@@ -8,11 +18,29 @@ import (
 	"gitlab.com/gitlab-org/gitaly/internal/helper"
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 	"gitlab.com/gitlab-org/gitaly/streamio"
+
+	"gocloud.dev/blob"
+	_ "gocloud.dev/blob/fileblob"
 )
 
-func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServer) error {
-	ctx := stream.Context()
+var packCache *cache
+var hello *log.Logger
+
+func init() {
+	var err error
+	packCache, err = newCache()
+	if err != nil {
+		panic(err)
+	}
+
+	f, err := os.OpenFile("/tmp/hello", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
+	if err != nil {
+		panic(err)
+	}
+	hello = log.New(f, "", log.LstdFlags)
+}
 
+func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServer) error {
 	firstRequest, err := stream.Recv()
 	if err != nil {
 		return helper.ErrInternal(err)
@@ -23,10 +51,13 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 		return err
 	}
 
-	stdin := streamio.NewReader(func() ([]byte, error) {
+	stdin, err := ioutil.ReadAll(streamio.NewReader(func() ([]byte, error) {
 		req, err := stream.Recv()
 		return req.GetStdin(), err
-	})
+	}))
+	if err != nil {
+		return err
+	}
 
 	var m sync.Mutex
 	stdout := streamio.NewSyncWriter(&m, func(p []byte) error {
@@ -36,10 +67,161 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 		return stream.Send(&gitalypb.PackObjectsHookResponse{Stderr: p})
 	})
 
-	cmd, err := command.New(ctx, exec.Command("git", append([]string{"-C", repoPath}, firstRequest.Args...)...), stdin, stdout, stderr)
+	var e *entry
+	key := packCache.key(firstRequest.Args, stdin)
+	packCache.Lock()
+	e = packCache.entries[key]
+	if e == nil {
+		e = packCache.newEntry(key)
+		packCache.entries[key] = e
+		go e.generateResponse(repoPath, firstRequest.Args, stdin, stdout, stderr)
+	}
+	packCache.Unlock()
+
+	var iErr, iOut int
+	for done := false; !done; {
+		e.Lock()
+		for !e.done && iErr == e.NErr() && iOut == e.NOut() {
+			e.Wait()
+		}
+		done = e.done
+		e.Unlock()
+
+		if err := e.SendErr(stderr, &iErr); err != nil {
+			return err
+		}
+
+		if err := e.SendOut(stdout, &iOut); err != nil {
+			return err
+		}
+	}
+
+	return e.err
+}
+
+type WriterFunc func([]byte) (int, error)
+
+func (f WriterFunc) Write(p []byte) (int, error) { return f(p) }
+
+type cache struct {
+	*blob.Bucket
+	cacheID string
+	entries map[string]*entry
+	sync.Mutex
+}
+
+func (c *cache) key(args []string, stdin []byte) string {
+	h := sha256.New()
+	h.Write([]byte(c.cacheID))
+	fmt.Fprintf(h, "%d\x00", len(args))
+	for _, a := range args {
+		fmt.Fprintf(h, "%s\x00", a)
+	}
+	h.Write(stdin)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+func (c *cache) newEntry(key string) *entry {
+	e := &entry{}
+	e.Cond = sync.NewCond(e)
+	return e
+}
+
+type entry struct {
+	sync.Mutex
+	*sync.Cond
+	done   bool
+	err    error
+	stderr []byte
+	stdout []byte
+}
+
+func (e *entry) NOut() int { return len(e.stdout) }
+func (e *entry) NErr() int { return len(e.stderr) }
+
+func (e *entry) SendOut(w io.Writer, pos *int) error {
+	e.Lock()
+	buf := e.stdout
+	e.Unlock()
+	if *pos > len(buf) {
+		return errors.New("stdout: invalid pos")
+	}
+	n, err := w.Write(buf[*pos:])
+	*pos += n
+	return err
+}
+
+func (e *entry) SendErr(w io.Writer, pos *int) error {
+	e.Lock()
+	buf := e.stderr
+	e.Unlock()
+	if *pos > len(buf) {
+		return errors.New("stderrt: invalid pos")
+	}
+	n, err := w.Write(buf[*pos:])
+	*pos += n
+	return err
+}
+
+func newCache() (*cache, error) {
+	c := &cache{
+		entries: make(map[string]*entry),
+	}
+	buf := make([]byte, 8)
+	if _, err := io.ReadFull(rand.Reader, buf); err != nil {
+		return nil, err
+	}
+	c.cacheID = fmt.Sprintf("%x", buf)
+
+	var err error
+	c.Bucket, err = blob.OpenBucket(context.Background(), "file:///tmp/gitaly-cache")
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	return cmd.Wait()
+	return c, nil
+}
+
+func (e *entry) generateResponse(repoPath string, args []string, stdin []byte, stdout io.Writer, stderr io.Writer) {
+	err := func() error {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		cmd, err := command.New(
+			ctx,
+			exec.Command("git", append([]string{"-C", repoPath}, args...)...),
+			bytes.NewReader(stdin),
+			WriterFunc(e.WriteStdout),
+			WriterFunc(e.WriteStderr),
+		)
+		if err != nil {
+			return err
+		}
+
+		return cmd.Wait()
+	}()
+
+	e.Lock()
+	e.done = true
+	if err != nil {
+		e.err = err
+	}
+	e.Unlock()
+	e.Broadcast()
+}
+
+func (e *entry) WriteStderr(p []byte) (int, error) {
+	e.Lock()
+	e.stderr = append(e.stderr, p...)
+	e.Unlock()
+	e.Broadcast()
+	return len(p), nil
+}
+
+func (e *entry) WriteStdout(p []byte) (int, error) {
+	e.Lock()
+	e.stdout = append(e.stdout, p...)
+	e.Unlock()
+	e.Broadcast()
+	return len(p), nil
 }
-- 
GitLab


From 49f0b6422ddb459b64f4ec494a673b70d378fbaa Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sat, 21 Nov 2020 15:55:12 +0100
Subject: [PATCH 06/22] Delete on error

---
 internal/gitaly/service/hook/pack_objects.go | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index 9a94b2c9c07..4860f884cb5 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -122,12 +122,14 @@ func (c *cache) key(args []string, stdin []byte) string {
 }
 
 func (c *cache) newEntry(key string) *entry {
-	e := &entry{}
+	e := &entry{c: c, key: key}
 	e.Cond = sync.NewCond(e)
 	return e
 }
 
 type entry struct {
+	c   *cache
+	key string
 	sync.Mutex
 	*sync.Cond
 	done   bool
@@ -136,6 +138,14 @@ type entry struct {
 	stdout []byte
 }
 
+func (e *entry) delete() {
+	e.c.Lock()
+	if e.c.entries[e.key] == e {
+		delete(e.c.entries, e.key)
+	}
+	e.c.Unlock()
+}
+
 func (e *entry) NOut() int { return len(e.stdout) }
 func (e *entry) NErr() int { return len(e.stderr) }
 
@@ -203,8 +213,9 @@ func (e *entry) generateResponse(repoPath string, args []string, stdin []byte, s
 
 	e.Lock()
 	e.done = true
+	e.err = err
 	if err != nil {
-		e.err = err
+		e.delete()
 	}
 	e.Unlock()
 	e.Broadcast()
-- 
GitLab


From 6bb1d1b4e9489ea07dad03ec218e35ba4bf136f9 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sat, 21 Nov 2020 16:56:55 +0100
Subject: [PATCH 07/22] Cache stdout in object storage

---
 go.sum                                       |   2 +
 internal/gitaly/service/hook/pack_objects.go | 147 +++++++++++++++----
 2 files changed, 122 insertions(+), 27 deletions(-)

diff --git a/go.sum b/go.sum
index 53d83ef1d77..fd7d3433111 100644
--- a/go.sum
+++ b/go.sum
@@ -42,6 +42,7 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.9.0 h1:oXnZyBjHB6hC8TnSle0AWW6pGJ29EuSo5ww+SFmdNBg=
 cloud.google.com/go/storage v1.9.0/go.mod h1:m+/etGaqZbylxaNT876QGXqEHp4PR2Rq5GMqICWb9bU=
 contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA=
 contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw=
@@ -260,6 +261,7 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
 github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/wire v0.4.0 h1:kXcsA/rIGzJImVqPdhfnr6q0xsS9gU0515q1EPpJ9fE=
 github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
 github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww=
 github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index 4860f884cb5..132c47e745c 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -1,6 +1,7 @@
 package hook
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"crypto/rand"
@@ -21,11 +22,14 @@ import (
 
 	"gocloud.dev/blob"
 	_ "gocloud.dev/blob/fileblob"
+	_ "gocloud.dev/blob/gcsblob"
 )
 
 var packCache *cache
 var hello *log.Logger
 
+const chunkSize = 20 * 1000 * 1000
+
 func init() {
 	var err error
 	packCache, err = newCache()
@@ -41,6 +45,7 @@ func init() {
 }
 
 func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServer) error {
+	ctx := stream.Context()
 	firstRequest, err := stream.Recv()
 	if err != nil {
 		return helper.ErrInternal(err)
@@ -87,11 +92,14 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 		done = e.done
 		e.Unlock()
 
-		if err := e.SendErr(stderr, &iErr); err != nil {
+		var err error
+		iErr, err = e.SendErr(ctx, stderr, iErr)
+		if err != nil {
 			return err
 		}
 
-		if err := e.SendOut(stdout, &iOut); err != nil {
+		iOut, err = e.SendOut(ctx, stdout, iOut)
+		if err != nil {
 			return err
 		}
 	}
@@ -108,6 +116,7 @@ type cache struct {
 	cacheID string
 	entries map[string]*entry
 	sync.Mutex
+	nextEntryID int
 }
 
 func (c *cache) key(args []string, stdin []byte) string {
@@ -122,8 +131,13 @@ func (c *cache) key(args []string, stdin []byte) string {
 }
 
 func (c *cache) newEntry(key string) *entry {
-	e := &entry{c: c, key: key}
+	e := &entry{
+		c:   c,
+		key: key,
+		id:  c.nextEntryID,
+	}
 	e.Cond = sync.NewCond(e)
+	c.nextEntryID++
 	return e
 }
 
@@ -132,10 +146,37 @@ type entry struct {
 	key string
 	sync.Mutex
 	*sync.Cond
-	done   bool
-	err    error
-	stderr []byte
-	stdout []byte
+	done             bool
+	err              error
+	stderr           []byte
+	nOutChunks       int
+	currentChunkSize int
+	outWriter        *blob.Writer
+	id               int
+}
+
+func (e *entry) CloseWriter() error {
+	e.Lock()
+	defer e.Unlock()
+
+	if e.outWriter == nil {
+		return nil
+	}
+
+	if err := e.outWriter.Close(); err != nil {
+		return err
+	}
+
+	e.outWriter = nil
+	e.nOutChunks++
+	e.currentChunkSize = 0
+	e.Broadcast()
+	return nil
+}
+
+func (e *entry) chunkKey(i int) string {
+	chunk := fmt.Sprintf("%02x", i)
+	return fmt.Sprintf("%s/%s/%s.%d", e.key, chunk[len(chunk)-2:], chunk, e.id)
 }
 
 func (e *entry) delete() {
@@ -146,31 +187,38 @@ func (e *entry) delete() {
 	e.c.Unlock()
 }
 
-func (e *entry) NOut() int { return len(e.stdout) }
+func (e *entry) NOut() int { return e.nOutChunks }
 func (e *entry) NErr() int { return len(e.stderr) }
 
-func (e *entry) SendOut(w io.Writer, pos *int) error {
+func (e *entry) SendOut(ctx context.Context, w io.Writer, pos int) (int, error) {
 	e.Lock()
-	buf := e.stdout
+	nOut := e.NOut()
 	e.Unlock()
-	if *pos > len(buf) {
-		return errors.New("stdout: invalid pos")
+
+	for i := pos; i < nOut; i++ {
+		r, err := e.c.Bucket.NewReader(ctx, e.chunkKey(i), nil)
+		if err != nil {
+			return i, err
+		}
+		defer r.Close()
+
+		if _, err := io.Copy(w, r); err != nil {
+			return i, err
+		}
 	}
-	n, err := w.Write(buf[*pos:])
-	*pos += n
-	return err
+	return nOut, nil
 }
 
-func (e *entry) SendErr(w io.Writer, pos *int) error {
+func (e *entry) SendErr(ctx context.Context, w io.Writer, pos int) (int, error) {
 	e.Lock()
 	buf := e.stderr
 	e.Unlock()
-	if *pos > len(buf) {
-		return errors.New("stderrt: invalid pos")
+	if pos > len(buf) {
+		return 0, errors.New("stderrt: invalid pos")
 	}
-	n, err := w.Write(buf[*pos:])
-	*pos += n
-	return err
+
+	n, err := w.Write(buf[pos:])
+	return pos + n, err
 }
 
 func newCache() (*cache, error) {
@@ -183,8 +231,13 @@ func newCache() (*cache, error) {
 	}
 	c.cacheID = fmt.Sprintf("%x", buf)
 
+	cacheURL := "file:///tmp/gitaly-cache"
+	if v := os.Getenv("GITALY_CACHE_BUCKET"); v != "" {
+		cacheURL = v
+	}
+
 	var err error
-	c.Bucket, err = blob.OpenBucket(context.Background(), "file:///tmp/gitaly-cache")
+	c.Bucket, err = blob.OpenBucket(context.Background(), cacheURL)
 	if err != nil {
 		return nil, err
 	}
@@ -197,18 +250,26 @@ func (e *entry) generateResponse(repoPath string, args []string, stdin []byte, s
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
 
+		stdout := bufio.NewWriter(WriterFunc(e.WriteStdout))
 		cmd, err := command.New(
 			ctx,
 			exec.Command("git", append([]string{"-C", repoPath}, args...)...),
 			bytes.NewReader(stdin),
-			WriterFunc(e.WriteStdout),
+			stdout,
 			WriterFunc(e.WriteStderr),
 		)
 		if err != nil {
 			return err
 		}
 
-		return cmd.Wait()
+		if err := cmd.Wait(); err != nil {
+			return err
+		}
+		if err := stdout.Flush(); err != nil {
+			return err
+		}
+
+		return e.CloseWriter()
 	}()
 
 	e.Lock()
@@ -230,9 +291,41 @@ func (e *entry) WriteStderr(p []byte) (int, error) {
 }
 
 func (e *entry) WriteStdout(p []byte) (int, error) {
+	w, err := e.getOutWriter()
+	if err != nil {
+		return 0, err
+	}
+	n, err := w.Write(p)
+	if err != nil {
+		return n, err
+	}
+
 	e.Lock()
-	e.stdout = append(e.stdout, p...)
+	e.currentChunkSize += n
 	e.Unlock()
-	e.Broadcast()
-	return len(p), nil
+
+	if e.chunkIsFull() {
+		return n, e.CloseWriter()
+	}
+
+	return n, nil
+}
+
+func (e *entry) getOutWriter() (io.Writer, error) {
+	e.Lock()
+	defer e.Unlock()
+	if e.outWriter == nil {
+		var err error
+		e.outWriter, err = e.c.Bucket.NewWriter(context.Background(), e.chunkKey(e.nOutChunks), nil)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return e.outWriter, nil
+}
+
+func (e *entry) chunkIsFull() bool {
+	e.Lock()
+	defer e.Unlock()
+	return e.currentChunkSize > chunkSize
 }
-- 
GitLab


From c7a6ed3f96f8b6c3dfb536e7d3619e05edecd38e Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sat, 21 Nov 2020 17:46:09 +0100
Subject: [PATCH 08/22] More structs

---
 internal/gitaly/service/hook/pack_objects.go | 186 ++++++++++---------
 1 file changed, 100 insertions(+), 86 deletions(-)

diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index 132c47e745c..8f5cdbc5946 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -83,23 +83,20 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 	}
 	packCache.Unlock()
 
-	var iErr, iOut int
+	var posErr, posOut int
 	for done := false; !done; {
 		e.Lock()
-		for !e.done && iErr == e.NErr() && iOut == e.NOut() {
+		for !e.done && posErr == e.stderr.SizeLocked() && posOut == e.stdout.SizeLocked() {
 			e.Wait()
 		}
 		done = e.done
 		e.Unlock()
 
-		var err error
-		iErr, err = e.SendErr(ctx, stderr, iErr)
-		if err != nil {
+		if err := e.stderr.Send(ctx, stderr, &posErr); err != nil {
 			return err
 		}
 
-		iOut, err = e.SendOut(ctx, stdout, iOut)
-		if err != nil {
+		if err := e.stdout.Send(ctx, stdout, &posOut); err != nil {
 			return err
 		}
 	}
@@ -137,6 +134,8 @@ func (c *cache) newEntry(key string) *entry {
 		id:  c.nextEntryID,
 	}
 	e.Cond = sync.NewCond(e)
+	e.stderr = &memBuffer{entry: e}
+	e.stdout = &blobBuffer{entry: e}
 	c.nextEntryID++
 	return e
 }
@@ -146,32 +145,11 @@ type entry struct {
 	key string
 	sync.Mutex
 	*sync.Cond
-	done             bool
-	err              error
-	stderr           []byte
-	nOutChunks       int
-	currentChunkSize int
-	outWriter        *blob.Writer
-	id               int
-}
-
-func (e *entry) CloseWriter() error {
-	e.Lock()
-	defer e.Unlock()
-
-	if e.outWriter == nil {
-		return nil
-	}
-
-	if err := e.outWriter.Close(); err != nil {
-		return err
-	}
-
-	e.outWriter = nil
-	e.nOutChunks++
-	e.currentChunkSize = 0
-	e.Broadcast()
-	return nil
+	done   bool
+	err    error
+	stderr *memBuffer
+	stdout *blobBuffer
+	id     int
 }
 
 func (e *entry) chunkKey(i int) string {
@@ -187,38 +165,32 @@ func (e *entry) delete() {
 	e.c.Unlock()
 }
 
-func (e *entry) NOut() int { return e.nOutChunks }
-func (e *entry) NErr() int { return len(e.stderr) }
-
-func (e *entry) SendOut(ctx context.Context, w io.Writer, pos int) (int, error) {
-	e.Lock()
-	nOut := e.NOut()
-	e.Unlock()
-
-	for i := pos; i < nOut; i++ {
-		r, err := e.c.Bucket.NewReader(ctx, e.chunkKey(i), nil)
-		if err != nil {
-			return i, err
-		}
-		defer r.Close()
+type memBuffer struct {
+	buf []byte
+	*entry
+}
 
-		if _, err := io.Copy(w, r); err != nil {
-			return i, err
-		}
-	}
-	return nOut, nil
+func (mb *memBuffer) Write(p []byte) (int, error) {
+	mb.Lock()
+	mb.buf = append(mb.buf, p...)
+	mb.Unlock()
+	mb.Broadcast()
+	return len(p), nil
 }
 
-func (e *entry) SendErr(ctx context.Context, w io.Writer, pos int) (int, error) {
-	e.Lock()
-	buf := e.stderr
-	e.Unlock()
-	if pos > len(buf) {
-		return 0, errors.New("stderrt: invalid pos")
+func (mb *memBuffer) SizeLocked() int { return len(mb.buf) }
+
+func (mb *memBuffer) Send(ctx context.Context, w io.Writer, pos *int) error {
+	mb.Lock()
+	buf := mb.buf
+	mb.Unlock()
+	if *pos > len(buf) {
+		return errors.New("memBuffer: invalid pos")
 	}
 
-	n, err := w.Write(buf[pos:])
-	return pos + n, err
+	n, err := w.Write(buf[*pos:])
+	*pos += n
+	return err
 }
 
 func newCache() (*cache, error) {
@@ -250,13 +222,13 @@ func (e *entry) generateResponse(repoPath string, args []string, stdin []byte, s
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
 
-		stdout := bufio.NewWriter(WriterFunc(e.WriteStdout))
+		stdout := bufio.NewWriter(e.stdout)
 		cmd, err := command.New(
 			ctx,
 			exec.Command("git", append([]string{"-C", repoPath}, args...)...),
 			bytes.NewReader(stdin),
 			stdout,
-			WriterFunc(e.WriteStderr),
+			e.stderr,
 		)
 		if err != nil {
 			return err
@@ -269,7 +241,7 @@ func (e *entry) generateResponse(repoPath string, args []string, stdin []byte, s
 			return err
 		}
 
-		return e.CloseWriter()
+		return e.stdout.CloseWriter()
 	}()
 
 	e.Lock()
@@ -282,16 +254,58 @@ func (e *entry) generateResponse(repoPath string, args []string, stdin []byte, s
 	e.Broadcast()
 }
 
-func (e *entry) WriteStderr(p []byte) (int, error) {
-	e.Lock()
-	e.stderr = append(e.stderr, p...)
-	e.Unlock()
-	e.Broadcast()
-	return len(p), nil
+type blobBuffer struct {
+	*entry
+	w                *blob.Writer
+	nChunks          int
+	currentChunkSize int
+}
+
+func (bb *blobBuffer) CloseWriter() error {
+	bb.Lock()
+	defer bb.Unlock()
+
+	if bb.w == nil {
+		return nil
+	}
+
+	if err := bb.w.Close(); err != nil {
+		return err
+	}
+
+	bb.w = nil
+	bb.nChunks++
+	bb.currentChunkSize = 0
+	bb.Broadcast()
+	return nil
+}
+
+func (bb *blobBuffer) SizeLocked() int { return bb.nChunks }
+
+func (bb *blobBuffer) Send(ctx context.Context, w io.Writer, pos *int) error {
+	bb.Lock()
+	nChunks := bb.SizeLocked()
+	bb.Unlock()
+
+	var err error
+	for ; *pos < nChunks && err == nil; *pos++ {
+		err = func() error {
+			r, err := bb.c.Bucket.NewReader(ctx, bb.chunkKey(*pos), nil)
+			if err != nil {
+				return err
+			}
+			defer r.Close()
+
+			_, err = io.Copy(w, r)
+			return err
+		}()
+	}
+
+	return err
 }
 
-func (e *entry) WriteStdout(p []byte) (int, error) {
-	w, err := e.getOutWriter()
+func (bb *blobBuffer) Write(p []byte) (int, error) {
+	w, err := bb.writer()
 	if err != nil {
 		return 0, err
 	}
@@ -300,32 +314,32 @@ func (e *entry) WriteStdout(p []byte) (int, error) {
 		return n, err
 	}
 
-	e.Lock()
-	e.currentChunkSize += n
-	e.Unlock()
+	bb.Lock()
+	bb.currentChunkSize += n
+	bb.Unlock()
 
-	if e.chunkIsFull() {
-		return n, e.CloseWriter()
+	if bb.chunkIsFull() {
+		return n, bb.CloseWriter()
 	}
 
 	return n, nil
 }
 
-func (e *entry) getOutWriter() (io.Writer, error) {
-	e.Lock()
-	defer e.Unlock()
-	if e.outWriter == nil {
+func (bb *blobBuffer) writer() (io.Writer, error) {
+	bb.Lock()
+	defer bb.Unlock()
+	if bb.w == nil {
 		var err error
-		e.outWriter, err = e.c.Bucket.NewWriter(context.Background(), e.chunkKey(e.nOutChunks), nil)
+		bb.w, err = bb.entry.c.Bucket.NewWriter(context.Background(), bb.chunkKey(bb.nChunks), nil)
 		if err != nil {
 			return nil, err
 		}
 	}
-	return e.outWriter, nil
+	return bb.w, nil
 }
 
-func (e *entry) chunkIsFull() bool {
-	e.Lock()
-	defer e.Unlock()
-	return e.currentChunkSize > chunkSize
+func (bb *blobBuffer) chunkIsFull() bool {
+	bb.Lock()
+	defer bb.Unlock()
+	return bb.currentChunkSize > chunkSize
 }
-- 
GitLab


From 28c3dc4a19deae13756804c4f14f8d916cbce057 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sat, 21 Nov 2020 18:02:40 +0100
Subject: [PATCH 09/22] Rename function

---
 internal/gitaly/service/hook/pack_objects.go | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index 8f5cdbc5946..a1017ea1b35 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -104,10 +104,6 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 	return e.err
 }
 
-type WriterFunc func([]byte) (int, error)
-
-func (f WriterFunc) Write(p []byte) (int, error) { return f(p) }
-
 type cache struct {
 	*blob.Bucket
 	cacheID string
@@ -241,7 +237,7 @@ func (e *entry) generateResponse(repoPath string, args []string, stdin []byte, s
 			return err
 		}
 
-		return e.stdout.CloseWriter()
+		return e.stdout.FlushChunk()
 	}()
 
 	e.Lock()
@@ -261,7 +257,7 @@ type blobBuffer struct {
 	currentChunkSize int
 }
 
-func (bb *blobBuffer) CloseWriter() error {
+func (bb *blobBuffer) FlushChunk() error {
 	bb.Lock()
 	defer bb.Unlock()
 
@@ -319,7 +315,7 @@ func (bb *blobBuffer) Write(p []byte) (int, error) {
 	bb.Unlock()
 
 	if bb.chunkIsFull() {
-		return n, bb.CloseWriter()
+		return n, bb.FlushChunk()
 	}
 
 	return n, nil
-- 
GitLab


From 6ae60ef50b0975fe4278f03c57f338a128d41981 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sat, 21 Nov 2020 20:12:41 +0100
Subject: [PATCH 10/22] renames

---
 internal/gitaly/service/hook/pack_objects.go | 26 +++++++++++---------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index a1017ea1b35..ec8aca97e85 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -86,17 +86,17 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 	var posErr, posOut int
 	for done := false; !done; {
 		e.Lock()
-		for !e.done && posErr == e.stderr.SizeLocked() && posOut == e.stdout.SizeLocked() {
+		for !e.done && posErr == e.stderr.len() && posOut == e.stdout.len() {
 			e.Wait()
 		}
 		done = e.done
 		e.Unlock()
 
-		if err := e.stderr.Send(ctx, stderr, &posErr); err != nil {
+		if err := e.stderr.send(ctx, stderr, &posErr); err != nil {
 			return err
 		}
 
-		if err := e.stdout.Send(ctx, stdout, &posOut); err != nil {
+		if err := e.stdout.send(ctx, stdout, &posOut); err != nil {
 			return err
 		}
 	}
@@ -174,9 +174,10 @@ func (mb *memBuffer) Write(p []byte) (int, error) {
 	return len(p), nil
 }
 
-func (mb *memBuffer) SizeLocked() int { return len(mb.buf) }
+// Caller must hold lock when calling len.
+func (mb *memBuffer) len() int { return len(mb.buf) }
 
-func (mb *memBuffer) Send(ctx context.Context, w io.Writer, pos *int) error {
+func (mb *memBuffer) send(ctx context.Context, w io.Writer, pos *int) error {
 	mb.Lock()
 	buf := mb.buf
 	mb.Unlock()
@@ -237,7 +238,7 @@ func (e *entry) generateResponse(repoPath string, args []string, stdin []byte, s
 			return err
 		}
 
-		return e.stdout.FlushChunk()
+		return e.stdout.flushChunk()
 	}()
 
 	e.Lock()
@@ -257,7 +258,7 @@ type blobBuffer struct {
 	currentChunkSize int
 }
 
-func (bb *blobBuffer) FlushChunk() error {
+func (bb *blobBuffer) flushChunk() error {
 	bb.Lock()
 	defer bb.Unlock()
 
@@ -276,11 +277,12 @@ func (bb *blobBuffer) FlushChunk() error {
 	return nil
 }
 
-func (bb *blobBuffer) SizeLocked() int { return bb.nChunks }
+// Caller must hold lock when calling len.
+func (bb *blobBuffer) len() int { return bb.nChunks }
 
-func (bb *blobBuffer) Send(ctx context.Context, w io.Writer, pos *int) error {
+func (bb *blobBuffer) send(ctx context.Context, w io.Writer, pos *int) error {
 	bb.Lock()
-	nChunks := bb.SizeLocked()
+	nChunks := bb.len()
 	bb.Unlock()
 
 	var err error
@@ -315,7 +317,7 @@ func (bb *blobBuffer) Write(p []byte) (int, error) {
 	bb.Unlock()
 
 	if bb.chunkIsFull() {
-		return n, bb.FlushChunk()
+		return n, bb.flushChunk()
 	}
 
 	return n, nil
@@ -326,7 +328,7 @@ func (bb *blobBuffer) writer() (io.Writer, error) {
 	defer bb.Unlock()
 	if bb.w == nil {
 		var err error
-		bb.w, err = bb.entry.c.Bucket.NewWriter(context.Background(), bb.chunkKey(bb.nChunks), nil)
+		bb.w, err = bb.entry.c.Bucket.NewWriter(context.Background(), bb.chunkKey(bb.len()), nil)
 		if err != nil {
 			return nil, err
 		}
-- 
GitLab


From 007987b7b08babf674cecda327fb0729a22709ca Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sun, 22 Nov 2020 12:56:08 +0100
Subject: [PATCH 11/22] Simplify key encoding

---
 internal/gitaly/service/hook/pack_objects.go | 24 ++++++++++++--------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index ec8aca97e85..fc6842a179a 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -6,6 +6,7 @@ import (
 	"context"
 	"crypto/rand"
 	"crypto/sha256"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -28,7 +29,7 @@ import (
 var packCache *cache
 var hello *log.Logger
 
-const chunkSize = 20 * 1000 * 1000
+const chunkSize = 50 * 1000 * 1000
 
 func init() {
 	var err error
@@ -73,7 +74,11 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 	})
 
 	var e *entry
-	key := packCache.key(firstRequest.Args, stdin)
+	key, err := packCache.key(firstRequest.Args, stdin)
+	if err != nil {
+		return err
+	}
+
 	packCache.Lock()
 	e = packCache.entries[key]
 	if e == nil {
@@ -112,15 +117,16 @@ type cache struct {
 	nextEntryID int
 }
 
-func (c *cache) key(args []string, stdin []byte) string {
+func (c *cache) key(args []string, stdin []byte) (string, error) {
 	h := sha256.New()
-	h.Write([]byte(c.cacheID))
-	fmt.Fprintf(h, "%d\x00", len(args))
-	for _, a := range args {
-		fmt.Fprintf(h, "%s\x00", a)
-	}
 	h.Write(stdin)
-	return fmt.Sprintf("%x", h.Sum(nil))
+	enc := json.NewEncoder(h)
+	for _, v := range []interface{}{c.cacheID, args} {
+		if err := enc.Encode(v); err != nil {
+			return "", err
+		}
+	}
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
 }
 
 func (c *cache) newEntry(key string) *entry {
-- 
GitLab


From d1caf9804da14f8bc226df85a5c0aa8504f984ae Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sun, 22 Nov 2020 17:40:37 +0100
Subject: [PATCH 12/22] Put stdin in file

---
 internal/gitaly/service/hook/pack_objects.go | 129 +++++++++++++------
 1 file changed, 87 insertions(+), 42 deletions(-)

diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index fc6842a179a..fc3ba4e6f3e 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -1,8 +1,6 @@
 package hook
 
 import (
-	"bufio"
-	"bytes"
 	"context"
 	"crypto/rand"
 	"crypto/sha256"
@@ -33,7 +31,7 @@ const chunkSize = 50 * 1000 * 1000
 
 func init() {
 	var err error
-	packCache, err = newCache()
+	packCache, err = newCache("file:///tmp/gitaly-cache")
 	if err != nil {
 		panic(err)
 	}
@@ -57,13 +55,16 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 		return err
 	}
 
-	stdin, err := ioutil.ReadAll(streamio.NewReader(func() ([]byte, error) {
-		req, err := stream.Recv()
-		return req.GetStdin(), err
-	}))
+	stdinHandoff := false
+	stdin, err := readPackObjectsStdin(stream)
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if !stdinHandoff {
+			stdin.Close()
+		}
+	}()
 
 	var m sync.Mutex
 	stdout := streamio.NewSyncWriter(&m, func(p []byte) error {
@@ -74,7 +75,7 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 	})
 
 	var e *entry
-	key, err := packCache.key(firstRequest.Args, stdin)
+	key, err := packCache.key(repoPath, firstRequest.Args, stdin)
 	if err != nil {
 		return err
 	}
@@ -84,7 +85,8 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 	if e == nil {
 		e = packCache.newEntry(key)
 		packCache.entries[key] = e
-		go e.generateResponse(repoPath, firstRequest.Args, stdin, stdout, stderr)
+		stdinHandoff = true
+		go e.fill(repoPath, firstRequest.Args, stdin, stdout, stderr)
 	}
 	packCache.Unlock()
 
@@ -109,6 +111,29 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 	return e.err
 }
 
+func readPackObjectsStdin(stream gitalypb.HookService_PackObjectsHookServer) (f *os.File, err error) {
+	defer func() {
+		if f != nil && err != nil {
+			f.Close()
+		}
+	}()
+
+	f, err = ioutil.TempFile("", "")
+	if err != nil {
+		return nil, err
+	}
+	if err := os.Remove(f.Name()); err != nil {
+		return nil, err
+	}
+
+	_, err = io.Copy(f, streamio.NewReader(func() ([]byte, error) {
+		req, err := stream.Recv()
+		return req.GetStdin(), err
+	}))
+
+	return f, err
+}
+
 type cache struct {
 	*blob.Bucket
 	cacheID string
@@ -117,11 +142,17 @@ type cache struct {
 	nextEntryID int
 }
 
-func (c *cache) key(args []string, stdin []byte) (string, error) {
+func (c *cache) key(repoPath string, args []string, stdin io.ReadSeeker) (string, error) {
 	h := sha256.New()
-	h.Write(stdin)
+	if _, err := stdin.Seek(0, io.SeekStart); err != nil {
+		return "", err
+	}
+	if _, err := io.Copy(h, stdin); err != nil {
+		return "", err
+	}
+
 	enc := json.NewEncoder(h)
-	for _, v := range []interface{}{c.cacheID, args} {
+	for _, v := range []interface{}{c.cacheID, args, repoPath} {
 		if err := enc.Encode(v); err != nil {
 			return "", err
 		}
@@ -133,12 +164,13 @@ func (c *cache) newEntry(key string) *entry {
 	e := &entry{
 		c:   c,
 		key: key,
-		id:  c.nextEntryID,
 	}
 	e.Cond = sync.NewCond(e)
 	e.stderr = &memBuffer{entry: e}
 	e.stdout = &blobBuffer{entry: e}
+	e.id = c.nextEntryID
 	c.nextEntryID++
+
 	return e
 }
 
@@ -156,7 +188,7 @@ type entry struct {
 
 func (e *entry) chunkKey(i int) string {
 	chunk := fmt.Sprintf("%02x", i)
-	return fmt.Sprintf("%s/%s/%s.%d", e.key, chunk[len(chunk)-2:], chunk, e.id)
+	return fmt.Sprintf("%s/%s/%s/%d", e.key, chunk[len(chunk)-2:], chunk, e.id)
 }
 
 func (e *entry) delete() {
@@ -196,7 +228,7 @@ func (mb *memBuffer) send(ctx context.Context, w io.Writer, pos *int) error {
 	return err
 }
 
-func newCache() (*cache, error) {
+func newCache(cacheURL string) (*cache, error) {
 	c := &cache{
 		entries: make(map[string]*entry),
 	}
@@ -206,7 +238,6 @@ func newCache() (*cache, error) {
 	}
 	c.cacheID = fmt.Sprintf("%x", buf)
 
-	cacheURL := "file:///tmp/gitaly-cache"
 	if v := os.Getenv("GITALY_CACHE_BUCKET"); v != "" {
 		cacheURL = v
 	}
@@ -220,32 +251,45 @@ func newCache() (*cache, error) {
 	return c, nil
 }
 
-func (e *entry) generateResponse(repoPath string, args []string, stdin []byte, stdout io.Writer, stderr io.Writer) {
-	err := func() error {
-		ctx, cancel := context.WithCancel(context.Background())
-		defer cancel()
-
-		stdout := bufio.NewWriter(e.stdout)
-		cmd, err := command.New(
-			ctx,
-			exec.Command("git", append([]string{"-C", repoPath}, args...)...),
-			bytes.NewReader(stdin),
-			stdout,
-			e.stderr,
-		)
-		if err != nil {
-			return err
-		}
+type readSeekCloser interface {
+	io.ReadSeeker
+	io.Closer
+}
 
-		if err := cmd.Wait(); err != nil {
-			return err
-		}
-		if err := stdout.Flush(); err != nil {
-			return err
-		}
+type writeFlusher interface {
+	io.Writer
+	Flush() error
+}
 
-		return e.stdout.flushChunk()
-	}()
+func runPackObjects(ctx context.Context, repoPath string, args []string, stdin readSeekCloser, stdout writeFlusher, stderr io.Writer) error {
+	defer stdin.Close()
+	if _, err := stdin.Seek(0, io.SeekStart); err != nil {
+		return err
+	}
+
+	cmd, err := command.New(
+		ctx,
+		exec.Command("git", append([]string{"-C", repoPath}, args...)...),
+		stdin,
+		stdout,
+		stderr,
+	)
+	if err != nil {
+		return err
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return err
+	}
+
+	return stdout.Flush()
+}
+
+func (e *entry) fill(repoPath string, args []string, stdin readSeekCloser, stdout io.Writer, stderr io.Writer) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	err := runPackObjects(ctx, repoPath, args, stdin, e.stdout, e.stderr)
 
 	e.Lock()
 	e.done = true
@@ -254,6 +298,7 @@ func (e *entry) generateResponse(repoPath string, args []string, stdin []byte, s
 		e.delete()
 	}
 	e.Unlock()
+
 	e.Broadcast()
 }
 
@@ -264,7 +309,7 @@ type blobBuffer struct {
 	currentChunkSize int
 }
 
-func (bb *blobBuffer) flushChunk() error {
+func (bb *blobBuffer) Flush() error {
 	bb.Lock()
 	defer bb.Unlock()
 
@@ -323,7 +368,7 @@ func (bb *blobBuffer) Write(p []byte) (int, error) {
 	bb.Unlock()
 
 	if bb.chunkIsFull() {
-		return n, bb.flushChunk()
+		return n, bb.Flush()
 	}
 
 	return n, nil
-- 
GitLab


From d448532be7d528af9cd856a102e613590d38a6cc Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sun, 22 Nov 2020 17:51:48 +0100
Subject: [PATCH 13/22] Refactor

---
 internal/gitaly/service/hook/pack_objects.go | 43 +++++++++++++-------
 1 file changed, 28 insertions(+), 15 deletions(-)

diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index fc3ba4e6f3e..f467e221b72 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -61,6 +61,9 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 		return err
 	}
 	defer func() {
+		// If we have a cache miss and we win the race to spawn a goroutine that
+		// fills the cache, that goroutine becomes responsible for closing the
+		// stdin tempfile.
 		if !stdinHandoff {
 			stdin.Close()
 		}
@@ -111,27 +114,29 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 	return e.err
 }
 
-func readPackObjectsStdin(stream gitalypb.HookService_PackObjectsHookServer) (f *os.File, err error) {
+func readPackObjectsStdin(stream gitalypb.HookService_PackObjectsHookServer) (_ *os.File, err error) {
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		return nil, err
+	}
 	defer func() {
-		if f != nil && err != nil {
+		if err != nil {
 			f.Close()
 		}
 	}()
 
-	f, err = ioutil.TempFile("", "")
-	if err != nil {
-		return nil, err
-	}
 	if err := os.Remove(f.Name()); err != nil {
 		return nil, err
 	}
 
-	_, err = io.Copy(f, streamio.NewReader(func() ([]byte, error) {
+	if _, err := io.Copy(f, streamio.NewReader(func() ([]byte, error) {
 		req, err := stream.Recv()
 		return req.GetStdin(), err
-	}))
+	})); err != nil {
+		return nil, err
+	}
 
-	return f, err
+	return f, nil
 }
 
 type cache struct {
@@ -309,6 +314,7 @@ type blobBuffer struct {
 	currentChunkSize int
 }
 
+// Flush finishes the current chunk.
 func (bb *blobBuffer) Flush() error {
 	bb.Lock()
 	defer bb.Unlock()
@@ -377,13 +383,20 @@ func (bb *blobBuffer) Write(p []byte) (int, error) {
 func (bb *blobBuffer) writer() (io.Writer, error) {
 	bb.Lock()
 	defer bb.Unlock()
-	if bb.w == nil {
-		var err error
-		bb.w, err = bb.entry.c.Bucket.NewWriter(context.Background(), bb.chunkKey(bb.len()), nil)
-		if err != nil {
-			return nil, err
-		}
+
+	if bb.w != nil {
+		return bb.w, nil
 	}
+
+	var err error
+	if bb.w, err = bb.entry.c.Bucket.NewWriter(
+		context.Background(),
+		bb.chunkKey(bb.len()),
+		&blob.WriterOptions{ContentType: "application/octet-stream"},
+	); err != nil {
+		return nil, err
+	}
+
 	return bb.w, nil
 }
 
-- 
GitLab


From fd18d8e0f39586de6b50a0d03470c23b24855577 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sun, 22 Nov 2020 18:07:01 +0100
Subject: [PATCH 14/22] lru

---
 go.mod                                       |  1 +
 go.sum                                       |  5 ++
 internal/gitaly/service/hook/pack_objects.go | 88 ++++++++++----------
 3 files changed, 50 insertions(+), 44 deletions(-)

diff --git a/go.mod b/go.mod
index ecb3003f2ec..7dff7180d18 100644
--- a/go.mod
+++ b/go.mod
@@ -6,6 +6,7 @@ require (
 	github.com/getsentry/sentry-go v0.7.0
 	github.com/git-lfs/git-lfs v1.5.1-0.20200916154635-9ea4eed5b112
 	github.com/gogo/protobuf v1.2.1 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e
 	github.com/golang/protobuf v1.4.2
 	github.com/google/uuid v1.1.1
 	github.com/grpc-ecosystem/go-grpc-middleware v1.0.0
diff --git a/go.sum b/go.sum
index fd7d3433111..f6ecf8412c0 100644
--- a/go.sum
+++ b/go.sum
@@ -52,9 +52,11 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/Azure/azure-amqp-common-go/v3 v3.0.0/go.mod h1:SY08giD/XbhTz07tJdpw1SoxQXHPN30+DI3Z04SYqyg=
 github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
+github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY=
 github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc=
 github.com/Azure/azure-sdk-for-go v37.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-service-bus-go v0.10.1/go.mod h1:E/FOceuKAFUfpbIJDKWz/May6guE+eGibfGT6q+n1to=
+github.com/Azure/azure-storage-blob-go v0.9.0 h1:kORqvzXP8ORhKbW13FflGUaSE5CMyDWun9UwMxY8gPs=
 github.com/Azure/azure-storage-blob-go v0.9.0/go.mod h1:8UBPbiOhrMQ4pLPi3gA1tXnpjrS76UYE/fo5A40vf4g=
 github.com/Azure/go-amqp v0.12.6/go.mod h1:qApuH6OFTSKZFmCOxccvAv5rLizBQf4v8pRmG138DPo=
 github.com/Azure/go-amqp v0.12.7/go.mod h1:qApuH6OFTSKZFmCOxccvAv5rLizBQf4v8pRmG138DPo=
@@ -95,6 +97,7 @@ github.com/avast/retry-go v2.4.2+incompatible h1:+ZjCypQT/CyP0kyJO2EcU4d/ZEJWSbP
 github.com/avast/retry-go v2.4.2+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.31.13 h1:UeWMTRTL0XAKLR7vxDL4/u7KOtz/LtfJr+lXtxN4YEQ=
 github.com/aws/aws-sdk-go v1.31.13/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
@@ -299,6 +302,7 @@ github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem
 github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
@@ -352,6 +356,7 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
 github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
+github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI=
 github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index f467e221b72..667b88c1ee4 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"os"
 	"os/exec"
 	"sync"
@@ -19,15 +18,15 @@ import (
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 	"gitlab.com/gitlab-org/gitaly/streamio"
 
+	"github.com/golang/groupcache/lru"
 	"gocloud.dev/blob"
+	_ "gocloud.dev/blob/azureblob"
 	_ "gocloud.dev/blob/fileblob"
 	_ "gocloud.dev/blob/gcsblob"
+	_ "gocloud.dev/blob/s3blob"
 )
 
 var packCache *cache
-var hello *log.Logger
-
-const chunkSize = 50 * 1000 * 1000
 
 func init() {
 	var err error
@@ -35,12 +34,6 @@ func init() {
 	if err != nil {
 		panic(err)
 	}
-
-	f, err := os.OpenFile("/tmp/hello", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
-	if err != nil {
-		panic(err)
-	}
-	hello = log.New(f, "", log.LstdFlags)
 }
 
 func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServer) error {
@@ -77,17 +70,17 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 		return stream.Send(&gitalypb.PackObjectsHookResponse{Stderr: p})
 	})
 
-	var e *entry
 	key, err := packCache.key(repoPath, firstRequest.Args, stdin)
 	if err != nil {
 		return err
 	}
 
+	var e *entry
 	packCache.Lock()
-	e = packCache.entries[key]
-	if e == nil {
-		e = packCache.newEntry(key)
-		packCache.entries[key] = e
+	if lruEntry, ok := packCache.entries.Get(key); ok {
+		e = lruEntry.(*entry)
+	} else {
+		e = packCache.createEntry(key)
 		stdinHandoff = true
 		go e.fill(repoPath, firstRequest.Args, stdin, stdout, stderr)
 	}
@@ -142,9 +135,34 @@ func readPackObjectsStdin(stream gitalypb.HookService_PackObjectsHookServer) (_
 type cache struct {
 	*blob.Bucket
 	cacheID string
-	entries map[string]*entry
+	entries *lru.Cache
 	sync.Mutex
 	nextEntryID int
+	chunkSize   int
+}
+
+func newCache(cacheURL string) (*cache, error) {
+	c := &cache{
+		entries:   lru.New(10000),
+		chunkSize: 50 * 1024 * 1024,
+	}
+	buf := make([]byte, 8)
+	if _, err := io.ReadFull(rand.Reader, buf); err != nil {
+		return nil, err
+	}
+	c.cacheID = fmt.Sprintf("%x", buf)
+
+	if v := os.Getenv("GITALY_CACHE_BUCKET"); v != "" {
+		cacheURL = v
+	}
+
+	var err error
+	c.Bucket, err = blob.OpenBucket(context.Background(), cacheURL)
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
 }
 
 func (c *cache) key(repoPath string, args []string, stdin io.ReadSeeker) (string, error) {
@@ -165,7 +183,8 @@ func (c *cache) key(repoPath string, args []string, stdin io.ReadSeeker) (string
 	return fmt.Sprintf("%x", h.Sum(nil)), nil
 }
 
-func (c *cache) newEntry(key string) *entry {
+// createEntry expects caller to lock/unlock cache.
+func (c *cache) createEntry(key string) *entry {
 	e := &entry{
 		c:   c,
 		key: key,
@@ -173,9 +192,12 @@ func (c *cache) newEntry(key string) *entry {
 	e.Cond = sync.NewCond(e)
 	e.stderr = &memBuffer{entry: e}
 	e.stdout = &blobBuffer{entry: e}
+
 	e.id = c.nextEntryID
 	c.nextEntryID++
 
+	c.entries.Add(key, e)
+
 	return e
 }
 
@@ -198,10 +220,11 @@ func (e *entry) chunkKey(i int) string {
 
 func (e *entry) delete() {
 	e.c.Lock()
-	if e.c.entries[e.key] == e {
-		delete(e.c.entries, e.key)
+	defer e.c.Unlock()
+
+	if lruEntry, ok := e.c.entries.Get(e.key); ok && lruEntry.(*entry) == e {
+		e.c.entries.Remove(e.key)
 	}
-	e.c.Unlock()
 }
 
 type memBuffer struct {
@@ -233,29 +256,6 @@ func (mb *memBuffer) send(ctx context.Context, w io.Writer, pos *int) error {
 	return err
 }
 
-func newCache(cacheURL string) (*cache, error) {
-	c := &cache{
-		entries: make(map[string]*entry),
-	}
-	buf := make([]byte, 8)
-	if _, err := io.ReadFull(rand.Reader, buf); err != nil {
-		return nil, err
-	}
-	c.cacheID = fmt.Sprintf("%x", buf)
-
-	if v := os.Getenv("GITALY_CACHE_BUCKET"); v != "" {
-		cacheURL = v
-	}
-
-	var err error
-	c.Bucket, err = blob.OpenBucket(context.Background(), cacheURL)
-	if err != nil {
-		return nil, err
-	}
-
-	return c, nil
-}
-
 type readSeekCloser interface {
 	io.ReadSeeker
 	io.Closer
@@ -403,5 +403,5 @@ func (bb *blobBuffer) writer() (io.Writer, error) {
 func (bb *blobBuffer) chunkIsFull() bool {
 	bb.Lock()
 	defer bb.Unlock()
-	return bb.currentChunkSize > chunkSize
+	return bb.currentChunkSize > bb.c.chunkSize
 }
-- 
GitLab


From c2e4eb39fa3f09312183c0a2df2a360eb7285240 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sun, 22 Nov 2020 19:25:47 +0100
Subject: [PATCH 15/22] Add time expiry

---
 internal/gitaly/service/hook/pack_objects.go | 39 ++++++++++++--------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index 667b88c1ee4..d3693b7bb98 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -2,7 +2,6 @@ package hook
 
 import (
 	"context"
-	"crypto/rand"
 	"crypto/sha256"
 	"encoding/json"
 	"errors"
@@ -12,9 +11,11 @@ import (
 	"os"
 	"os/exec"
 	"sync"
+	"time"
 
 	"gitlab.com/gitlab-org/gitaly/internal/command"
 	"gitlab.com/gitlab-org/gitaly/internal/helper"
+	"gitlab.com/gitlab-org/gitaly/internal/helper/text"
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 	"gitlab.com/gitlab-org/gitaly/streamio"
 
@@ -57,9 +58,11 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 		// If we have a cache miss and we win the race to spawn a goroutine that
 		// fills the cache, that goroutine becomes responsible for closing the
 		// stdin tempfile.
-		if !stdinHandoff {
-			stdin.Close()
+		if stdinHandoff {
+			return
 		}
+
+		stdin.Close()
 	}()
 
 	var m sync.Mutex
@@ -77,7 +80,7 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 
 	var e *entry
 	packCache.Lock()
-	if lruEntry, ok := packCache.entries.Get(key); ok {
+	if lruEntry, ok := packCache.entries.Get(key); ok && !lruEntry.(*entry).isStale() {
 		e = lruEntry.(*entry)
 	} else {
 		e = packCache.createEntry(key)
@@ -139,24 +142,26 @@ type cache struct {
 	sync.Mutex
 	nextEntryID int
 	chunkSize   int
+	maxAge      time.Duration
 }
 
 func newCache(cacheURL string) (*cache, error) {
 	c := &cache{
 		entries:   lru.New(10000),
 		chunkSize: 50 * 1024 * 1024,
+		maxAge:    15 * time.Minute,
 	}
-	buf := make([]byte, 8)
-	if _, err := io.ReadFull(rand.Reader, buf); err != nil {
+
+	var err error
+	c.cacheID, err = text.RandomHex(8)
+	if err != nil {
 		return nil, err
 	}
-	c.cacheID = fmt.Sprintf("%x", buf)
 
 	if v := os.Getenv("GITALY_CACHE_BUCKET"); v != "" {
 		cacheURL = v
 	}
 
-	var err error
 	c.Bucket, err = blob.OpenBucket(context.Background(), cacheURL)
 	if err != nil {
 		return nil, err
@@ -186,8 +191,9 @@ func (c *cache) key(repoPath string, args []string, stdin io.ReadSeeker) (string
 // createEntry expects caller to lock/unlock cache.
 func (c *cache) createEntry(key string) *entry {
 	e := &entry{
-		c:   c,
-		key: key,
+		c:       c,
+		key:     key,
+		created: time.Now(),
 	}
 	e.Cond = sync.NewCond(e)
 	e.stderr = &memBuffer{entry: e}
@@ -206,13 +212,16 @@ type entry struct {
 	key string
 	sync.Mutex
 	*sync.Cond
-	done   bool
-	err    error
-	stderr *memBuffer
-	stdout *blobBuffer
-	id     int
+	done    bool
+	err     error
+	stderr  *memBuffer
+	stdout  *blobBuffer
+	id      int
+	created time.Time
 }
 
+func (e *entry) isStale() bool { return time.Since(e.created) > e.c.maxAge }
+
 func (e *entry) chunkKey(i int) string {
 	chunk := fmt.Sprintf("%02x", i)
 	return fmt.Sprintf("%s/%s/%s/%d", e.key, chunk[len(chunk)-2:], chunk, e.id)
-- 
GitLab


From 8fb818986f4c052fe63bfcc039a7181ec88b6ae0 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sun, 22 Nov 2020 19:52:33 +0100
Subject: [PATCH 16/22] Add ssh

---
 internal/git/hooks.go                            |  2 +-
 internal/gitaly/service/register.go              |  1 +
 internal/gitaly/service/smarthttp/upload_pack.go |  2 +-
 internal/gitaly/service/ssh/server.go            |  8 ++++++++
 internal/gitaly/service/ssh/testhelper_test.go   | 11 +++++++++++
 internal/gitaly/service/ssh/upload_pack.go       |  8 +++++++-
 6 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/internal/git/hooks.go b/internal/git/hooks.go
index 41d7ef22b49..d4b34316b95 100644
--- a/internal/git/hooks.go
+++ b/internal/git/hooks.go
@@ -36,7 +36,7 @@ func WithRefTxHook(ctx context.Context, repo *gitalypb.Repository, cfg config.Cf
 	}
 }
 
-func WithPackObjectsHook(ctx context.Context, repo *gitalypb.Repository, cfg config.Cfg) CmdOpt {
+func WithPackObjectsHookEnv(ctx context.Context, repo *gitalypb.Repository, cfg config.Cfg) CmdOpt {
 	return func(cc *cmdCfg) error {
 		if repo == nil {
 			return fmt.Errorf("missing repo: %w", ErrInvalidArg)
diff --git a/internal/gitaly/service/register.go b/internal/gitaly/service/register.go
index d02aaf73ea1..08d60d1bdf2 100644
--- a/internal/gitaly/service/register.go
+++ b/internal/gitaly/service/register.go
@@ -80,6 +80,7 @@ func RegisterAll(grpcServer *grpc.Server, cfg config.Cfg, rubyServer *rubyserver
 	gitalypb.RegisterSSHServiceServer(grpcServer, ssh.NewServer(
 		locator,
 		ssh.WithPackfileNegotiationMetrics(sshPackfileNegotiationMetrics),
+		ssh.WithConfig(cfg),
 	))
 	gitalypb.RegisterSmartHTTPServiceServer(grpcServer, smarthttp.NewServer(
 		locator,
diff --git a/internal/gitaly/service/smarthttp/upload_pack.go b/internal/gitaly/service/smarthttp/upload_pack.go
index 5d0dd913f48..628eff03416 100644
--- a/internal/gitaly/service/smarthttp/upload_pack.go
+++ b/internal/gitaly/service/smarthttp/upload_pack.go
@@ -87,7 +87,7 @@ func (s *server) PostUploadPack(stream gitalypb.SmartHTTPService_PostUploadPackS
 		Args:  []string{repoPath},
 	},
 		git.WithGitProtocol(ctx, req),
-		git.WithPackObjectsHook(ctx, req.Repository, s.cfg),
+		git.WithPackObjectsHookEnv(ctx, req.Repository, s.cfg),
 	)
 
 	if err != nil {
diff --git a/internal/gitaly/service/ssh/server.go b/internal/gitaly/service/ssh/server.go
index 3a92c72b530..962af9f94b8 100644
--- a/internal/gitaly/service/ssh/server.go
+++ b/internal/gitaly/service/ssh/server.go
@@ -4,6 +4,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"gitlab.com/gitlab-org/gitaly/internal/gitaly/config"
 	"gitlab.com/gitlab-org/gitaly/internal/storage"
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 )
@@ -18,6 +19,7 @@ type server struct {
 	uploadPackRequestTimeout    time.Duration
 	uploadArchiveRequestTimeout time.Duration
 	packfileNegotiationMetrics  *prometheus.CounterVec
+	cfg                         config.Cfg
 }
 
 // NewServer creates a new instance of a grpc SSHServer
@@ -61,3 +63,9 @@ func WithPackfileNegotiationMetrics(c *prometheus.CounterVec) ServerOpt {
 		s.packfileNegotiationMetrics = c
 	}
 }
+
+func WithConfig(cfg config.Cfg) ServerOpt {
+	return func(s *server) {
+		s.cfg = cfg
+	}
+}
diff --git a/internal/gitaly/service/ssh/testhelper_test.go b/internal/gitaly/service/ssh/testhelper_test.go
index 1d805651b4e..41271e16743 100644
--- a/internal/gitaly/service/ssh/testhelper_test.go
+++ b/internal/gitaly/service/ssh/testhelper_test.go
@@ -1,12 +1,15 @@
 package ssh
 
 import (
+	"net"
 	"os"
 	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 	"gitlab.com/gitlab-org/gitaly/internal/gitaly/config"
+	gitalyhook "gitlab.com/gitlab-org/gitaly/internal/gitaly/hook"
+	"gitlab.com/gitlab-org/gitaly/internal/gitaly/service/hook"
 	"gitlab.com/gitlab-org/gitaly/internal/testhelper"
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 	"google.golang.org/grpc"
@@ -42,10 +45,18 @@ func testMain(m *testing.M) int {
 func runSSHServer(t *testing.T, serverOpts ...ServerOpt) (string, func()) {
 	srv := testhelper.NewServer(t, nil, nil)
 
+	serverOpts = append([]ServerOpt{WithConfig(config.Config)}, serverOpts...)
 	gitalypb.RegisterSSHServiceServer(srv.GrpcServer(), NewServer(config.NewLocator(config.Config), serverOpts...))
+	gitalypb.RegisterHookServiceServer(srv.GrpcServer(), hook.NewServer(gitalyhook.NewManager(gitalyhook.GitlabAPIStub, config.Config)))
 	reflection.Register(srv.GrpcServer())
 
+	internalListener, err := net.Listen("unix", config.Config.GitalyInternalSocketPath())
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	require.NoError(t, srv.Start())
+	go srv.GrpcServer().Serve(internalListener)
 
 	return "unix://" + srv.Socket(), srv.Stop
 }
diff --git a/internal/gitaly/service/ssh/upload_pack.go b/internal/gitaly/service/ssh/upload_pack.go
index 8bb1aed2d24..4a5fd7d593e 100644
--- a/internal/gitaly/service/ssh/upload_pack.go
+++ b/internal/gitaly/service/ssh/upload_pack.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"path/filepath"
 	"sync"
 
 	"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
@@ -79,6 +80,8 @@ func (s *server) sshUploadPack(stream gitalypb.SSHService_SSHUploadPackServer, r
 	git.WarnIfTooManyBitmaps(ctx, req.GetRepository())
 
 	globalOpts := git.UploadPackFilterConfig()
+	hookBin := filepath.Join(s.cfg.BinDir, "gitaly-hooks")
+	globalOpts = append(globalOpts, git.ValueFlag{"-c", "uploadpack.packobjectshook=" + hookBin})
 	for _, o := range req.GitConfigOptions {
 		globalOpts = append(globalOpts, git.ValueFlag{"-c", o})
 	}
@@ -106,7 +109,10 @@ func (s *server) sshUploadPack(stream gitalypb.SSHService_SSHUploadPackServer, r
 	cmd, monitor, err := monitorStdinCommand(ctx, stdin, stdout, stderr, nil, globalOpts, git.SubCmd{
 		Name: "upload-pack",
 		Args: []string{repoPath},
-	}, git.WithGitProtocol(ctx, req))
+	},
+		git.WithGitProtocol(ctx, req),
+		git.WithPackObjectsHookEnv(ctx, req.Repository, s.cfg),
+	)
 
 	if err != nil {
 		return err
-- 
GitLab


From 767295f8e04c6195cdc010633d085f77a6bfd71f Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sun, 22 Nov 2020 19:55:01 +0100
Subject: [PATCH 17/22] remove debug logger

---
 cmd/gitaly-hooks/hooks.go | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/cmd/gitaly-hooks/hooks.go b/cmd/gitaly-hooks/hooks.go
index c078b59b4a7..e527c3ae581 100644
--- a/cmd/gitaly-hooks/hooks.go
+++ b/cmd/gitaly-hooks/hooks.go
@@ -34,9 +34,6 @@ func main() {
 		logger.Fatalf("requires hook name. args: %v", os.Args)
 	}
 
-	f, _ := os.OpenFile("/tmp/hello", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
-	hello := log.New(f, "gitaly-hooks: ", log.LstdFlags)
-
 	subCmd := os.Args[1]
 
 	if subCmd == "check" {
@@ -236,14 +233,14 @@ func main() {
 
 		packObjectsHookStream, err := hookClient.PackObjectsHook(ctx)
 		if err != nil {
-			hello.Fatalf("error when getting stream client for %q: %v", subCmd, err)
+			logger.Fatalf("error when getting stream client for %q: %v", subCmd, err)
 		}
 
 		if err := packObjectsHookStream.Send(&gitalypb.PackObjectsHookRequest{
 			Repository: repository,
 			Args:       args,
 		}); err != nil {
-			hello.Fatalf("error when sending request for %q: %v", subCmd, err)
+			logger.Fatalf("error when sending request for %q: %v", subCmd, err)
 		}
 
 		f := sendFunc(streamio.NewWriter(func(p []byte) error {
@@ -253,7 +250,7 @@ func main() {
 		if hookStatus, err = stream.Handler(func() (stream.StdoutStderrResponse, error) {
 			return packObjectsHookStream.Recv()
 		}, f, os.Stdout, os.Stderr); err != nil {
-			hello.Fatalf("error when receiving data for %q: %v", subCmd, err)
+			logger.Fatalf("error when receiving data for %q: %v", subCmd, err)
 		}
 	default:
 		logger.Fatalf("subcommand name invalid: %q", subCmd)
-- 
GitLab


From aeebf76b1eba435d7780d57e001d21a1866da5b7 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Sun, 22 Nov 2020 22:03:55 +0100
Subject: [PATCH 18/22] normalize repo path

---
 internal/gitaly/service/hook/pack_objects.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index d3693b7bb98..e4dec109a57 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -10,6 +10,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"sync"
 	"time"
 
@@ -180,7 +181,7 @@ func (c *cache) key(repoPath string, args []string, stdin io.ReadSeeker) (string
 	}
 
 	enc := json.NewEncoder(h)
-	for _, v := range []interface{}{c.cacheID, args, repoPath} {
+	for _, v := range []interface{}{c.cacheID, args, filepath.Clean(repoPath)} {
 		if err := enc.Encode(v); err != nil {
 			return "", err
 		}
-- 
GitLab


From 586a4fc57875a4ebeb5ab44fd49b0944c06f1ef9 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Mon, 23 Nov 2020 08:59:09 +0100
Subject: [PATCH 19/22] Prevent getting ahead of consumers

---
 internal/gitaly/service/hook/pack_objects.go | 132 ++++++++++++++-----
 1 file changed, 99 insertions(+), 33 deletions(-)

diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index e4dec109a57..d389c0e85f9 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -88,12 +88,13 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 		stdinHandoff = true
 		go e.fill(repoPath, firstRequest.Args, stdin, stdout, stderr)
 	}
+	defer e.consumerDone()
 	packCache.Unlock()
 
 	var posErr, posOut int
 	for done := false; !done; {
 		e.Lock()
-		for !e.done && posErr == e.stderr.len() && posOut == e.stdout.len() {
+		for !e.done && !e.stderr.updatedAfter(posErr) && !e.stdout.updatedAfter(posOut) {
 			e.Wait()
 		}
 		done = e.done
@@ -192,13 +193,15 @@ func (c *cache) key(repoPath string, args []string, stdin io.ReadSeeker) (string
 // createEntry expects caller to lock/unlock cache.
 func (c *cache) createEntry(key string) *entry {
 	e := &entry{
-		c:       c,
-		key:     key,
-		created: time.Now(),
+		c:         c,
+		key:       key,
+		created:   time.Now(),
+		consumers: 1,
 	}
+
 	e.Cond = sync.NewCond(e)
 	e.stderr = &memBuffer{entry: e}
-	e.stdout = &blobBuffer{entry: e}
+	e.stdout = newBlobBuffer(e)
 
 	e.id = c.nextEntryID
 	c.nextEntryID++
@@ -213,14 +216,35 @@ type entry struct {
 	key string
 	sync.Mutex
 	*sync.Cond
-	done    bool
-	err     error
-	stderr  *memBuffer
-	stdout  *blobBuffer
-	id      int
-	created time.Time
+	done      bool
+	err       error
+	stderr    *memBuffer
+	stdout    *blobBuffer
+	id        int
+	created   time.Time
+	consumers int
+}
+
+func (e *entry) consumerDone() {
+	e.Lock()
+	defer e.Unlock()
+
+	e.consumers--
+	if e.consumers < 0 {
+		panic("pack-objects: negative reference count")
+	}
 }
 
+func (e *entry) allConsumersGone() bool {
+	e.Lock()
+	defer e.Unlock()
+	return e.consumers == 0
+}
+
+var errAllConsumersGone = errors.New("pack-objects: all consumers are gone")
+
+func (e *entry) notifyConsumers() { e.Cond.Broadcast() }
+
 func (e *entry) isStale() bool { return time.Since(e.created) > e.c.maxAge }
 
 func (e *entry) chunkKey(i int) string {
@@ -243,15 +267,20 @@ type memBuffer struct {
 }
 
 func (mb *memBuffer) Write(p []byte) (int, error) {
+	if mb.allConsumersGone() {
+		return 0, errAllConsumersGone
+	}
+
 	mb.Lock()
 	mb.buf = append(mb.buf, p...)
 	mb.Unlock()
-	mb.Broadcast()
+
+	mb.notifyConsumers()
 	return len(p), nil
 }
 
 // Caller must hold lock when calling len.
-func (mb *memBuffer) len() int { return len(mb.buf) }
+func (mb *memBuffer) updatedAfter(pos int) bool { return pos < len(mb.buf) }
 
 func (mb *memBuffer) send(ctx context.Context, w io.Writer, pos *int) error {
 	mb.Lock()
@@ -304,24 +333,53 @@ func (e *entry) fill(repoPath string, args []string, stdin readSeekCloser, stdou
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	err := runPackObjects(ctx, repoPath, args, stdin, e.stdout, e.stderr)
+	err := func() (err error) {
+		defer func() {
+			if p := recover(); p != nil {
+				err = fmt.Errorf("recovered panic: %v", err)
+			}
+		}()
+
+		return runPackObjects(ctx, repoPath, args, stdin, e.stdout, e.stderr)
+	}()
 
 	e.Lock()
+	defer e.Unlock()
+
 	e.done = true
 	e.err = err
 	if err != nil {
 		e.delete()
 	}
-	e.Unlock()
 
-	e.Broadcast()
+	e.notifyConsumers()
 }
 
 type blobBuffer struct {
 	*entry
-	w                *blob.Writer
-	nChunks          int
-	currentChunkSize int
+	w                    *blob.Writer
+	nChunks              int
+	currentChunkSize     int
+	highestReceivedChunk int
+	*sync.Cond
+}
+
+func newBlobBuffer(e *entry) *blobBuffer {
+	bb := &blobBuffer{entry: e}
+	bb.Cond = sync.NewCond(e)
+	return bb
+}
+
+func (bb *blobBuffer) oneConsumerCaughtUp() bool { return bb.highestReceivedChunk == bb.nChunks-1 }
+
+func (bb *blobBuffer) notifyReceivedChunk(i int) {
+	bb.Lock()
+	defer bb.Unlock()
+
+	if i > bb.highestReceivedChunk {
+		bb.highestReceivedChunk = i
+	}
+	bb.Cond.Broadcast()
 }
 
 // Flush finishes the current chunk.
@@ -340,20 +398,30 @@ func (bb *blobBuffer) Flush() error {
 	bb.w = nil
 	bb.nChunks++
 	bb.currentChunkSize = 0
-	bb.Broadcast()
+	bb.notifyConsumers()
+
+	for !bb.oneConsumerCaughtUp() && bb.entry.consumers > 0 {
+		bb.Cond.Wait()
+	}
+
+	if bb.entry.consumers == 0 {
+		return errAllConsumersGone
+	}
+
 	return nil
 }
 
-// Caller must hold lock when calling len.
-func (bb *blobBuffer) len() int { return bb.nChunks }
+func (bb *blobBuffer) updatedAfter(pos int) bool { return pos < bb.nChunks }
 
 func (bb *blobBuffer) send(ctx context.Context, w io.Writer, pos *int) error {
 	bb.Lock()
-	nChunks := bb.len()
+	nChunks := bb.nChunks
 	bb.Unlock()
 
 	var err error
 	for ; *pos < nChunks && err == nil; *pos++ {
+		bb.notifyReceivedChunk(*pos)
+
 		err = func() error {
 			r, err := bb.c.Bucket.NewReader(ctx, bb.chunkKey(*pos), nil)
 			if err != nil {
@@ -370,6 +438,10 @@ func (bb *blobBuffer) send(ctx context.Context, w io.Writer, pos *int) error {
 }
 
 func (bb *blobBuffer) Write(p []byte) (int, error) {
+	if bb.allConsumersGone() {
+		return 0, errAllConsumersGone
+	}
+
 	w, err := bb.writer()
 	if err != nil {
 		return 0, err
@@ -380,10 +452,10 @@ func (bb *blobBuffer) Write(p []byte) (int, error) {
 	}
 
 	bb.Lock()
-	bb.currentChunkSize += n
-	bb.Unlock()
+	defer bb.Unlock()
 
-	if bb.chunkIsFull() {
+	bb.currentChunkSize += n
+	if bb.currentChunkSize > bb.c.chunkSize {
 		return n, bb.Flush()
 	}
 
@@ -401,7 +473,7 @@ func (bb *blobBuffer) writer() (io.Writer, error) {
 	var err error
 	if bb.w, err = bb.entry.c.Bucket.NewWriter(
 		context.Background(),
-		bb.chunkKey(bb.len()),
+		bb.chunkKey(bb.nChunks),
 		&blob.WriterOptions{ContentType: "application/octet-stream"},
 	); err != nil {
 		return nil, err
@@ -409,9 +481,3 @@ func (bb *blobBuffer) writer() (io.Writer, error) {
 
 	return bb.w, nil
 }
-
-func (bb *blobBuffer) chunkIsFull() bool {
-	bb.Lock()
-	defer bb.Unlock()
-	return bb.currentChunkSize > bb.c.chunkSize
-}
-- 
GitLab


From 45d0357b7ccd992e077e564abea9a3256bbc7ab1 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Mon, 23 Nov 2020 11:30:08 +0100
Subject: [PATCH 20/22] Fix reference counting

---
 internal/gitaly/service/hook/pack_objects.go | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index d389c0e85f9..f7373a9faa9 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -79,11 +79,9 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ
 		return err
 	}
 
-	var e *entry
 	packCache.Lock()
-	if lruEntry, ok := packCache.entries.Get(key); ok && !lruEntry.(*entry).isStale() {
-		e = lruEntry.(*entry)
-	} else {
+	e := packCache.getEntry(key)
+	if e == nil {
 		e = packCache.createEntry(key)
 		stdinHandoff = true
 		go e.fill(repoPath, firstRequest.Args, stdin, stdout, stderr)
@@ -211,6 +209,17 @@ func (c *cache) createEntry(key string) *entry {
 	return e
 }
 
+func (c *cache) getEntry(key string) *entry {
+	if lruEntry, ok := packCache.entries.Get(key); ok && !lruEntry.(*entry).isStale() {
+		e := lruEntry.(*entry)
+		e.Lock()
+		e.consumers++
+		e.Unlock()
+		return e
+	}
+	return nil
+}
+
 type entry struct {
 	c   *cache
 	key string
-- 
GitLab


From 87daf9d2e84deab0987d846b24fa67f24e302baf Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Mon, 23 Nov 2020 11:51:16 +0100
Subject: [PATCH 21/22] Fix deadlock

---
 internal/gitaly/service/hook/pack_objects.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index f7373a9faa9..1dff03431b1 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -461,12 +461,12 @@ func (bb *blobBuffer) Write(p []byte) (int, error) {
 	}
 
 	bb.Lock()
-	defer bb.Unlock()
-
 	bb.currentChunkSize += n
 	if bb.currentChunkSize > bb.c.chunkSize {
+		bb.Unlock()
 		return n, bb.Flush()
 	}
+	bb.Unlock()
 
 	return n, nil
 }
-- 
GitLab


From 1504b30e1befd60c87984393d6638667f1881cac Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <jacob@gitlab.com>
Date: Mon, 23 Nov 2020 17:51:58 +0100
Subject: [PATCH 22/22] Use more radix path scheme for key

---
 internal/gitaly/service/hook/pack_objects.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go
index 1dff03431b1..5387f73d0ee 100644
--- a/internal/gitaly/service/hook/pack_objects.go
+++ b/internal/gitaly/service/hook/pack_objects.go
@@ -258,7 +258,7 @@ func (e *entry) isStale() bool { return time.Since(e.created) > e.c.maxAge }
 
 func (e *entry) chunkKey(i int) string {
 	chunk := fmt.Sprintf("%02x", i)
-	return fmt.Sprintf("%s/%s/%s/%d", e.key, chunk[len(chunk)-2:], chunk, e.id)
+	return fmt.Sprintf("%s/%s/%s/%s/%s/%d", e.key[:2], e.key[2:4], e.key[4:], chunk[len(chunk)-2:], chunk, e.id)
 }
 
 func (e *entry) delete() {
-- 
GitLab