From c5ff4a48466b5b16921f087fda350fb56292b654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Thu, 18 Apr 2019 17:58:59 +0200 Subject: [PATCH 1/3] Cache info/refs is logAllRefUpdates is enabled --- internal/service/smarthttp/inforefs.go | 144 ++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 5 deletions(-) diff --git a/internal/service/smarthttp/inforefs.go b/internal/service/smarthttp/inforefs.go index 3947c1cd943..f5e9bdbeaec 100644 --- a/internal/service/smarthttp/inforefs.go +++ b/internal/service/smarthttp/inforefs.go @@ -4,6 +4,9 @@ import ( "context" "fmt" "io" + "io/ioutil" + "os" + "path/filepath" grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus" log "github.com/sirupsen/logrus" @@ -16,10 +19,54 @@ import ( "google.golang.org/grpc/status" ) +func gitConfigOptions(req *gitalypb.InfoRefsRequest) []string { + args := []string{} + for _, params := range req.GitConfigOptions { + args = append(args, "-c", params) + } + return args +} + +func getConfig(ctx context.Context, req *gitalypb.InfoRefsRequest, key string) (string, error) { + args := gitConfigOptions(req) + args = append(args, "config", "core.logAllRefUpdates") + cmd, err := git.Command(ctx, req.Repository, args...) + if err != nil { + if _, ok := status.FromError(err); ok { + return "", err + } + return "", status.Errorf(codes.Internal, "getConfig: cmd: %v", err) + } + + data, err := ioutil.ReadAll(cmd) + if err != nil { + return "", status.Errorf(codes.Internal, "getConfig: cmd: %v", err) + } + + return string(data), cmd.Wait() +} + +func supportsInfoRefsCaching(ctx context.Context, req *gitalypb.InfoRefsRequest) (bool, error) { + configOption, err := getConfig(ctx, req, "core.logAllRefUpdates") + if err != nil { + if _, ok := status.FromError(err); ok { + return false, err + } + return false, status.Errorf(codes.Internal, "GetCachedInfoRefs: cmd: %v", err) + } + + if configOption != "always" { + return false, nil + } + + return true, nil +} + func (s *server) InfoRefsUploadPack(in *gitalypb.InfoRefsRequest, stream gitalypb.SmartHTTPService_InfoRefsUploadPackServer) error { w := streamio.NewWriter(func(p []byte) error { return stream.Send(&gitalypb.InfoRefsResponse{Data: p}) }) + return handleInfoRefs(stream.Context(), "upload-pack", in, w) } @@ -30,6 +77,97 @@ func (s *server) InfoRefsReceivePack(in *gitalypb.InfoRefsRequest, stream gitaly return handleInfoRefs(stream.Context(), "receive-pack", in, w) } +func readCachedInfoRefs(ctx context.Context, req *gitalypb.InfoRefsRequest, w io.Writer) error { + repoPath, err := helper.GetRepoPath(req.Repository) + if err != nil { + return err + } + + // if logs/refs is present it means that we have stale cache + _, err = os.Lstat( + filepath.Join(repoPath, "logs", "refs")) + if err == nil { + return os.ErrNotExist + } else if !os.IsNotExist(err) { + return err + } + + infoRefs, err := os.Open( + filepath.Join(repoPath, "info-refs")) + if err != nil { + return err + } + defer infoRefs.Close() + + _, err = io.Copy(w, infoRefs) + if err != nil { + return err + } + return nil +} + +func hasLogsRefs(ctx context.Context, req *gitalypb.InfoRefsRequest) (bool, error) { + repoPath, err := helper.GetRepoPath(req.Repository) + if err != nil { + return false, err + } + + logsRefRepoPath := filepath.Join(repoPath, "logs", "refs") + _, err = os.Lstat(logsRefRepoPath) + + if err == nil { + return true, nil + } else if os.IsNotExist(err) { + return false, nil + } else { + return false, err + } +} + +func createCachedInfoRefs(ctx context.Context, service string, req *gitalypb.InfoRefsRequest, w io.Writer) error { + repoPath, err := helper.GetRepoPath(req.Repository) + if err != nil { + return err + } + + // try to remove all existing log refs + // we do not care about errors, as execute that on best efforr + os.RemoveAll(filepath.Join(repoPath, "logs", "refs")) + + // create temporary file + tmpPath := filepath.Join(repoPath, "tmp") + tmpInfoLogs, err := ioutil.TempFile(tmpPath, "info-logs") + if err != nil { + return status.Errorf(codes.Internal, "CachedInfoRefs: %v", err) + } + defer tmpInfoLogs.Close() + defer os.Remove(tmpInfoLogs.Name()) + + // write to output, and to file + multiWriter := io.MultiWriter(tmpInfoLogs, w) + err = handleInfoRefs(ctx, service, req, multiWriter) + if err != nil { + return status.Errorf(codes.Internal, "CachedInfoRefs: %v", err) + } + tmpInfoLogs.Close() + + // we do not care about errors, as execute that on best effort + infoRefsPath := filepath.Join(repoPath, "info-refs") + os.Rename(tmpInfoLogs.Name(), infoRefsPath) + return nil +} + +func handleCachedInfoRefs(ctx context.Context, service string, req *gitalypb.InfoRefsRequest, w io.Writer) error { + err := readCachedInfoRefs(ctx, req, w) + if os.IsNotExist(err) { + return createCachedInfoRefs(ctx, service, req, w) + } else if err != nil { + return status.Errorf(codes.Internal, "CachedInfoRefs: cmd: %v", err) + } + + return nil +} + func handleInfoRefs(ctx context.Context, service string, req *gitalypb.InfoRefsRequest, w io.Writer) error { grpc_logrus.Extract(ctx).WithFields(log.Fields{ "service": service, @@ -42,11 +180,7 @@ func handleInfoRefs(ctx context.Context, service string, req *gitalypb.InfoRefsR return err } - args := []string{} - for _, params := range req.GitConfigOptions { - args = append(args, "-c", params) - } - + args := gitConfigOptions(req) args = append(args, service, "--stateless-rpc", "--advertise-refs", repoPath) cmd, err := git.BareCommand(ctx, nil, nil, nil, env, args...) -- GitLab From 0c8ccc56056e4f19d50a9969e20a9c953dd20034 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Thu, 18 Apr 2019 18:29:16 +0200 Subject: [PATCH 2/3] Make it work --- internal/service/smarthttp/inforefs.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/service/smarthttp/inforefs.go b/internal/service/smarthttp/inforefs.go index f5e9bdbeaec..4c304c6b869 100644 --- a/internal/service/smarthttp/inforefs.go +++ b/internal/service/smarthttp/inforefs.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus" log "github.com/sirupsen/logrus" @@ -55,7 +56,7 @@ func supportsInfoRefsCaching(ctx context.Context, req *gitalypb.InfoRefsRequest) return false, status.Errorf(codes.Internal, "GetCachedInfoRefs: cmd: %v", err) } - if configOption != "always" { + if !strings.Contains(configOption, "always") { return false, nil } @@ -67,6 +68,11 @@ func (s *server) InfoRefsUploadPack(in *gitalypb.InfoRefsRequest, stream gitalyp return stream.Send(&gitalypb.InfoRefsResponse{Data: p}) }) + supported, _ := supportsInfoRefsCaching(stream.Context(), in) + if supported { + return handleCachedInfoRefs(stream.Context(), "upload-pack", in, w) + } + return handleInfoRefs(stream.Context(), "upload-pack", in, w) } @@ -135,8 +141,7 @@ func createCachedInfoRefs(ctx context.Context, service string, req *gitalypb.Inf os.RemoveAll(filepath.Join(repoPath, "logs", "refs")) // create temporary file - tmpPath := filepath.Join(repoPath, "tmp") - tmpInfoLogs, err := ioutil.TempFile(tmpPath, "info-logs") + tmpInfoLogs, err := ioutil.TempFile(repoPath, "info-logs") if err != nil { return status.Errorf(codes.Internal, "CachedInfoRefs: %v", err) } -- GitLab From f6640bbcd799a3947de9fa8d6fb7fdb8c765852e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Fri, 19 Apr 2019 10:52:59 +0200 Subject: [PATCH 3/3] Remove unused method --- internal/service/smarthttp/inforefs.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/internal/service/smarthttp/inforefs.go b/internal/service/smarthttp/inforefs.go index 4c304c6b869..267ebb7a81d 100644 --- a/internal/service/smarthttp/inforefs.go +++ b/internal/service/smarthttp/inforefs.go @@ -112,24 +112,6 @@ func readCachedInfoRefs(ctx context.Context, req *gitalypb.InfoRefsRequest, w io return nil } -func hasLogsRefs(ctx context.Context, req *gitalypb.InfoRefsRequest) (bool, error) { - repoPath, err := helper.GetRepoPath(req.Repository) - if err != nil { - return false, err - } - - logsRefRepoPath := filepath.Join(repoPath, "logs", "refs") - _, err = os.Lstat(logsRefRepoPath) - - if err == nil { - return true, nil - } else if os.IsNotExist(err) { - return false, nil - } else { - return false, err - } -} - func createCachedInfoRefs(ctx context.Context, service string, req *gitalypb.InfoRefsRequest, w io.Writer) error { repoPath, err := helper.GetRepoPath(req.Repository) if err != nil { -- GitLab