diff --git a/internal/git/catfile/catfile.go b/internal/git/catfile/catfile.go index a100ed306856c4847b9d4b222399980599ffee85..3c5f59a20cf1090eb42d8d2eca15b1aba5a744e3 100644 --- a/internal/git/catfile/catfile.go +++ b/internal/git/catfile/catfile.go @@ -40,6 +40,14 @@ func (c *Batch) Commit(revspec string) (io.Reader, error) { return c.batch.reader(revspec, "commit") } +// Tag returns a raw tag object. It is an error if revspec does not +// point to a commit. To prevent this first use Info to resolve the revspec +// and check the object type. Caller must consume the Reader before +// making another call on C. +func (c *Batch) Tag(revspec string) (io.Reader, error) { + return c.batch.reader(revspec, "tag") +} + // Blob returns a reader for the requested blob. The entire blob must be // read before any new objects can be requested from this Batch instance. // diff --git a/internal/git/log/commit.go b/internal/git/log/commit.go index 0ded22414d9a3d8e2dd3bb03c31de94bed6e885c..916b12d000e370e5bbb676cfc36f9f843a39d2af 100644 --- a/internal/git/log/commit.go +++ b/internal/git/log/commit.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "io" "io/ioutil" "strconv" "strings" @@ -42,15 +43,50 @@ func GetCommitCatfile(c *catfile.Batch, revision string) (*gitalypb.GitCommit, e return nil, err } - raw, err := ioutil.ReadAll(r) + return parseRawCommit(r, info) +} + +// GetCommitMessage looks up a commit message and returns it in its entirety. +func GetCommitMessage(ctx context.Context, repo *gitalypb.Repository, revision string) ([]byte, error) { + c, err := catfile.New(ctx, repo) + if err != nil { + return nil, err + } + info, err := c.Info(revision + "^{commit}") + if err != nil { + if catfile.IsNotFound(err) { + return nil, nil + } + + return nil, err + } + + r, err := c.Commit(info.Oid) if err != nil { return nil, err } - return parseRawCommit(raw, info) + _, body, err := splitRawCommit(r) + if err != nil { + return nil, err + } + return body, nil } -func parseRawCommit(raw []byte, info *catfile.ObjectInfo) (*gitalypb.GitCommit, error) { +func parseRawCommit(r io.Reader, info *catfile.ObjectInfo) (*gitalypb.GitCommit, error) { + header, body, err := splitRawCommit(r) + if err != nil { + return nil, err + } + return buildCommit(header, body, info) +} + +func splitRawCommit(r io.Reader) ([]byte, []byte, error) { + raw, err := ioutil.ReadAll(r) + if err != nil { + return nil, nil, err + } + split := bytes.SplitN(raw, []byte("\n\n"), 2) header := split[0] @@ -59,13 +95,18 @@ func parseRawCommit(raw []byte, info *catfile.ObjectInfo) (*gitalypb.GitCommit, body = split[1] } + return header, body, nil +} + +func buildCommit(header, body []byte, info *catfile.ObjectInfo) (*gitalypb.GitCommit, error) { commit := &gitalypb.GitCommit{ Id: info.Oid, + BodySize: int64(len(body)), Body: body, Subject: subjectFromBody(body), - BodySize: int64(len(body)), } - if max := helper.MaxCommitOrTagMessageSize; len(commit.Body) > max { + + if max := helper.MaxCommitOrTagMessageSize; len(body) > max { commit.Body = commit.Body[:max] } diff --git a/internal/git/log/commit_test.go b/internal/git/log/commit_test.go index b3144aceddcaacec094d6097dc63fcb412e266a7..7a73b72824ff43efc95778ddb70429eb5fd214d3 100644 --- a/internal/git/log/commit_test.go +++ b/internal/git/log/commit_test.go @@ -1,6 +1,7 @@ package log import ( + "bytes" "testing" "github.com/golang/protobuf/ptypes/timestamp" @@ -93,9 +94,10 @@ func TestParseRawCommit(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { info.Size = int64(len(tc.in)) - out, err := parseRawCommit(tc.in, info) + out, err := parseRawCommit(bytes.NewBuffer(tc.in), info) require.NoError(t, err, "parse error") - require.Equal(t, *tc.out, *out) + require.Equal(t, tc.out, out) + }) } } diff --git a/internal/git/log/tags.go b/internal/git/log/tags.go new file mode 100644 index 0000000000000000000000000000000000000000..0487d2f65033d23af243315e3017998933e19336 --- /dev/null +++ b/internal/git/log/tags.go @@ -0,0 +1,130 @@ +package log + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "strings" + + "gitlab.com/gitlab-org/gitaly-proto/go/gitalypb" + "gitlab.com/gitlab-org/gitaly/internal/git" + "gitlab.com/gitlab-org/gitaly/internal/git/catfile" + "gitlab.com/gitlab-org/gitaly/internal/helper" +) + +// GetAllTags retrieves all tag objects for a given repository +func GetAllTags(ctx context.Context, repo *gitalypb.Repository) ([]*gitalypb.Tag, error) { + var tags []*gitalypb.Tag + + c, err := catfile.New(ctx, repo) + if err != nil { + return tags, nil + } + + tagsCmd, err := git.Command(ctx, repo, "for-each-ref", "--format", "%(objectname) %(refname:short)", "refs/tags/") + if err != nil { + return tags, err + } + s := bufio.NewScanner(tagsCmd) + for s.Scan() { + line := strings.SplitN(s.Text(), " ", 2) + oid := strings.TrimSpace(line[0]) + info, err := c.Info(oid) + if err != nil { + return tags, err + } + if info.IsBlob() { + continue + } + tag, err := buildTag(oid, info.Type, line[1], c) + if err != nil { + return tags, err + } + tags = append(tags, tag) + } + + return tags, nil +} + +func buildTag(oid, objectType, tagName string, b *catfile.Batch) (*gitalypb.Tag, error) { + var tag *gitalypb.Tag + var err error + switch objectType { + // annotated tag + case "tag": + tag, err = buildAnnotatedTag(b, oid) + if err != nil { + return nil, err + } + // lightweight tag + case "commit": + tag, err = buildLightweightTag(b, oid, tagName) + if err != nil { + return nil, err + } + default: + return nil, errors.New("unknown object type for tag") + } + return tag, nil +} + +func buildAnnotatedTag(b *catfile.Batch, oid string) (*gitalypb.Tag, error) { + r, err := b.Tag(oid) + if err != nil { + return nil, fmt.Errorf("buildAnnotatedTag error when getting tag reader: %v", err) + } + header, body, err := splitRawCommit(r) + if err != nil { + return nil, fmt.Errorf("buildAnnotatedTag error when splitting commit: %v", err) + } + + tag := &gitalypb.Tag{ + Id: oid, + MessageSize: int64(len(body)), + Message: body, + } + + if max := helper.MaxCommitOrTagMessageSize; len(body) > max { + tag.Message = tag.Message[:max] + } + + s := bufio.NewScanner(bytes.NewReader(header)) + for s.Scan() { + line := s.Text() + if len(line) == 0 || line[0] == ' ' { + continue + } + headerSplit := strings.SplitN(line, " ", 2) + switch headerSplit[0] { + case "object": + commit, err := GetCommitCatfile(b, headerSplit[1]) + if err != nil { + return nil, fmt.Errorf("buildAnnotatedTag error when getting target commit: %v", err) + } + tag.TargetCommit = commit + case "tag": + tag.Name = []byte(headerSplit[1]) + case "tagger": + tag.Tagger = parseCommitAuthor(headerSplit[1]) + } + } + + return tag, nil +} + +func buildLightweightTag(b *catfile.Batch, oid, name string) (*gitalypb.Tag, error) { + commit, err := GetCommitCatfile(b, oid) + if err != nil { + return nil, err + } + + tag := &gitalypb.Tag{ + Id: oid, + Name: []byte(name), + TargetCommit: commit, + } + + return tag, nil +} diff --git a/internal/service/commit/commit_messages.go b/internal/service/commit/commit_messages.go index 7d041ae941aef44adf7937b6d08567a8dc0a431a..7e4f2f23f0553a31663dcbc0e47ce016075668ef 100644 --- a/internal/service/commit/commit_messages.go +++ b/internal/service/commit/commit_messages.go @@ -1,10 +1,13 @@ package commit import ( + "bytes" "fmt" + "io" "gitlab.com/gitlab-org/gitaly-proto/go/gitalypb" - "gitlab.com/gitlab-org/gitaly/internal/rubyserver" + "gitlab.com/gitlab-org/gitaly/internal/git/log" + "gitlab.com/gitlab-org/gitaly/streamio" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -13,33 +16,32 @@ func (s *server) GetCommitMessages(request *gitalypb.GetCommitMessagesRequest, s if err := validateGetCommitMessagesRequest(request); err != nil { return status.Errorf(codes.InvalidArgument, "GetCommitMessages: %v", err) } - ctx := stream.Context() + for _, commitID := range request.GetCommitIds() { + msg, err := log.GetCommitMessage(ctx, request.GetRepository(), commitID) + if err != nil { + return status.Errorf(codes.Internal, "failed to get commit message: %v", err) + } - client, err := s.CommitServiceClient(ctx) - if err != nil { - return err - } - - clientCtx, err := rubyserver.SetHeaders(ctx, request.GetRepository()) - if err != nil { - return err - } - - rubyStream, err := client.GetCommitMessages(clientCtx, request) - if err != nil { - return err - } - - return rubyserver.Proxy(func() error { - resp, err := rubyStream.Recv() + msgReader := bytes.NewReader(msg) + resp := &gitalypb.GetCommitMessagesResponse{ + CommitId: commitID, + } + sw := streamio.NewWriter(func(p []byte) error { + resp.Message = p + err := stream.Send(resp) + resp.CommitId = "" + if err != nil { + return err + } + return nil + }) + _, err = io.Copy(sw, msgReader) if err != nil { - md := rubyStream.Trailer() - stream.SetTrailer(md) - return err + return status.Errorf(codes.Internal, "failed to send response: %v", err) } - return stream.Send(resp) - }) + } + return nil } func validateGetCommitMessagesRequest(request *gitalypb.GetCommitMessagesRequest) error { diff --git a/internal/service/commit/commit_messages_test.go b/internal/service/commit/commit_messages_test.go index 48bda100769609bcf1e5b7b4cc95541b5845a189..0eb832758cc63d23ce31eb09c4ef4c0a74fca383 100644 --- a/internal/service/commit/commit_messages_test.go +++ b/internal/service/commit/commit_messages_test.go @@ -25,7 +25,8 @@ func TestSuccessfulGetCommitMessagesRequest(t *testing.T) { ctx, cancel := testhelper.Context() defer cancel() - message1 := strings.Repeat("a\n", helper.MaxCommitOrTagMessageSize*2) + // message1 := strings.Repeat("a\n", helper.MaxCommitOrTagMessageSize*2) + message1 := strings.Repeat("a\n", 3) message2 := strings.Repeat("b\n", helper.MaxCommitOrTagMessageSize*2) commit1ID := testhelper.CreateCommit(t, testRepoPath, "local-big-commits", &testhelper.CreateCommitOpts{Message: message1}) diff --git a/internal/service/ref/refs.go b/internal/service/ref/refs.go index e986e0ede910236a08d63b0e081cfa00a0fb3f9b..4044e941d68abd11ef5c54b238cf942d51730e01 100644 --- a/internal/service/ref/refs.go +++ b/internal/service/ref/refs.go @@ -3,17 +3,20 @@ package ref import ( "bufio" "bytes" + "errors" "fmt" "regexp" "strings" + "gitlab.com/gitlab-org/gitaly/internal/helper" + grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus" log "github.com/sirupsen/logrus" "gitlab.com/gitlab-org/gitaly-proto/go/gitalypb" "gitlab.com/gitlab-org/gitaly/internal/git" "gitlab.com/gitlab-org/gitaly/internal/git/catfile" + gitlog "gitlab.com/gitlab-org/gitaly/internal/git/log" "gitlab.com/gitlab-org/gitaly/internal/helper/lines" - "gitlab.com/gitlab-org/gitaly/internal/rubyserver" "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -73,30 +76,28 @@ func (s *server) FindAllTagNames(in *gitalypb.FindAllTagNamesRequest, stream git func (s *server) FindAllTags(in *gitalypb.FindAllTagsRequest, stream gitalypb.RefService_FindAllTagsServer) error { ctx := stream.Context() - client, err := s.RefServiceClient(ctx) - if err != nil { - return err + if err := validateFindAllTagsRequest(in); err != nil { + return status.Errorf(codes.InvalidArgument, "FindAllTags: %v", err) } - clientCtx, err := rubyserver.SetHeaders(ctx, in.GetRepository()) + tags, err := gitlog.GetAllTags(ctx, in.GetRepository()) if err != nil { - return err + return status.Errorf(codes.Internal, "error when getting all tags: %v", err) } + return stream.Send(&gitalypb.FindAllTagsResponse{ + Tags: tags, + }) +} - rubyStream, err := client.FindAllTags(clientCtx, in) +func validateFindAllTagsRequest(request *gitalypb.FindAllTagsRequest) error { + if request.GetRepository() == nil { + return errors.New("empty Repository") + } + _, err := helper.GetRepoPath(request.GetRepository()) if err != nil { - return err + return errors.New("invalid git directory") } - - return rubyserver.Proxy(func() error { - resp, err := rubyStream.Recv() - if err != nil { - md := rubyStream.Trailer() - stream.SetTrailer(md) - return err - } - return stream.Send(resp) - }) + return nil } func _findBranchNames(ctx context.Context, repo *gitalypb.Repository) ([][]byte, error) { diff --git a/internal/service/ref/refs_test.go b/internal/service/ref/refs_test.go index f675c2c9a5632f0884b86968199f558384202096..6dbeacd70344f18368363b732aac9f048300e6e9 100644 --- a/internal/service/ref/refs_test.go +++ b/internal/service/ref/refs_test.go @@ -410,7 +410,9 @@ func TestSuccessfulFindAllTagsRequest(t *testing.T) { bigCommit, err := log.GetCommit(ctx, testRepoCopy, bigCommitID) require.NoError(t, err) - annotatedTagID := testhelper.CreateTag(t, testRepoCopyPath, "v1.2.0", blobID, &testhelper.CreateTagOpts{Message: "Blob tag"}) + annotatedTagID := testhelper.CreateTag(t, testRepoCopyPath, "v1.2.0", commitID, &testhelper.CreateTagOpts{Message: "Annotated tag"}) + annotatedTagTimestamp, err := testhelper.GetTagDate(t, testRepoCopyPath, annotatedTagID) + require.NoError(t, err) testhelper.CreateTag(t, testRepoCopyPath, "v1.3.0", commitID, nil) testhelper.CreateTag(t, testRepoCopyPath, "v1.4.0", blobID, nil) @@ -424,6 +426,8 @@ func TestSuccessfulFindAllTagsRequest(t *testing.T) { // A tag with a big message bigMessage := strings.Repeat("a", 11*1024) bigMessageTag1ID := testhelper.CreateTag(t, testRepoCopyPath, "v1.7.0", commitID, &testhelper.CreateTagOpts{Message: bigMessage}) + bigMessageTagTimestamp, err := testhelper.GetTagDate(t, testRepoCopyPath, bigMessageTag1ID) + require.NoError(t, err) client, conn := newRefServiceClient(t, serverSocketPath) defer conn.Close() @@ -454,6 +458,11 @@ func TestSuccessfulFindAllTagsRequest(t *testing.T) { TargetCommit: gitCommit, Message: []byte("Release"), MessageSize: 7, + Tagger: &gitalypb.CommitAuthor{ + Name: []byte("Dmitriy Zaporozhets"), + Email: []byte("dmitriy.zaporozhets@gmail.com"), + Date: ×tamp.Timestamp{Seconds: 1393491299}, + }, }, { Name: []byte("v1.1.0"), @@ -477,22 +486,45 @@ func TestSuccessfulFindAllTagsRequest(t *testing.T) { }, Message: []byte("Version 1.1.0"), MessageSize: 13, + Tagger: &gitalypb.CommitAuthor{ + Name: []byte("Dmitriy Zaporozhets"), + Email: []byte("dmitriy.zaporozhets@gmail.com"), + Date: ×tamp.Timestamp{Seconds: 1393505709}, + }, }, { - Name: []byte("v1.2.0"), - Id: string(annotatedTagID), - Message: []byte("Blob tag"), - MessageSize: 8, + Name: []byte("v1.2.0"), + Id: string(annotatedTagID), + TargetCommit: &gitalypb.GitCommit{ + Id: "6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9", + Subject: []byte("More submodules"), + Body: []byte("More submodules\n\nSigned-off-by: Dmitriy Zaporozhets \n"), + Author: &gitalypb.CommitAuthor{ + Name: []byte("Dmitriy Zaporozhets"), + Email: []byte("dmitriy.zaporozhets@gmail.com"), + Date: ×tamp.Timestamp{Seconds: 1393491261}, + }, + Committer: &gitalypb.CommitAuthor{ + Name: []byte("Dmitriy Zaporozhets"), + Email: []byte("dmitriy.zaporozhets@gmail.com"), + Date: ×tamp.Timestamp{Seconds: 1393491261}, + }, + ParentIds: []string{"d14d6c0abdd253381df51a723d58691b2ee1ab08"}, + BodySize: 84, + }, + Message: []byte("Annotated tag"), + MessageSize: 13, + Tagger: &gitalypb.CommitAuthor{ + Name: []byte("Scrooge McDuck"), + Email: []byte("scrooge@mcduck.com"), + Date: ×tamp.Timestamp{Seconds: annotatedTagTimestamp}, + }, }, { Name: []byte("v1.3.0"), Id: string(commitID), TargetCommit: gitCommit, }, - { - Name: []byte("v1.4.0"), - Id: string(blobID), - }, { Name: []byte("v1.5.0"), Id: string(commitID), @@ -509,6 +541,11 @@ func TestSuccessfulFindAllTagsRequest(t *testing.T) { Message: []byte(bigMessage[:helper.MaxCommitOrTagMessageSize]), MessageSize: int64(len(bigMessage)), TargetCommit: gitCommit, + Tagger: &gitalypb.CommitAuthor{ + Name: []byte("Scrooge McDuck"), + Email: []byte("scrooge@mcduck.com"), + Date: ×tamp.Timestamp{Seconds: bigMessageTagTimestamp}, + }, }, } diff --git a/internal/testhelper/tag.go b/internal/testhelper/tag.go index 29e01e78cc0069ffb29c31424d21d2d5d643f70f..a94d99685e528cf04810c4c2158ac3d7b66809dd 100644 --- a/internal/testhelper/tag.go +++ b/internal/testhelper/tag.go @@ -3,6 +3,7 @@ package testhelper import ( "bytes" "fmt" + "strconv" "strings" "testing" ) @@ -43,3 +44,14 @@ func CreateTag(t *testing.T, repoPath, tagName, targetID string, opts *CreateTag tagID := MustRunCommand(t, nil, "git", "-C", repoPath, "rev-parse", tagName) return strings.TrimSpace(string(tagID)) } + +// GetTagDate gets the tag date in a timestamp form +func GetTagDate(t *testing.T, repoPath, tagName string) (int64, error) { + tagInfoLines := strings.Split(string(MustRunCommand(t, nil, "git", "-C", repoPath, "show", "--date=unix", tagName)), "\n") + timestampString := strings.TrimSpace(strings.SplitN(tagInfoLines[2], "Date:", 2)[1]) + timestamp, err := strconv.ParseInt(strings.TrimSpace(string(timestampString)), 10, 64) + if err != nil { + return 0, nil + } + return timestamp, nil +}