diff --git a/changelogs/unreleased/zj-repo-languages-failed-precond.yml b/changelogs/unreleased/zj-repo-languages-failed-precond.yml new file mode 100644 index 0000000000000000000000000000000000000000..d38c00a0bf4f459a08f22aac7d8647b907566e7d --- /dev/null +++ b/changelogs/unreleased/zj-repo-languages-failed-precond.yml @@ -0,0 +1,5 @@ +--- +title: Return FailedPrecondition when commit languages are requested too often +merge_request: 1690 +author: +type: performance diff --git a/internal/service/commit/languages.go b/internal/service/commit/languages.go index 8d73542e66671e496da7b720596d954cb6d4a63a..a9c2c06cb1713b10e3a12d2123cd77f9a79fef09 100644 --- a/internal/service/commit/languages.go +++ b/internal/service/commit/languages.go @@ -3,14 +3,16 @@ package commit import ( "context" "io/ioutil" + "os" + "path/filepath" "sort" + "time" "gitlab.com/gitlab-org/gitaly/internal/git" "gitlab.com/gitlab-org/gitaly/internal/helper" "gitlab.com/gitlab-org/gitaly/internal/helper/text" "gitlab.com/gitlab-org/gitaly/internal/linguist" "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag" - "gitlab.com/gitlab-org/gitaly/internal/service/ref" "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -19,18 +21,11 @@ import ( func (*server) CommitLanguages(ctx context.Context, req *gitalypb.CommitLanguagesRequest) (*gitalypb.CommitLanguagesResponse, error) { repo := req.Repository - if err := git.ValidateRevisionAllowEmpty(req.Revision); err != nil { + if err := git.ValidateRevision(req.Revision); err != nil { return nil, helper.ErrInvalidArgument(err) } revision := string(req.Revision) - if revision == "" { - defaultBranch, err := ref.DefaultBranchName(ctx, req.Repository) - if err != nil { - return nil, err - } - revision = string(defaultBranch) - } commitID, err := lookupRevision(ctx, repo, revision) if err != nil { @@ -41,6 +36,11 @@ func (*server) CommitLanguages(ctx context.Context, req *gitalypb.CommitLanguage if err != nil { return nil, err } + + if cacheRecentlyUpdated(repoPath) { + return nil, helper.ErrPreconditionFailedf("repository languages recently scanned") + } + stats, err := linguist.Stats(ctx, repoPath, commitID) if err != nil { return nil, err @@ -104,3 +104,18 @@ func lookupRevision(ctx context.Context, repo *gitalypb.Repository, revision str return text.ChompBytes(revParseBytes), nil } + +// Gitaly uses Linguist to detect the languages, and to improve performance, the +// results will be cached. If the modified time is later than a trashold it's +// considered recently updated enough that we do not scan again +func cacheRecentlyUpdated(path string) bool { + fullPath := filepath.Join(path, "language-stats.cache") + + fi, err := os.Stat(fullPath) + if err != nil { + return false + } + + const sentinel = -time.Hour * 24 * 7 + return fi.ModTime().After(time.Now().Add(sentinel)) +} diff --git a/internal/service/commit/languages_test.go b/internal/service/commit/languages_test.go index b456719a52b032e8d85b27f43e1cf95a98e21ddf..7218bcf519b5e5b484b6ffbf01d8c13f27727a22 100644 --- a/internal/service/commit/languages_test.go +++ b/internal/service/commit/languages_test.go @@ -53,6 +53,12 @@ func TestLanguages(t *testing.T) { actualLanguage := resp.Languages[i] requireLanguageEqual(t, &el, actualLanguage) } + + // if another request is made within short succession (1 week), the second + // one is rejected + _, err = client.CommitLanguages(ctx, request) + require.Error(t, err) + testhelper.RequireGrpcError(t, err, codes.FailedPrecondition) } func TestFileCountIsZeroWhenFeatureIsDisabled(t *testing.T) { @@ -92,36 +98,6 @@ func requireLanguageEqual(t *testing.T, expected, actual *gitalypb.CommitLanguag require.Equal(t, expected.Bytes, actual.Bytes) } -func TestLanguagesEmptyRevision(t *testing.T) { - server, serverSocketPath := startTestServices(t) - defer server.Stop() - - client, conn := newCommitServiceClient(t, serverSocketPath) - defer conn.Close() - - testRepo, _, cleanupFn := testhelper.NewTestRepo(t) - defer cleanupFn() - - request := &gitalypb.CommitLanguagesRequest{ - Repository: testRepo, - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - resp, err := client.CommitLanguages(ctx, request) - require.NoError(t, err) - - require.NotZero(t, len(resp.Languages), "number of languages in response") - - foundRuby := false - for _, l := range resp.Languages { - if l.Name == "Ruby" { - foundRuby = true - } - } - require.True(t, foundRuby, "expected to find Ruby as a language on HEAD") -} - func TestInvalidCommitLanguagesRequestRevision(t *testing.T) { server, serverSocketPath := startTestServices(t) defer server.Stop()