diff --git a/cmd/gitaly/main.go b/cmd/gitaly/main.go index 1571c1d73f43a7a7d7335aa3adcc62d17015f060..d62f9fc9c3eb2193a16bb2a6636aa643141fa7df 100644 --- a/cmd/gitaly/main.go +++ b/cmd/gitaly/main.go @@ -99,7 +99,22 @@ func configure(configPath string) (config.Cfg, error) { glog.Configure(glog.Loggers, cfg.Logging.Format, cfg.Logging.Level) if err := cgroups.NewManager(cfg.Cgroups).Setup(); err != nil { - return config.Cfg{}, fmt.Errorf("failed setting up cgroups: %w", err) + log.WithError(err).Error("failed setting up cgroups") + go func() { + ticker := time.NewTicker(5 * time.Minute) + defer ticker.Stop() + + for { + <-ticker.C + log.Info("attempting to setup cgroups") + + if err := cgroups.NewManager(cfg.Cgroups).Setup(); err != nil { + log.WithError(err).Error("failed setting up cgroups") + } else { + break + } + } + }() } if err := verifyGitVersion(cfg); err != nil { diff --git a/internal/git/command_factory.go b/internal/git/command_factory.go index c3be0eb6072dbf46ec1a2e9f7cc0a75839e6ce55..84afc108dbc30cf5b9f63afce20f79a60fc781de 100644 --- a/internal/git/command_factory.go +++ b/internal/git/command_factory.go @@ -6,6 +6,7 @@ import ( "fmt" "os/exec" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" "github.com/prometheus/client_golang/prometheus" "gitlab.com/gitlab-org/gitaly/v14/internal/cgroups" "gitlab.com/gitlab-org/gitaly/v14/internal/command" @@ -142,7 +143,7 @@ func (cf *ExecCommandFactory) newCommand(ctx context.Context, repo repository.Gi if featureflag.RunCommandsInCGroup.IsEnabled(ctx) { if err := cf.cgroupsManager.AddCommand(command); err != nil { - return nil, err + ctxlogrus.Extract(ctx).WithError(err).Error("could not add command to cgroup") } } diff --git a/internal/git/command_factory_cgroup_test.go b/internal/git/command_factory_cgroup_test.go index d2b0ab2f9a8fe85e7b4ce668dc8ae065d7427ee4..81dc56fab9d00f9ecb14c7cb43755fc3800d9b61 100644 --- a/internal/git/command_factory_cgroup_test.go +++ b/internal/git/command_factory_cgroup_test.go @@ -1,11 +1,13 @@ package git import ( + "errors" "os" "path/filepath" "testing" "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v14/internal/command" @@ -15,6 +17,14 @@ import ( "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper" ) +type mockErrorCgroupsManager struct { + mockCgroupsManager +} + +func (m *mockErrorCgroupsManager) AddCommand(_ *command.Command) error { + return errors.New("could not add command") +} + type mockCgroupsManager struct { commands []*command.Command } @@ -99,3 +109,50 @@ func TestNewCommandAddsToCgroup(t *testing.T) { }) } } + +func TestCgroupError(t *testing.T) { + root := testhelper.TempDir(t) + + cfg := config.Cfg{ + SocketPath: "/path/to/socket", + Cgroups: cgroups.Config{ + Count: 1, + }, + Storages: []config.Storage{{ + Name: "storage-1", + Path: root, + }}, + BinDir: filepath.Join(root, "bin.d"), + } + + require.NoError(t, os.MkdirAll(cfg.BinDir, 0o644)) + require.NoError(t, cfg.SetGitPath()) + require.NotEmpty(t, cfg.Git.BinPath) + + gitCmdFactory := NewExecCommandFactory(cfg) + + var manager mockErrorCgroupsManager + gitCmdFactory.cgroupsManager = &manager + + logger := testhelper.NewDiscardingLogEntry(t) + logHook := test.NewLocal(logger.Logger) + ctx, cancel := testhelper.Context(testhelper.ContextWithLogger(logger)) + defer cancel() + + cmd := SubCmd{ + Name: "hash-object", + } + + dir := testhelper.TempDir(t) + ctx = featureflag.IncomingCtxWithFeatureFlag(ctx, featureflag.RunCommandsInCGroup, true) + _, err := gitCmdFactory.NewWithDir(ctx, dir, &cmd) + require.NoError(t, err) + + var found bool + for _, entry := range logHook.AllEntries() { + if entry.Message == "could not add command to cgroup" { + found = true + } + } + assert.True(t, found) +}