diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dcf6e3375bc35e683d5801c2b1befeedf350ecfd..e3f40deeb69901690643056ae9b8a4db96dafda3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -256,7 +256,7 @@ test: - GO_VERSION: !reference [.versions, go_supported] TEST_TARGET: test - TEST_TARGET: - [test-with-praefect, race-go, test-wal, test-with-praefect-wal] + [test-with-praefect, race-go, test-wal, test-raft, test-with-praefect-wal] # We also verify that things work as expected with a non-bundled Git # version matching our minimum required Git version. - TEST_TARGET: test @@ -312,7 +312,7 @@ test:nightly: parallel: matrix: - GIT_VERSION: ["master", "next"] - TEST_TARGET: [test, test-with-praefect] + TEST_TARGET: [test, test-with-praefect, test-wal, test-raft] rules: - if: '$CI_PIPELINE_SOURCE == "schedule"' allow_failure: false @@ -331,7 +331,7 @@ test:sha256: parallel: matrix: - TEST_TARGET: - [test, test-with-praefect, test-wal, test-with-praefect-wal] + [test, test-with-praefect, test-wal, test-raft, test-with-praefect-wal] TEST_WITH_SHA256: "YesPlease" test:fips: @@ -354,7 +354,7 @@ test:fips: - test "$(cat /proc/sys/crypto/fips_enabled)" = "1" || (echo "System is not running in FIPS mode" && exit 1) parallel: matrix: - - TEST_TARGET: [test, test-with-praefect] + - TEST_TARGET: [test, test-with-praefect, test-wal, test-raft] FIPS_MODE: "YesPlease" GO_VERSION: !reference [.versions, go_supported] rules: diff --git a/Makefile b/Makefile index 1aacac7a1846b6aaaa6c36880bde3856255f40c7..d5f131946520b0b98e4f68c083933633c77b967d 100644 --- a/Makefile +++ b/Makefile @@ -406,6 +406,12 @@ test-with-praefect: test-go test-wal: export GITALY_TEST_WAL = YesPlease test-wal: test-go +.PHONY: test-raft +## Run Go tests with write-ahead logging + Raft enabled. +test-raft: export GITALY_TEST_WAL = YesPlease +test-raft: export GITALY_TEST_RAFT = YesPlease +test-raft: test-go + .PHONY: test-with-praefect-wal ## Run Go tests with write-ahead logging and Praefect enabled. test-with-praefect-wal: export GITALY_TEST_WAL = YesPlease diff --git a/go.mod b/go.mod index 0fca8abe8f210ac68e3bd43677fc38dab566e6ea..e1a13751d6839ed9a0e041d88f939050f2978ad8 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/urfave/cli/v2 v2.27.5 gitlab.com/gitlab-org/labkit v1.21.2 + go.etcd.io/etcd/raft/v3 v3.5.16 go.uber.org/automaxprocs v1.6.0 go.uber.org/goleak v1.3.0 gocloud.dev v0.40.0 diff --git a/go.sum b/go.sum index 6b8da5724b6c59a6b5175c1bcdb8479ce76c0554..fb668be6d88458e9c608a40f9c3417ee44e1e1bc 100644 --- a/go.sum +++ b/go.sum @@ -189,6 +189,8 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI= github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= @@ -645,6 +647,10 @@ gitlab.com/gitlab-org/go/reopen v1.0.0 h1:6BujZ0lkkjGIejTUJdNO1w56mN1SI10qcVQyQl gitlab.com/gitlab-org/go/reopen v1.0.0/go.mod h1:D6OID8YJDzEVZNYW02R/Pkj0v8gYFSIhXFTArAsBQw8= gitlab.com/gitlab-org/labkit v1.21.2 h1:GlFHh8OdkrIMH3Qi0ByOzva0fGYXMICsuahGpJe4KNQ= gitlab.com/gitlab-org/labkit v1.21.2/go.mod h1:Q++SWyCH/abH2pytnX2SU/3mrCX6aK/xKz/WpM1hLbA= +go.etcd.io/etcd/client/pkg/v3 v3.5.16 h1:ZgY48uH6UvB+/7R9Yf4x574uCO3jIx0TRDyetSfId3Q= +go.etcd.io/etcd/client/pkg/v3 v3.5.16/go.mod h1:V8acl8pcEK0Y2g19YlOV9m9ssUe6MgiDSobSoaBAM0E= +go.etcd.io/etcd/raft/v3 v3.5.16 h1:zBXA3ZUpYs1AwiLGPafYAKKl/CORn/uaxYDwlNwndAk= +go.etcd.io/etcd/raft/v3 v3.5.16/go.mod h1:P4UP14AxofMJ/54boWilabqqWoW9eLodl6I5GdGzazI= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/internal/backup/partition_backup_test.go b/internal/backup/partition_backup_test.go index 44a3cba1b5d6f5246bdc78413c5250f585bedd3d..5d2e65784e72841aa01e716b576900d8586813cc 100644 --- a/internal/backup/partition_backup_test.go +++ b/internal/backup/partition_backup_test.go @@ -110,8 +110,12 @@ func TestPartitionBackup_CreateSuccess(t *testing.T) { require.NoError(t, err) + testhelper.SkipWithRaft(t, `The test asserts the existence of backup files based on the latest + LSN. When Raft is not enabled, the LSN is not static. The test should fetch the latest + LSN instead https://gitlab.com/gitlab-org/gitaly/-/issues/6459`) + lsn := testhelper.WithOrWithoutRaft(storage.LSN(3), storage.LSN(1)) for _, expectedArchive := range tc.expectedArchives { - tarPath := filepath.Join(backupRoot, cfg.Storages[0].Name, expectedArchive, storage.LSN(1).String()) + ".tar" + tarPath := filepath.Join(backupRoot, cfg.Storages[0].Name, expectedArchive, lsn.String()+".tar") tar, err := os.Open(tarPath) require.NoError(t, err) testhelper.MustClose(t, tar) diff --git a/internal/cli/gitaly/serve.go b/internal/cli/gitaly/serve.go index 00e5dfd4e4dfd417f24f37979fe3d89318a04937..4d3be3b3e38258432b9b5ebf2c44c066271f5304 100644 --- a/internal/cli/gitaly/serve.go +++ b/internal/cli/gitaly/serve.go @@ -37,6 +37,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue/databasemgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/mdfile" nodeimpl "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/node" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/partition" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/snapshot" @@ -404,6 +405,13 @@ func run(appCtx *cli.Context, cfg config.Cfg, logger log.Logger) error { logConsumer = walArchiver } + var raftManagerFactory raftmgr.RaftManagerFactory + if cfg.Raft.Enabled { + raftManagerFactory = func(ptnID storage.PartitionID, storageName string, db keyvalue.Transactioner, logger log.Logger) (*raftmgr.Manager, error) { + return raftmgr.NewManager(ptnID, storageName, cfg.Raft, db, logger) + } + } + nodeMgr, err := nodeimpl.NewManager( cfg.Storages, storagemgr.NewFactory( @@ -414,6 +422,7 @@ func run(appCtx *cli.Context, cfg config.Cfg, logger log.Logger) error { localrepo.NewFactory(logger, locator, gitCmdFactory, catfileCache), partitionMetrics, logConsumer, + raftManagerFactory, ), 2, storageMetrics, @@ -460,6 +469,7 @@ func run(appCtx *cli.Context, cfg config.Cfg, logger log.Logger) error { localrepo.NewFactory(logger, locator, gitCmdFactory, catfileCache), partitionMetrics, nil, + nil, ), // In recovery mode we don't want to keep inactive partitions active. The cache // however can't be disabled so simply set it to one. diff --git a/internal/cli/gitaly/subcmd_recovery.go b/internal/cli/gitaly/subcmd_recovery.go index 9d0337ac5d22c7720a9220e92c5cdd1b07b87cfb..a46c1648718f1764d22c0d7d225bfb1ad893b40e 100644 --- a/internal/cli/gitaly/subcmd_recovery.go +++ b/internal/cli/gitaly/subcmd_recovery.go @@ -126,6 +126,7 @@ func recoveryStatusAction(ctx *cli.Context) (returnErr error) { localrepo.NewFactory(logger, locator, gitCmdFactory, catfileCache), partitionMetrics, nil, + nil, ), 1, storageMetrics, diff --git a/internal/cli/gitaly/subcmd_recovery_test.go b/internal/cli/gitaly/subcmd_recovery_test.go index e0d78a56b004a17fd220accd163232743c991d54..d70e2896eccebb6a4fe1110cd4e35ca4baf8d985 100644 --- a/internal/cli/gitaly/subcmd_recovery_test.go +++ b/internal/cli/gitaly/subcmd_recovery_test.go @@ -168,6 +168,7 @@ Relative paths: localrepo.NewFactory(logger, locator, gitCmdFactory, catfileCache), partition.NewMetrics(housekeeping.NewMetrics(cfg.Prometheus), snapshot.NewMetrics()), nil, + nil, ), 1, storagemgr.NewMetrics(cfg.Prometheus), diff --git a/internal/cli/gitalybackup/partition_test.go b/internal/cli/gitalybackup/partition_test.go index 29e0bc7f24e5f2ad24b580610452a0560e8cd5a6..c74c953bb5def2e8e8c7661bb9437c489d6e85a7 100644 --- a/internal/cli/gitalybackup/partition_test.go +++ b/internal/cli/gitalybackup/partition_test.go @@ -109,6 +109,9 @@ func TestPartitionSubcommand_Create(t *testing.T) { } require.NoError(t, err) + testhelper.SkipWithRaft(t, `The test asserts the existence of backup files based on the latest + LSN. When Raft is not enabled, the LSN is not static. The test should fetch the latest + LSN instead https://gitlab.com/gitlab-org/gitaly/-/issues/6459`) lsn := storage.LSN(1) tarPath := filepath.Join(path, cfg.Storages[0].Name, "2", lsn.String()) + ".tar" tar, err := os.Open(tarPath) diff --git a/internal/git/housekeeping/manager/testhelper_test.go b/internal/git/housekeeping/manager/testhelper_test.go index ffdbf029bd0a7bbdb8d5d48106e043c1d7a4759a..4f2a8547ca5d0e689635dd74a71826fe12de16cf 100644 --- a/internal/git/housekeeping/manager/testhelper_test.go +++ b/internal/git/housekeeping/manager/testhelper_test.go @@ -113,6 +113,7 @@ func testWithAndWithoutTransaction(t *testing.T, ctx context.Context, desc strin snapshot.NewMetrics(), ), nil, + nil, ), storagemgr.DefaultMaxInactivePartitions, storagemgr.NewMetrics(cfg.Prometheus), diff --git a/internal/git/objectpool/fetch_test.go b/internal/git/objectpool/fetch_test.go index b502b45925a64a120a268c46253d7a1424e3abd0..89f7a454fb0c25c6a07abc89c8d4610ccd6a9cac 100644 --- a/internal/git/objectpool/fetch_test.go +++ b/internal/git/objectpool/fetch_test.go @@ -431,6 +431,7 @@ func testWithAndWithoutTransaction(t *testing.T, ctx context.Context, testFunc f localRepoFactory, partition.NewMetrics(housekeeping.NewMetrics(cfg.Prometheus), snapshot.NewMetrics()), nil, + nil, ), storagemgr.DefaultMaxInactivePartitions, storagemgr.NewMetrics(cfg.Prometheus), diff --git a/internal/gitaly/config/config.go b/internal/gitaly/config/config.go index 4f92c006f6aaeee7d8b80e19b8a81b98e5e5a742..8b775f39b892ecebe3ff76667df3815b56431f2d 100644 --- a/internal/gitaly/config/config.go +++ b/internal/gitaly/config/config.go @@ -1235,8 +1235,6 @@ type Raft struct { Enabled bool `json:"enabled" toml:"enabled"` // ClusterID is the unique ID of the cluster. It ensures the current node joins the right cluster. ClusterID string `json:"cluster_id" toml:"cluster_id"` - // NodeID is the unique ID of the node. - NodeID uint64 `json:"node_id" toml:"node_id"` // RTTMilliseconds is the maximum round trip between two nodes in the cluster. It's used to // calculate multiple types of timeouts of Raft protocol. RTTMilliseconds uint64 `json:"rtt_milliseconds" toml:"rtt_milliseconds"` @@ -1258,6 +1256,12 @@ const ( RaftDefaultHeartbeatTicks = 2 ) +// DefaultRaftConfig returns a Raft configuration filled with default values. +func DefaultRaftConfig(clusterID string) Raft { + r := Raft{Enabled: true, ClusterID: clusterID} + return r.fulfillDefaults() +} + func (r Raft) fulfillDefaults() Raft { if r.RTTMilliseconds == 0 { r.RTTMilliseconds = RaftDefaultRTT @@ -1287,7 +1291,6 @@ func (r Raft) Validate(transactions Transactions) error { cfgErr = cfgErr. Append(cfgerror.NotEmpty(r.ClusterID), "cluster_id"). - Append(cfgerror.Comparable(r.NodeID).GreaterThan(0), "node_id"). Append(cfgerror.Comparable(r.RTTMilliseconds).GreaterThan(0), "rtt_millisecond"). Append(cfgerror.Comparable(r.ElectionTicks).GreaterThan(0), "election_rtt"). Append(cfgerror.Comparable(r.HeartbeatTicks).GreaterThan(0), "heartbeat_rtt") diff --git a/internal/gitaly/config/config_test.go b/internal/gitaly/config/config_test.go index dcc94004ad17a2c3db9a0615961e32f1d3e7ffdb..b49490bde7d5c984a02457109efff6f1fd0af1f3 100644 --- a/internal/gitaly/config/config_test.go +++ b/internal/gitaly/config/config_test.go @@ -2802,7 +2802,6 @@ func TestRaftConfig_Validate(t *testing.T) { cfgRaft: Raft{ Enabled: true, ClusterID: "4f04a0e2-0db8-4bfa-b846-01b5b4a093fb", - NodeID: 1, RTTMilliseconds: 200, ElectionTicks: 20, HeartbeatTicks: 2, @@ -2814,7 +2813,6 @@ func TestRaftConfig_Validate(t *testing.T) { cfgRaft: Raft{ Enabled: true, ClusterID: "", - NodeID: 1, RTTMilliseconds: 200, ElectionTicks: 20, HeartbeatTicks: 2, @@ -2832,7 +2830,6 @@ func TestRaftConfig_Validate(t *testing.T) { cfgRaft: Raft{ Enabled: true, ClusterID: "1234", - NodeID: 1, RTTMilliseconds: 200, ElectionTicks: 20, HeartbeatTicks: 2, @@ -2845,30 +2842,11 @@ func TestRaftConfig_Validate(t *testing.T) { ), }, }, - { - name: "invalid node ID", - cfgRaft: Raft{ - Enabled: true, - ClusterID: "4f04a0e2-0db8-4bfa-b846-01b5b4a093fb", - NodeID: 0, - RTTMilliseconds: 200, - ElectionTicks: 20, - HeartbeatTicks: 2, - }, - cfgTransactions: Transactions{Enabled: true}, - expectedErr: cfgerror.ValidationErrors{ - cfgerror.NewValidationError( - fmt.Errorf("%w: 0 is not greater than 0", cfgerror.ErrNotInRange), - "node_id", - ), - }, - }, { name: "invalid RTT", cfgRaft: Raft{ Enabled: true, ClusterID: "4f04a0e2-0db8-4bfa-b846-01b5b4a093fb", - NodeID: 1, RTTMilliseconds: 0, ElectionTicks: 20, HeartbeatTicks: 2, @@ -2886,7 +2864,6 @@ func TestRaftConfig_Validate(t *testing.T) { cfgRaft: Raft{ Enabled: true, ClusterID: "4f04a0e2-0db8-4bfa-b846-01b5b4a093fb", - NodeID: 1, RTTMilliseconds: 200, ElectionTicks: 0, HeartbeatTicks: 2, @@ -2904,7 +2881,6 @@ func TestRaftConfig_Validate(t *testing.T) { cfgRaft: Raft{ Enabled: true, ClusterID: "4f04a0e2-0db8-4bfa-b846-01b5b4a093fb", - NodeID: 1, RTTMilliseconds: 200, ElectionTicks: 20, HeartbeatTicks: 0, @@ -2922,7 +2898,6 @@ func TestRaftConfig_Validate(t *testing.T) { cfgRaft: Raft{ Enabled: true, ClusterID: "4f04a0e2-0db8-4bfa-b846-01b5b4a093fb", - NodeID: 1, RTTMilliseconds: 200, ElectionTicks: 20, HeartbeatTicks: 2, @@ -2958,7 +2933,6 @@ initial_members = {1 = "localhost:4001", 2 = "localhost:4002", 3 = "localhost:40 Raft: Raft{ Enabled: true, ClusterID: "7766d7c1-7266-4bc9-9dad-5ee8617c455b", - NodeID: 1, RTTMilliseconds: 200, ElectionTicks: 20, HeartbeatTicks: 0, diff --git a/internal/gitaly/service/partition/backup_partition_test.go b/internal/gitaly/service/partition/backup_partition_test.go index 501680df67f1b06522bdf7a09fd0b54e926656fd..a75ef5092c2eed013040a6f69f96fdd74dfa19d6 100644 --- a/internal/gitaly/service/partition/backup_partition_test.go +++ b/internal/gitaly/service/partition/backup_partition_test.go @@ -136,6 +136,9 @@ func TestBackupPartition(t *testing.T) { require.NoError(t, err) testhelper.ProtoEqual(t, &gitalypb.BackupPartitionResponse{}, resp) + testhelper.SkipWithRaft(t, `The test asserts the existence of backup files based on the latest + LSN. When Raft is not enabled, the LSN is not static. The test should fetch the latest + LSN instead https://gitlab.com/gitlab-org/gitaly/-/issues/6459`) lsn := storage.LSN(2) tarPath := filepath.Join(backupRoot, data.storageName, data.partitionID, lsn.String()) + ".tar" tar, err := os.Open(tarPath) diff --git a/internal/gitaly/storage/raftmgr/entry_recorder.go b/internal/gitaly/storage/raftmgr/entry_recorder.go new file mode 100644 index 0000000000000000000000000000000000000000..2af9103d45dbc31830dc235c34d4b9d4ad81ddc4 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/entry_recorder.go @@ -0,0 +1,138 @@ +package raftmgr + +import ( + "sync" + + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" + "google.golang.org/protobuf/proto" +) + +// EntryRecorder is a utility for recording and classifying log entries processed by the Raft manager. In addition to +// standard log entries, Raft may generate internal entries such as configuration changes or empty logs for verification +// purposes. These internal entries are backfilled into the Write-Ahead Log (WAL), occupying Log Sequence Number (LSN) +// slots. Consequently, the LSN sequence may diverge from expectations when Raft is enabled. +// +// This recorder is equipped with the capability to offset the LSN, which is particularly useful in testing environments +// to mitigate differences in LSN sequences. It is strongly advised that this feature be restricted to testing purposes +// and not utilized in production or other non-testing scenarios. +type EntryRecorder struct { + mu sync.Mutex // Mutex for safe concurrent access + Items []Item // Slice to store recorded log entries +} + +// Item represents an entry recorded by the LogEntryRecorder, with a flag indicating if it's from Raft. +type Item struct { + FromRaft bool + LSN storage.LSN + Entry *gitalypb.LogEntry +} + +// NewEntryRecorder returns a new instance of NewEntryRecorder. +func NewEntryRecorder() *EntryRecorder { + return &EntryRecorder{} +} + +// Record logs an entry, marking it as originating from Raft if specified. +// If the LSN is from the past, it removes all entries after that LSN. +func (r *EntryRecorder) Record(fromRaft bool, lsn storage.LSN, entry *gitalypb.LogEntry) { + r.mu.Lock() + defer r.mu.Unlock() + + // Trim items that have an LSN greater than the current LSN + if len(r.Items) > 0 { + idx := -1 + for i, itm := range r.Items { + if itm.LSN >= lsn { + idx = i + break + } + } + if idx != -1 { + r.Items = r.Items[:idx] + } + } + + // Append the new entry + r.Items = append(r.Items, Item{ + FromRaft: fromRaft, + LSN: lsn, + Entry: proto.Clone(entry).(*gitalypb.LogEntry), + }) +} + +// Offset adjusts the log sequence number (LSN) by accounting for internal Raft entries that may occupy slots. +func (r *EntryRecorder) Offset(lsn storage.LSN) storage.LSN { + r.mu.Lock() + defer r.mu.Unlock() + + offset := lsn + for _, itm := range r.Items { + if itm.FromRaft { + offset++ + } + if itm.LSN >= offset { + break + } + } + return offset +} + +// Metadata returns metadata of an entry if it exists. +func (r *EntryRecorder) Metadata(lsn storage.LSN) *gitalypb.LogEntry_Metadata { + r.mu.Lock() + defer r.mu.Unlock() + + for _, itm := range r.Items { + if itm.LSN == lsn { + return itm.Entry.GetMetadata() + } + } + return nil +} + +// Latest returns the latest recorded LSN. +func (r *EntryRecorder) Latest() storage.LSN { + r.mu.Lock() + defer r.mu.Unlock() + + if len(r.Items) == 0 { + return 0 + } + return r.Items[len(r.Items)-1].LSN +} + +// Len returns the length of recorded entries. +func (r *EntryRecorder) Len() int { + r.mu.Lock() + defer r.mu.Unlock() + + return len(r.Items) +} + +// IsFromRaft returns true if the asserting LSN is an entry emitted by Raft. +func (r *EntryRecorder) IsFromRaft(lsn storage.LSN) bool { + r.mu.Lock() + defer r.mu.Unlock() + + for _, itm := range r.Items { + if itm.LSN == lsn { + return itm.FromRaft + } + } + return false +} + +// FromRaft retrieves all log entries that originated from the Raft system. +func (r *EntryRecorder) FromRaft() map[storage.LSN]*gitalypb.LogEntry { + r.mu.Lock() + defer r.mu.Unlock() + + raftEntries := map[storage.LSN]*gitalypb.LogEntry{} + for _, itm := range r.Items { + if itm.FromRaft { + raftEntries[itm.LSN] = itm.Entry + } + } + return raftEntries +} diff --git a/internal/gitaly/storage/raftmgr/entry_recorder_test.go b/internal/gitaly/storage/raftmgr/entry_recorder_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6e1d35c60a7c6527c25f3c3e090f0d7c2f2e406b --- /dev/null +++ b/internal/gitaly/storage/raftmgr/entry_recorder_test.go @@ -0,0 +1,137 @@ +package raftmgr + +import ( + "testing" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" +) + +func TestLogEntryRecorder(t *testing.T) { + testCases := []struct { + desc string + records []Item + initialLSN storage.LSN + expectedOffset storage.LSN + }{ + { + desc: "no entries", + records: []Item{}, + initialLSN: 1, + expectedOffset: 1, + }, + { + desc: "no internal raft entries", + records: []Item{ + {FromRaft: false, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 2, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 1, + expectedOffset: 1, + }, + { + desc: "no internal raft entries", + records: []Item{ + {FromRaft: false, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 2, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 2, + expectedOffset: 2, + }, + { + desc: "all raft entries", + records: []Item{ + {FromRaft: true, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: true, LSN: 2, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 1, + expectedOffset: 3, // Two raft entries increment the offset by 2 + }, + { + desc: "mix of raft and non-raft, with raft at beginning", + records: []Item{ + {FromRaft: true, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 2, Entry: &gitalypb.LogEntry{}}, + {FromRaft: true, LSN: 3, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 1, + expectedOffset: 2, + }, + { + desc: "mix of raft and non-raft, with raft at beginning", + records: []Item{ + {FromRaft: true, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 2, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 3, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 2, + expectedOffset: 3, + }, + { + desc: "mix of raft and non-raft, with raft at beginning", + records: []Item{ + {FromRaft: true, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 2, Entry: &gitalypb.LogEntry{}}, + {FromRaft: true, LSN: 3, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 3, + expectedOffset: 5, + }, + { + desc: "mix of raft and non-raft, no raft at beginning", + records: []Item{ + {FromRaft: false, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: true, LSN: 2, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 3, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 1, + expectedOffset: 1, + }, + { + desc: "mix of raft and non-raft, no raft at beginning", + records: []Item{ + {FromRaft: false, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: true, LSN: 2, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 3, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 3, + expectedOffset: 4, + }, + { + desc: "initial LSN beyond recorded entries", + records: []Item{ + {FromRaft: false, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: true, LSN: 2, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 3, + expectedOffset: 4, + }, + { + desc: "replace entries with a past LSN", + records: []Item{ + {FromRaft: true, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 2, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 3, Entry: &gitalypb.LogEntry{}}, + {FromRaft: true, LSN: 2, Entry: &gitalypb.LogEntry{}}, // Insert with past LSN, should remove LSN 2 and 3 entry + }, + initialLSN: 1, + expectedOffset: 3, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + recorder := EntryRecorder{} + + // Record entries as specified in the test case + for _, record := range tc.records { + recorder.Record(record.FromRaft, record.LSN, record.Entry) + } + + // Validate the offset + offset := recorder.Offset(tc.initialLSN) + require.Equal(t, tc.expectedOffset, offset) + }) + } +} diff --git a/internal/gitaly/storage/raftmgr/leadership.go b/internal/gitaly/storage/raftmgr/leadership.go new file mode 100644 index 0000000000000000000000000000000000000000..fa130a5513bba9cc5bdc9644158958cc8d1bdc05 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/leadership.go @@ -0,0 +1,63 @@ +package raftmgr + +import ( + "sync" + "time" +) + +// Leadership manages the state of leadership in a distributed system using the Raft consensus algorithm. It tracks the +// current leader's ID, whether this node is the leader, and the time since the last leadership change. It also provides +// a notification mechanism for leadership changes through a channel. +type Leadership struct { + sync.Mutex + leaderID uint64 + isLeader bool + lastChange time.Time + newLeaderC chan struct{} +} + +// NewLeadership initializes a new Leadership instance with the current time and a buffered channel. +func NewLeadership() *Leadership { + return &Leadership{ + lastChange: time.Now(), + newLeaderC: make(chan struct{}, 1), + } +} + +// SetLeader updates the leadership information if there is a change in the leaderID. +// It returns a boolean indicating whether a change occurred and the duration of the last leadership. +func (l *Leadership) SetLeader(leaderID uint64, isLeader bool) (changed bool, lastDuration time.Duration) { + l.Lock() + defer l.Unlock() + + if l.leaderID == leaderID && l.isLeader == isLeader { + return false, 0 + } + + l.leaderID = leaderID + l.isLeader = isLeader + now := time.Now() + lastDuration = now.Sub(l.lastChange) + l.lastChange = now + + select { + case l.newLeaderC <- struct{}{}: + default: + } + + return true, lastDuration +} + +// IsLeader returns true if the current instance is the leader. +func (l *Leadership) IsLeader() bool { + l.Lock() + defer l.Unlock() + return l.isLeader +} + +// GetLeaderID retrieves the current leader's ID. +func (l *Leadership) GetLeaderID() uint64 { + l.Lock() + defer l.Unlock() + return l.leaderID +} diff --git a/internal/gitaly/storage/raftmgr/leadership_test.go b/internal/gitaly/storage/raftmgr/leadership_test.go new file mode 100644 index 0000000000000000000000000000000000000000..a4f9bf9ff06a9276e1c111a36546637f1c0e6612 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/leadership_test.go @@ -0,0 +1,116 @@ +package raftmgr + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestLeadership(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + initialLeader uint64 + leaderID uint64 + isLeader bool + wantChanged bool + wantDuration time.Duration // Maximum acceptable duration difference for test purposes + wantIsLeader bool + }{ + { + name: "Initial leader set", + initialLeader: 0, + leaderID: 1, + isLeader: true, + wantChanged: true, + wantDuration: 0, + wantIsLeader: true, + }, + { + name: "No leader change", + initialLeader: 1, + leaderID: 1, + isLeader: true, + wantChanged: false, + wantDuration: 0, + wantIsLeader: true, + }, + { + name: "Change leader and role", + initialLeader: 1, + leaderID: 2, + isLeader: false, + wantChanged: true, + wantDuration: 0, + wantIsLeader: false, + }, + { + name: "Become leader again", + initialLeader: 2, + leaderID: 2, + isLeader: true, + wantChanged: false, + wantDuration: 0, + wantIsLeader: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + leadership := NewLeadership() + // Initial leader setup if needed + if tc.initialLeader != 0 { + leadership.SetLeader(tc.initialLeader, true) + } + + changed, duration := leadership.SetLeader(tc.leaderID, tc.isLeader) + require.Equal(t, tc.wantChanged, changed, "Leadership change status mismatch") + if tc.wantChanged { + require.InDelta(t, tc.wantDuration, duration.Milliseconds(), 100, "Unexpected leadership duration difference") + } + require.Equal(t, tc.wantIsLeader, leadership.IsLeader(), "Leadership role mismatch") + require.Equal(t, tc.leaderID, leadership.GetLeaderID(), "Leader ID mismatch") + + // Verify the channel behavior if changed + if tc.wantChanged { + select { + case <-leadership.newLeaderC: + // Success, channel was correctly notified + case <-time.After(1 * time.Second): + t.Fatal("Expected newLeaderC to be notified but it wasn't") + } + } + }) + } +} + +func TestLeadership_MultipleChanges(t *testing.T) { + t.Parallel() + leadership := NewLeadership() + + changes := []struct { + leaderID uint64 + isLeader bool + }{ + {1, true}, + {2, false}, + {3, true}, + } + + for _, change := range changes { + // newLeaderC should not block. + leadership.SetLeader(change.leaderID, change.isLeader) + } + + require.Equal(t, true, leadership.IsLeader(), "Leadership role mismatch") + require.Equal(t, uint64(3), leadership.GetLeaderID(), "Leader ID mismatch") + + select { + case <-leadership.newLeaderC: + // Channel was correctly notified + default: + t.Fatal("Expected newLeaderC to be notified") + } +} diff --git a/internal/gitaly/storage/raftmgr/logger.go b/internal/gitaly/storage/raftmgr/logger.go new file mode 100644 index 0000000000000000000000000000000000000000..710721981e2aabe6c821438353051f6724b76c8f --- /dev/null +++ b/internal/gitaly/storage/raftmgr/logger.go @@ -0,0 +1,93 @@ +package raftmgr + +import ( + "fmt" + "strings" + + "gitlab.com/gitlab-org/gitaly/v16/internal/log" +) + +// raftLogger is a wrapper around the log.Logger interface to +// implement etcd/raft's logger interface methods. +type raftLogger struct { + logger log.Logger +} + +// Debug logs a debug level message using variable arguments. +func (l *raftLogger) Debug(args ...any) { + l.Debugf(l.generateFormatString(len(args)), args...) +} + +// Debugf logs a formatted debug level message. +func (l *raftLogger) Debugf(format string, args ...any) { + l.logger.Debug(fmt.Sprintf(format, args...)) +} + +// Info logs an info level message using variable arguments. +func (l *raftLogger) Info(args ...any) { + l.Infof(l.generateFormatString(len(args)), args...) +} + +// Infof logs a formatted info level message. +func (l *raftLogger) Infof(format string, args ...any) { + l.logger.Info(fmt.Sprintf(format, args...)) +} + +// Warning logs a warning level message using variable arguments. +func (l *raftLogger) Warning(args ...any) { + l.Warningf(l.generateFormatString(len(args)), args...) +} + +// Warningf logs a formatted warning level message. +func (l *raftLogger) Warningf(format string, args ...any) { + l.logger.Warn(fmt.Sprintf(format, args...)) +} + +// Error logs an error level message using variable arguments. +func (l *raftLogger) Error(args ...any) { + l.Errorf(l.generateFormatString(len(args)), args...) +} + +// Errorf logs a formatted error level message. +func (l *raftLogger) Errorf(format string, args ...any) { + l.logger.Error(fmt.Sprintf(format, args...)) +} + +// Fatal logs a fatal level message using variable arguments. +func (l *raftLogger) Fatal(args ...any) { + l.Fatalf(l.generateFormatString(len(args)), args...) +} + +// Fatalf logs a formatted fatal level message using error level log. +func (l *raftLogger) Fatalf(format string, args ...any) { + l.logger.Error(fmt.Sprintf(format, args...)) +} + +// Panic logs a panic level message using variable arguments. +func (l *raftLogger) Panic(args ...any) { + l.Panicf(l.generateFormatString(len(args)), args...) +} + +// Panicf logs a formatted panic level message. +func (l *raftLogger) Panicf(format string, args ...any) { + msg := fmt.Sprintf(format, args...) + l.logger.Error(msg) + panic(msg) +} + +// generateFormatString generates a format string for the provided number of arguments. +func (l *raftLogger) generateFormatString(argCount int) string { + switch argCount { + case 0: + return "" + case 1: + // Fast path, most common use cases. + return "%s" + default: + formatSpecifiers := make([]string, argCount) + for i := 0; i < argCount; i++ { + formatSpecifiers[i] = "%s" + } + return strings.Join(formatSpecifiers, " ") + } +} diff --git a/internal/gitaly/storage/raftmgr/logger_test.go b/internal/gitaly/storage/raftmgr/logger_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0bac85f3707e3bf67b43fdda562085f70ec6595c --- /dev/null +++ b/internal/gitaly/storage/raftmgr/logger_test.go @@ -0,0 +1,135 @@ +package raftmgr + +import ( + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" +) + +func TestRaftLogger(t *testing.T) { + t.Parallel() + + newLogger := func(t *testing.T) (*raftLogger, testhelper.LoggerHook) { + logger := testhelper.NewLogger(t, testhelper.WithLevel(logrus.DebugLevel)) + hook := testhelper.AddLoggerHook(logger) + hook.Reset() + return &raftLogger{logger: logger}, hook + } + + tests := []struct { + name string + logFunc func(logger *raftLogger, message string, args ...any) + level string + message string + args []any + expected string + }{ + { + name: "Debugf", + logFunc: func(l *raftLogger, msg string, args ...any) { l.Debugf(msg, args...) }, + level: "debug", + message: "debug message %s", + args: []any{"test"}, + expected: "debug message test", + }, + { + name: "Debug", + logFunc: func(l *raftLogger, msg string, args ...any) { l.Debug(msg) }, + level: "debug", + message: "debug message test", + expected: "debug message test", + }, + { + name: "Infof", + logFunc: func(l *raftLogger, msg string, args ...any) { l.Infof(msg, args...) }, + level: "info", + message: "info message %+s", + args: []any{"test"}, + expected: "info message test", + }, + { + name: "Info", + logFunc: func(l *raftLogger, msg string, args ...any) { l.Info(msg) }, + level: "info", + message: "info message test", + expected: "info message test", + }, + { + name: "Warningf", + logFunc: func(l *raftLogger, msg string, args ...any) { l.Warningf(msg, args...) }, + level: "warning", + message: "warning message %s", + args: []any{"test"}, + expected: "warning message test", + }, + { + name: "Warning", + logFunc: func(l *raftLogger, msg string, args ...any) { l.Warning(msg) }, + level: "warning", + message: "warning message test", + expected: "warning message test", + }, + { + name: "Errorf", + logFunc: func(l *raftLogger, msg string, args ...any) { l.Errorf(msg, args...) }, + level: "error", + message: "error message %s", + args: []any{"test"}, + expected: "error message test", + }, + { + name: "Error", + logFunc: func(l *raftLogger, msg string, args ...any) { l.Error(msg) }, + level: "error", + message: "error message test", + expected: "error message test", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + logger, hook := newLogger(t) + + tc.logFunc(logger, tc.message, tc.args...) + + entries := hook.AllEntries() + require.Equal(t, 1, len(entries), "expected exactly one log entry") + require.Equal(t, tc.expected, entries[0].Message) + require.Equal(t, tc.level, entries[0].Level.String()) + }) + } + + t.Run("Fatal", func(t *testing.T) { + t.Parallel() + + logger, hook := newLogger(t) + logger.Fatal("fatal message", "test") + + entries := hook.AllEntries() + require.Equal(t, 1, len(entries), "expected exactly one log entry after fatal") + require.Equal(t, "fatal message test", entries[0].Message) + require.Equal(t, "error", entries[0].Level.String()) // Assuming Fatal logs as error level before exiting + }) + + t.Run("Panic", func(t *testing.T) { + t.Parallel() + + logger, hook := newLogger(t) + func() { + defer func() { + r := recover() + require.NotNilf(t, r, "calling logger.Panic should raise a panic") + + entries := hook.AllEntries() + require.Equal(t, 1, len(entries), "expected exactly one log entry after panic") + require.Equal(t, "panic message test", entries[0].Message) + require.Equal(t, "error", entries[0].Level.String()) // Assuming Panic logs as error level before panicking + }() + logger.Panic("panic message", "test") + }() + }) +} diff --git a/internal/gitaly/storage/raftmgr/manager.go b/internal/gitaly/storage/raftmgr/manager.go new file mode 100644 index 0000000000000000000000000000000000000000..ec06e2e3e796ac095b2d4e4c4fd1416c4988ecd6 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/manager.go @@ -0,0 +1,689 @@ +package raftmgr + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" + "gitlab.com/gitlab-org/gitaly/v16/internal/safe" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" + "go.etcd.io/etcd/raft/v3" + "go.etcd.io/etcd/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +const ( + defaultMaxSizePerMsg = 10 * 1024 * 1024 + defaultMaxInflightMsgs = 256 +) + +// ready manages the readiness signaling. The caller polls from the channel. +type ready struct { + c chan struct{} // Channel to signal readiness + once sync.Once // Ensures readiness is signaled only once +} + +// set signals readiness by closing the channel, ensuring it +// happens only once. +func (r *ready) set() { + r.once.Do(func() { close(r.c) }) +} + +// ManagerOptions encapsulates optional parameters for configuring +// the Manager. +type ManagerOptions struct { + // readyTimeout sets an timeout when waiting for Raft to be ready. + readyTimeout time.Duration + // opTimeout sets a timeout for propose, append, and commit operations. It's meaningful in tests so that we + // could detect deadlocks and intolerant performance degrades. + opTimeout time.Duration + // entryRecorder indicates whether to store all log entries processed by Raft. It is primarily + // used for testing purposes. + entryRecorder *EntryRecorder + // recordTransport indicates whether to store all messages supposed to sent over the network. It is + // primarily used for testing purposes. + recordTransport bool +} + +// OptionFunc defines a function type for applying optional +// configuration to ManagerOptions. +type OptionFunc func(opt ManagerOptions) ManagerOptions + +// WithReadyTimeout sets a timeout when waiting for Raft service to be ready. +// The default timeout is 5 * election timeout. +func WithReadyTimeout(t time.Duration) OptionFunc { + return func(opt ManagerOptions) ManagerOptions { + opt.readyTimeout = t + return opt + } +} + +// WithOpTimeout sets a timeout for individual operation. It's supposed to be used in testing environment only. +func WithOpTimeout(t time.Duration) OptionFunc { + return func(opt ManagerOptions) ManagerOptions { + opt.opTimeout = t + return opt + } +} + +// WithEntryRecorder configures ManagerOptions to store Raft log entries, useful for testing scenarios. It should +// not be enabled outside of testing environments. +func WithEntryRecorder(recorder *EntryRecorder) OptionFunc { + return func(opt ManagerOptions) ManagerOptions { + opt.entryRecorder = recorder + return opt + } +} + +// WithRecordTransport configures ManagerOptions to store Raft transport messages, useful for testing scenarios. It +// should not be enabled outside of testing environments. +func WithRecordTransport() OptionFunc { + return func(opt ManagerOptions) ManagerOptions { + opt.recordTransport = true + return opt + } +} + +// RaftManagerFactory defines a function type that creates a new Manager instance for a particular parittion. +type RaftManagerFactory func(storage.PartitionID, string, keyvalue.Transactioner, log.Logger) (*Manager, error) + +// Manager orchestrates the Raft consensus protocol, managing configuration, +// state synchronization, and communication between distributed nodes. +type Manager struct { + sync.Mutex + ctx context.Context + cancel context.CancelCauseFunc + + // Configurations. + ptnID storage.PartitionID // Unique identifier for the managed partition + storage string // Name of the storage this partition belong to. + node raft.Node // etcd/raft node representation. + raftCfg config.Raft // etcd/raft configurations. + options ManagerOptions // Additional manager options + logger log.Logger // Logger for internal activities. + db keyvalue.Transactioner // Key-value DB to resume Raft activities after restart. + wal storage.WriteAheadLog // Write-ahead logging manager. + state *persistentState // Persistent storage for Raft log entries and states. + registry *Registry // Event registry, used to track events emitted by the manager. + transport Transport // Manage data transportation between nodes. + leadership *Leadership // Store up-to-date leadership information. + syncer safe.Syncer + wg sync.WaitGroup + ready *ready + started bool + + // EntryRecorder stores the list of internal Raft log entries such as config changes, empty verification + // entrires, etc. They are used in testing environments. + EntryRecorder *EntryRecorder +} + +// applyOptions creates a set of default manager options and then apply option setting functions. It also validates the +// resulting options. +func applyOptions(raftCfg config.Raft, opts []OptionFunc) (ManagerOptions, error) { + baseRTT := time.Duration(raftCfg.RTTMilliseconds) * time.Millisecond + options := ManagerOptions{ + // readyTimeout is set to 5 election timeout by default. The Raft manager waits for some couple of + // initial self elections, just in case. + readyTimeout: 5 * time.Duration(raftCfg.ElectionTicks) * baseRTT, + } + for _, opt := range opts { + options = opt(options) + } + if options.readyTimeout == 0 { + return options, fmt.Errorf("readyTimeout must not be zero") + } else if options.readyTimeout < time.Duration(raftCfg.ElectionTicks)*baseRTT { + return options, fmt.Errorf("readyTimeout must not be less than election timeout") + } + return options, nil +} + +// NewManager creates an instance of Manager. +func NewManager( + partitionID storage.PartitionID, + storageName string, + raftCfg config.Raft, + db keyvalue.Transactioner, + logger log.Logger, + opts ...OptionFunc, +) (*Manager, error) { + options, err := applyOptions(raftCfg, opts) + if err != nil { + return nil, fmt.Errorf("invalid raft manager option: %w", err) + } + + logger = logger.WithFields(log.Fields{ + "component": "raft", + "raft.partition": partitionID, + "raft.authority": storageName, + }) + + // We haven't taken care of network transportation at this point. Let's use a no-op transportation to swallow + // external messages. + transport := NewNoopTransport(logger.WithField("raft.component", "transportation"), options.recordTransport) + return &Manager{ + storage: storageName, + ptnID: partitionID, + raftCfg: raftCfg, + options: options, + db: db, + logger: logger, + registry: NewRegistry(), + transport: transport, + syncer: safe.NewSyncer(), + leadership: NewLeadership(), + ready: &ready{c: make(chan struct{})}, + EntryRecorder: options.entryRecorder, + }, nil +} + +// Run starts the Raft manager, including etcd/raft Node instance and internal processing goroutine. +func (mgr *Manager) Run(ctx context.Context, wal storage.WriteAheadLog) error { + if !mgr.raftCfg.Enabled { + return fmt.Errorf("raft is not enabled") + } + + mgr.Lock() + defer mgr.Unlock() + + if mgr.started { + return fmt.Errorf("raft manager for partition %q is re-used", mgr.ptnID) + } + mgr.started = true + + mgr.ctx, mgr.cancel = context.WithCancelCause(ctx) + mgr.wal = wal + mgr.state = newPersistentState(mgr.db, wal) + + bootstrapped, err := mgr.state.Bootstrapped() + if err != nil { + return fmt.Errorf("failed to load raft initial state: %w", err) + } + + // A partition is identified globally by (authorityName, partitionID) tuple. A node (including primary) is + // identified by (authorityName, partitionID, nodeID). Instead of setting the node ID statically in the Raft + // config, Raft manager uses the LSN of config change log entry that contains node joining event. This approach + // yields some benefits: + // - No need for setting node ID statically, thus avoiding dual storage name/node ID identity system. + // - No need for a global node registration system. + // - Work better in the scenario where a node leaves and then re-join the cluster. Each joining event leads to a + // new unique node ID. + // At this stage, Gitaly supports single-node clusters only. Thus, the node ID is always equal to 1. When + // networking replication is implemented, newly joint replicas are aware of their own node ID by examining + // messages at the transportation layer. When so, a node could self-bootstrap replicas. + // Issue: https://gitlab.com/gitlab-org/gitaly/-/issues/6304 + var nodeID uint64 = 1 + + config := &raft.Config{ + ID: nodeID, + ElectionTick: int(mgr.raftCfg.ElectionTicks), + HeartbeatTick: int(mgr.raftCfg.HeartbeatTicks), + Storage: mgr.state, + MaxSizePerMsg: defaultMaxSizePerMsg, + MaxInflightMsgs: defaultMaxInflightMsgs, + Logger: &raftLogger{logger: mgr.logger.WithFields(log.Fields{"raft.component": "manager"})}, + // etcd/raft supports Proposal Forwarding. When a node receives a proposal, and it does not hold the + // primary role, it forwards the entry to the current leader. Unfortunately, it doesn't work for Gitaly. + // + // In Gitaly WAL, a transaction is verified, committed, and then applied in order. Transactions are serialized. + // The next transaction is verified based on the latest state after applying the previous one. Raft is + // involved in the process at the committing phase. The log entry is broadcasted after an entry is + // verified and admitted. In addition, a transaction depends on a snapshot. This snapshot is essentially + // the point-of-time state of a partition that a transaction reads the data from. + // + // If Proposal Forwarding is used, two nodes are allowed to accept mutator requests simultaneously and + // then commit transactions independently. Although the resulting log entries are handled by the + // primary, the latter entry is not verified against the prior one. + // + // Replica: A -> B -> C -> Start D -> Forward to Primary + // Primary: A -> B -> C -> Start E -> Commit E -> Receive D -> Commit D + // => D is not verified against E even though E commits before D. + // + // As a result, we disable proposal forwarding in favor of request forwarding/proxying instead. + // For more information: https://gitlab.com/gitlab-org/gitaly/-/issues/6465 + DisableProposalForwarding: true, + } + + if !bootstrapped { + // When the raft group for this partition bootstraps for the first time, it is typically the next step + // after that partition is created by the storage manager. At this stage, the only peer is itself. + peers := []raft.Peer{{ID: nodeID}} + mgr.node = raft.StartNode(config, peers) + } else { + // Applied index is set to the latest committed log entry index processed by the application. WAL + // doesn't wait until the application phase. As soon as a log entry is committed, it unlocks clients + // return results. Thus, the Applied index (from etcd/raft point of view) is essentially WAL's + // committedLSN. + config.Applied = uint64(mgr.wal.LastLSN()) + mgr.node = raft.RestartNode(config) + } + + go mgr.run(bootstrapped) + return nil +} + +func (mgr *Manager) run(bootstrapped bool) { + defer func() { + if p := recover(); p != nil { + err, ok := p.(error) + if !ok { + err = fmt.Errorf("raft loop panic: %v", p) + } + mgr.logger.WithError(err).Error("raft event loop panic") + mgr.cancel(err) + + } + }() + + mgr.wg.Add(1) + defer mgr.wg.Done() + + ticker := time.NewTicker(time.Duration(mgr.raftCfg.RTTMilliseconds) * time.Millisecond) + defer ticker.Stop() + + // If the Raft group for this partition has not been bootstrapped, Raft needs to wait until complete the first + // conf change event before able to handle the first log entry. After each restart, the current Raft state is + // resumed from the persistent storage. A single cluster doesn't need to wait until the next election. Thus, it + // could be marked as ready instantly. + if bootstrapped { + mgr.ready.set() + } + + // Main event processing loop. + for { + select { + case <-ticker.C: + // etcd/raft library implements an internal tick clock. The application must increase the tick + // manually. Election and timeout depends on the number of ticks. + mgr.node.Tick() + case rd := <-mgr.node.Ready(): + if err := mgr.handleReady(&rd); err != nil { + mgr.ready.set() + if errors.Is(err, context.Canceled) { + mgr.logger.WithError(err).Info("raft event loop stopped") + mgr.cancel(nil) + } else { + mgr.logger.WithError(err).Error("raft event loop failed") + mgr.cancel(err) + } + return + } + mgr.node.Advance() + case <-mgr.ctx.Done(): + return + } + } +} + +// WaitReady waits until the Raft manager is ready to service or exceeds readyTimeout. +func (mgr *Manager) WaitReady() error { + tick := time.NewTicker(mgr.options.readyTimeout) + defer tick.Stop() + + select { + case <-tick.C: + return fmt.Errorf("ready timeout exceeded") + case <-mgr.ready.c: + return nil + } +} + +// Done returns a channel for polling running status of the Raft manager. The caller is expected to call Err() to fetch +// underlying error (if any). +func (mgr *Manager) Done() <-chan struct{} { + return mgr.ctx.Done() +} + +// Err returns the running error of the Raft manager. If the returned value is nil, it means either Raft is still +// running or it exits susccessully. Hence, it should only be called after polling from Done(). +func (mgr *Manager) Err() error { + err := mgr.ctx.Err() + if err != nil && context.Cause(mgr.ctx) != nil { + err = context.Cause(mgr.ctx) + } + return err +} + +// Stop terminates all activities of Raft. +func (mgr *Manager) Stop() { + // The node has not started yet, properly it fails during initialization. + if mgr.node != nil { + mgr.node.Stop() + } + // If the context has been cancelled beforehand, the event processing loop failed with an error. Skip + // cancellation in that case. + if mgr.ctx != nil && mgr.ctx.Err() == nil { + mgr.cancel(nil) + } +} + +// Propose proposes a log entry change to the cluster. The caller is blocked until either the log entry is committed, +// timeout exceeded, or the request is rejected by the cluster. +func (mgr *Manager) Propose(ctx context.Context, logEntry *gitalypb.LogEntry, logEntryPath string) error { + mgr.wg.Add(1) + defer mgr.wg.Done() + + w := mgr.registry.Register() + defer mgr.registry.Untrack(w.ID) + + message := &gitalypb.RaftMessageV1{ + Id: uint64(w.ID), + ClusterId: mgr.raftCfg.ClusterID, + AuthorityName: mgr.storage, + PartitionId: uint64(mgr.ptnID), + LogEntry: logEntry, + LogData: &gitalypb.RaftMessageV1_Referenced{ + Referenced: &gitalypb.RaftMessageV1_ReferencedLogData{ + Path: []byte(logEntryPath), + }, + }, + } + data, err := proto.Marshal(message) + if err != nil { + return fmt.Errorf("marshaling Raft message: %w", err) + } + + // Set an optional timeout to prevent proposal processing takes forever. This option is + // more useful in testing environments. + if mgr.options.opTimeout != 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, mgr.options.opTimeout) + defer cancel() + } + + if err := mgr.node.Propose(ctx, data); err != nil { + return fmt.Errorf("proposing Raft message: %w", err) + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-mgr.ctx.Done(): + return fmt.Errorf("raft manager is closed: %w", mgr.ctx.Err()) + case <-w.C: + return w.Err + } +} + +// handleReady processes the next state signaled by etcd/raft. It includes 3 main stepsS: +// - Persist states, including SoftState, HardState, and Entries (uncommitted). +// - Send messages to other nodes via Transport. +// - Publish commit entries. Committed entries are entries acknowledged by the majority of group members. +// Those steps are noted in https://pkg.go.dev/go.etcd.io/etcd/raft/v3#section-readme. +func (mgr *Manager) handleReady(rd *raft.Ready) error { + // Save current volatile state of a Node. It is used for observability and quick determining the current + // leadership of the group. + if err := mgr.handleSoftState(rd); err != nil { + return fmt.Errorf("handling soft state: %w", err) + } + + // Save essential state to resume its activities after a restart. + if err := mgr.handleHardState(rd); err != nil { + return fmt.Errorf("handling hard state: %w", err) + } + + // Save entries log entries to disk. Those entries haven't been committed yet. They could be overridden by + // entries with the same LSN but with higher term. WAL must cleans up subsequent appended entries if the + // inserting position overlaps. + if err := mgr.saveEntries(rd); err != nil { + return fmt.Errorf("saving entries: %w", err) + } + + // Send outbound messages. From the thesis (10.2) and etcd/raft doc, this process could be pipelined so that + // entries could be sent in parallel as soon as they are persisted to the disk first. However, as WAL serializes + // transactions, only one log entry is processed at a time. Thus, that optimization is not needed. This + // assumption is subject to change in the future if the WAL can process multiple transactions simultaneously. + // Source: https://github.com/ongardie/dissertation/blob/master/stanford.pdf + // + // At this stage, auto-compaction feature is completely ignored. In theory, the application creates "snapshots" + // occasionally. A snapshot is a point-of-time persistent state of the state machine. After a snapshot is + // created, the application can remove all log entries til the point where the snapshot is created. A new + // joining node loads the latest snapshot and replays WAL entries from there. We skipped snapshotting in the + // first iteration because the current node acts as the only primary for a single-node Gitaly cluster. It hasn't + // replicated log entries anywhere else + // + // For more information: https://gitlab.com/gitlab-org/gitaly/-/issues/6463 + if err := mgr.sendMessages(rd); err != nil { + return fmt.Errorf("sending messages: %w", err) + } + + // Mark committed entries in WAL. If the Raft group has only one member, there's a good chance the rd.Entries + // and rd.CommittedEntries overlap (partially or completely). It means those entries are sent to the committed + // state without going the need for sending external messages. + if err := mgr.commitEntries(rd); err != nil { + return fmt.Errorf("processing committed entries: %w", err) + } + return nil +} + +func (mgr *Manager) saveEntries(rd *raft.Ready) error { + if len(rd.Entries) == 0 { + return nil + } + + // Discard all in-flight events having duplicated LSN but with lower term. The corresponding on-disk log entries + // will be wiped by WAL. Events without associated LSN haven't advanced to this stage yet, hence stay instact. + firstLSN := storage.LSN(rd.Entries[0].Index) + mgr.registry.UntrackSince(firstLSN) + + for i := range rd.Entries { + switch rd.Entries[i].Type { + case raftpb.EntryNormal: + lsn := storage.LSN(rd.Entries[i].Index) + + if len(rd.Entries[i].Data) == 0 { + if err := mgr.insertEmptyLogEntry(lsn); err != nil { + return fmt.Errorf("inserting empty log entry: %w", err) + } + } else { + var message gitalypb.RaftMessageV1 + if err := proto.Unmarshal(rd.Entries[i].Data, &message); err != nil { + return fmt.Errorf("unmarshalling entry type: %w", err) + } + + var logEntryPath string + switch m := message.GetLogData().(type) { + case *gitalypb.RaftMessageV1_Packed: + // Log entry data is packed and unpacked in the Transport layer. Before sending + // messages, it packages and compresses log entry directory. Likely, after + // receiving, the data is converted to identical on-disk files. The whole + // process is transparent to Raft manager. + return fmt.Errorf("packed log entry data must be unpacked in Transport layer") + case *gitalypb.RaftMessageV1_Referenced: + logEntryPath = string(m.Referenced.GetPath()) + } + + logEntry := message.GetLogEntry() + // The Raft term is carved into the log entry. LSN and term are only available at this + // stage. Thus, we could not set this value beforehand. + logEntry.Metadata = &gitalypb.LogEntry_Metadata{ + RaftTerm: rd.Entries[i].Index, + RaftType: uint64(gitalypb.RaftMessageType_NORMAL), + } + if err := mgr.wal.AppendLogEntry(mgr.ctx, lsn, logEntry, logEntryPath); err != nil { + return fmt.Errorf("appending log entry: %w", err) + } + mgr.recordEntryIfNeeded(false, lsn, logEntry) + + // Associate the LSN of the log entry to the event. + mgr.registry.AssignLSN(EventID(message.GetId()), lsn) + } + case raftpb.EntryConfChange, raftpb.EntryConfChangeV2: + if err := mgr.saveConfigChange(rd.Entries[i]); err != nil { + return fmt.Errorf("saving config change: %w", err) + } + default: + return fmt.Errorf("raft entry type not supported: %s", rd.Entries[i].Type) + } + } + return nil +} + +func (mgr *Manager) saveConfigChange(entry raftpb.Entry) error { + if entry.Type == raftpb.EntryConfChange { + var cc raftpb.ConfChange + if err := cc.Unmarshal(entry.Data); err != nil { + return fmt.Errorf("unmarshalling EntryConfChange: %w", err) + } + } else { + var cc raftpb.ConfChangeV2 + if err := cc.Unmarshal(entry.Data); err != nil { + return fmt.Errorf("unmarshalling EntryConfChangeV2: %w", err) + } + } + + ctx := mgr.ctx + if mgr.options.opTimeout != 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, mgr.options.opTimeout) + defer cancel() + } + + // Raft manager haven't supported membership management, yet. The only member of the group is the one who + // initiates the Raft group. Thus, there is no need to maintain the list of nodes inside DB. We now place an + // entry log entry, similar to empty verification message. This placement is temporary. + lsn := storage.LSN(entry.Index) + logEntry, err := mgr.wal.InsertLogEntry(ctx, lsn, nil, &gitalypb.LogEntry_Metadata{ + RaftTerm: uint64(lsn), + RaftType: uint64(gitalypb.RaftMessageType_CONFIG_CHANGE), + }) + if err != nil { + return fmt.Errorf("inserting config change log entry: %w", err) + } + mgr.recordEntryIfNeeded(true, lsn, logEntry) + return nil +} + +// insertEmptyLogEntry inserts an empty log entry at the desirable position in WAL. +func (mgr *Manager) insertEmptyLogEntry(lsn storage.LSN) error { + ctx := mgr.ctx + if mgr.options.opTimeout != 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, mgr.options.opTimeout) + defer cancel() + } + logEntry, err := mgr.wal.InsertLogEntry(ctx, lsn, nil, &gitalypb.LogEntry_Metadata{ + RaftTerm: uint64(lsn), + RaftType: uint64(gitalypb.RaftMessageType_VERIFICATION), + }) + if err != nil { + return fmt.Errorf("inserting empty verification log entry: %w", err) + } + mgr.recordEntryIfNeeded(true, lsn, logEntry) + return nil +} + +func (mgr *Manager) commitEntries(rd *raft.Ready) error { + var shouldNotify bool + for i := range rd.Entries { + switch rd.Entries[i].Type { + case raftpb.EntryNormal: + lsn := storage.LSN(rd.Entries[i].Index) + + var message gitalypb.RaftMessageV1 + if err := proto.Unmarshal(rd.Entries[i].Data, &message); err != nil { + return fmt.Errorf("unmarshalling entry type: %w", err) + } + if err := mgr.wal.CommitLogEntry(mgr.ctx, lsn); err != nil { + return fmt.Errorf("processing committed entry: %w", err) + } + + // When Raft injects internal log entries, there are two main scenarios: + // - Those entries are piggy-backed when WAL processes a transaction. + // - They are injected independently while WAL loop is idle. + // In the first scenario, after complete a transaction, WAL circles back to process entries + // since the last committed LSN. Internal log entries are then processed as normal. + // In the second scenario, Raft needs to notify WAL about new committed entries. Otherwise, they + // are not applied until the next incoming transaction. + selfSubmit := mgr.registry.Untrack(EventID(message.GetId())) + metadata := message.GetLogEntry().GetMetadata() + if metadata.GetRaftType() != uint64(gitalypb.RaftMessageType_NORMAL) || !selfSubmit { + shouldNotify = true + } + case raftpb.EntryConfChange, raftpb.EntryConfChangeV2: + if err := mgr.commitConfChange(rd.Entries[i]); err != nil { + return fmt.Errorf("processing config change: %w", err) + } + shouldNotify = true + default: + return fmt.Errorf("raft entry type not supported: %s", rd.Entries[i].Type) + } + } + if shouldNotify { + mgr.wal.NotifyNewCommittedEntry() + } + return nil +} + +func (mgr *Manager) commitConfChange(entry raftpb.Entry) error { + var cc raftpb.ConfChangeI + if entry.Type == raftpb.EntryConfChange { + var cc1 raftpb.ConfChange + if err := cc1.Unmarshal(entry.Data); err != nil { + return fmt.Errorf("unmarshalling EntryConfChange: %w", err) + } + cc = cc1 + } else { + var cc2 raftpb.ConfChangeV2 + if err := cc2.Unmarshal(entry.Data); err != nil { + return fmt.Errorf("unmarshalling EntryConfChangeV2: %w", err) + } + cc = cc2 + } + if err := mgr.wal.CommitLogEntry(mgr.ctx, storage.LSN(entry.Index)); err != nil { + return fmt.Errorf("committing log entry: %w", err) + } + confState := mgr.node.ApplyConfChange(cc) + if err := mgr.state.SaveConfState(confState); err != nil { + return fmt.Errorf("saving config state: %w", err) + } + + // Mark Raft ready for service after the first commit change entry is applied. After restart, + // we don't need this anymore. + mgr.ready.set() + return nil +} + +func (mgr *Manager) sendMessages(rd *raft.Ready) error { + return mgr.transport.Send(mgr.ctx, mgr.wal.LSNDirPath, rd.Messages) +} + +func (mgr *Manager) handleSoftState(rd *raft.Ready) error { + state := rd.SoftState + if state == nil { + return nil + } + prevLeader := mgr.leadership.GetLeaderID() + changed, duration := mgr.leadership.SetLeader(state.Lead, state.RaftState == raft.StateLeader) + + if changed { + mgr.logger.WithFields(log.Fields{ + "raft.leader_id": mgr.leadership.GetLeaderID(), + "raft.is_leader": mgr.leadership.IsLeader(), + "raft.previous_leader_id": prevLeader, + "raft.leadership_duration": duration, + }).Info("leadership updated") + } + return nil +} + +func (mgr *Manager) handleHardState(rd *raft.Ready) error { + if raft.IsEmptyHardState(rd.HardState) { + return nil + } + if err := mgr.state.SaveHardState(rd.HardState); err != nil { + return fmt.Errorf("saving hard state: %w", err) + } + return nil +} + +func (mgr *Manager) recordEntryIfNeeded(fromRaft bool, lsn storage.LSN, logEntry *gitalypb.LogEntry) { + if mgr.EntryRecorder != nil { + mgr.EntryRecorder.Record(fromRaft, lsn, logEntry) + } +} diff --git a/internal/gitaly/storage/raftmgr/persistent_state.go b/internal/gitaly/storage/raftmgr/persistent_state.go new file mode 100644 index 0000000000000000000000000000000000000000..52612224ffe507da3ff4bba57f394a5cdda9a7e2 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/persistent_state.go @@ -0,0 +1,239 @@ +package raftmgr + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v4" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" + "go.etcd.io/etcd/raft/v3" + "go.etcd.io/etcd/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +var ( + // KeyHardState stores the current state of a Node to be saved to DB before Messages are sent. + KeyHardState = []byte("hard_state") + // KeyConfState stores the current config state used when generating snapshot. + KeyConfState = []byte("conf_state") +) + +// persistentState is responsible for managing the persistence of Raft's state, log entries, and configuration state, +// using a write-ahead log (WAL) and a KV database for durability. It ensures that state is saved and retrievable across +// restarts and node failures, thus aiding in recovery and consistency within the Raft protocol. Raft states are updated +// after each batch of entry processing. Thus, we cannot use WAL as the persistent storage for that kind of data. +// This struct implements `raft.Storage` interface from etcd/raft library. +type persistentState struct { + db keyvalue.Transactioner + wal storage.WriteAheadLog +} + +func newPersistentState(db keyvalue.Transactioner, wal storage.WriteAheadLog) *persistentState { + return &persistentState{ + db: db, + wal: wal, + } +} + +// Entries implements raft.Storage. +func (ps *persistentState) Entries(lo uint64, hi uint64, maxSize uint64) ([]raftpb.Entry, error) { + firstLSN := uint64(ps.wal.FirstLSN()) + lastLSN := uint64(ps.wal.LastLSN()) + if lo < firstLSN { + return nil, raft.ErrCompacted + } + if firstLSN > lastLSN { + return nil, raft.ErrUnavailable + } + if hi > lastLSN+1 { + return nil, fmt.Errorf("reading out-of-bound entries %d > %d", hi, lastLSN) + } + + var entries []raftpb.Entry + + for lsn := lo; lsn < hi; lsn++ { + entry, err := ps.wal.ReadLogEntry(storage.LSN(lsn)) + if err != nil { + return nil, raft.ErrCompacted + } + msg, err := proto.Marshal(entry) + if err != nil { + return nil, fmt.Errorf("marshaling log entry: %w", err) + } + entries = append(entries, raftpb.Entry{ + Term: entry.GetMetadata().GetRaftTerm(), + Index: lsn, + Type: raftpb.EntryType(entry.GetMetadata().GetRaftType()), + Data: msg, + }) + } + return entries, nil +} + +// InitialState retrieves the initial Raft HardState and ConfState from persistent storage. It is used to initialize the +// Raft node with the previously saved state. +func (ps *persistentState) InitialState() (raftpb.HardState, raftpb.ConfState, error) { + hardState, err := ps.readHardState() + if err != nil { + return raftpb.HardState{}, raftpb.ConfState{}, fmt.Errorf("reading hard state: %w", err) + } + + confState, err := ps.readConfState() + if err != nil { + return raftpb.HardState{}, raftpb.ConfState{}, fmt.Errorf("reading conf state: %w", err) + } + + return hardState, confState, nil +} + +// LastIndex returns the last index of all entries currently available in the log. +// This corresponds to the last LSN in the write-ahead log. +func (ps *persistentState) LastIndex() (uint64, error) { + return uint64(ps.wal.LastLSN()), nil +} + +// FirstIndex returns the first index of all entries currently available in the log. +// This corresponds to the first LSN in the write-ahead log. +func (ps *persistentState) FirstIndex() (uint64, error) { + return uint64(ps.wal.FirstLSN()), nil +} + +// Snapshot returns the latest snapshot of the state machine. As we haven't supported autocompaction feature, this +// method always returns Unavailable error. +// For more information: https://gitlab.com/gitlab-org/gitaly/-/issues/6463 +func (ps *persistentState) Snapshot() (raftpb.Snapshot, error) { + return raftpb.Snapshot{}, raft.ErrSnapshotTemporarilyUnavailable +} + +// Term returns the term of the entry at a given index. It retrieves the term from log entry's metadata. If the +// corresponding log entry is compacted, it retrieves the term from the hard state conditionallly. +func (ps *persistentState) Term(i uint64) (uint64, error) { + firstLSN := uint64(ps.wal.FirstLSN()) + lastLSN := uint64(ps.wal.LastLSN()) + if i < firstLSN { + // If firstLSN > lastLSN, it means all log entries are committed and cleaned up from WAL. If the queried + // log entry is equal to lastLSN, we could imply its term from the hard state. + if i == lastLSN { + if hardState, err := ps.readHardState(); err == nil && hardState.Commit == i { + return hardState.Term, nil + } + } + return 0, raft.ErrCompacted + } + if i > lastLSN { + return 0, raft.ErrUnavailable + } + + entry, err := ps.wal.ReadLogEntry(storage.LSN(i)) + if err != nil { + // If the entry does not exist here, there's a good chance WAL cleaned it up after we query the first + // LSN. Similar to the above case, we could fall back to use hard state. + if i == lastLSN { + if hardState, err := ps.readHardState(); err == nil && hardState.Commit == i { + return hardState.Term, nil + } + } + return 0, fmt.Errorf("reading log entry: %w", err) + } + return entry.GetMetadata().GetRaftTerm(), nil +} + +// Bootstrapped checks whether the node has completed its initial bootstrap process +// by verifying the existence of a saved hard state. +func (ps *persistentState) Bootstrapped() (bool, error) { + var hardState gitalypb.RaftHardStateV1 + if err := ps.readKey(KeyHardState, &hardState); err != nil { + if errors.Is(err, badger.ErrKeyNotFound) { + return false, nil + } + return false, err + } + return true, nil +} + +// SaveConfState persists the current Raft configuration state to disk, ensuring that configuration changes are durable. +func (ps *persistentState) SaveHardState(hardState raftpb.HardState) error { + return ps.setKey(KeyHardState, &gitalypb.RaftHardStateV1{ + Term: hardState.Term, + Vote: hardState.Vote, + Commit: hardState.Commit, + }) +} + +func (ps *persistentState) readHardState() (raftpb.HardState, error) { + var hardState gitalypb.RaftHardStateV1 + if err := ps.readKey(KeyHardState, &hardState); err != nil { + if errors.Is(err, badger.ErrKeyNotFound) { + return raftpb.HardState{}, nil + } + return raftpb.HardState{}, err + } + return raftpb.HardState{ + Term: hardState.GetTerm(), + Vote: hardState.GetVote(), + Commit: hardState.GetCommit(), + }, nil +} + +// SaveConfState persists node's current conf state. It is used when generating snapshot. +func (ps *persistentState) SaveConfState(confState *raftpb.ConfState) error { + return ps.setKey(KeyConfState, &gitalypb.RaftConfStateV1{ + Voters: confState.Voters, + Learners: confState.Learners, + VotersOutgoing: confState.VotersOutgoing, + LearnersNext: confState.LearnersNext, + AutoLeave: confState.AutoLeave, + }) +} + +func (ps *persistentState) readConfState() (raftpb.ConfState, error) { + var confState gitalypb.RaftConfStateV1 + if err := ps.readKey(KeyConfState, &confState); err != nil { + if errors.Is(err, badger.ErrKeyNotFound) { + return raftpb.ConfState{}, nil + } + return raftpb.ConfState{}, err + } + return raftpb.ConfState{ + Voters: confState.GetVoters(), + Learners: confState.GetLearners(), + VotersOutgoing: confState.GetVotersOutgoing(), + LearnersNext: confState.GetLearnersNext(), + AutoLeave: confState.GetAutoLeave(), + }, nil +} + +// setKey marshals and stores a given protocol buffer message into the database under the given key. +func (ps *persistentState) setKey(key []byte, value proto.Message) error { + marshaledValue, err := proto.Marshal(value) + if err != nil { + return fmt.Errorf("marshal value: %w", err) + } + + writeBatch := ps.db.NewWriteBatch() + defer writeBatch.Cancel() + + if err := writeBatch.Set(key, marshaledValue); err != nil { + return fmt.Errorf("set: %w", err) + } + + return writeBatch.Flush() +} + +// readKey reads a key from the database and unmarshals its value in to the destination protocol +// buffer message. +func (ps *persistentState) readKey(key []byte, destination proto.Message) error { + return ps.db.View(func(txn keyvalue.ReadWriter) error { + item, err := txn.Get(key) + if err != nil { + return fmt.Errorf("get: %w", err) + } + + return item.Value(func(value []byte) error { return proto.Unmarshal(value, destination) }) + }) +} + +// Compile-time type check. +var _ = (raft.Storage)(&persistentState{}) diff --git a/internal/gitaly/storage/raftmgr/registry.go b/internal/gitaly/storage/raftmgr/registry.go new file mode 100644 index 0000000000000000000000000000000000000000..468a95dc70930b2691e0574d5fbc4a14a2874082 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/registry.go @@ -0,0 +1,103 @@ +package raftmgr + +import ( + "fmt" + "sync" + + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" +) + +// ErrObsoleted is returned when an event associated with a LSN is shadowed by another one with higher term. That event +// must be unlocked and removed from the registry. +var ErrObsoleted = fmt.Errorf("obsoleted event, shadowed by a log entry with higher term") + +// EventID uniquely identifies an event in the registry. +type EventID uint64 + +// Waiter holds the information required to wait for an event to be committed. +type Waiter struct { + ID EventID + LSN storage.LSN + C chan struct{} + Err error +} + +// Registry manages events and their associated waiters, enabling the registration +// and removal of waiters upon event commitment. +type Registry struct { + mu sync.Mutex + nextEventID EventID + waiters map[EventID]*Waiter +} + +// NewRegistry initializes and returns a new instance of Registry. +func NewRegistry() *Registry { + return &Registry{ + waiters: make(map[EventID]*Waiter), + } +} + +// Register creates a new Waiter for an upcoming event and returns it. +// It must be called whenever an event is proposed, with the event ID embedded +// in the corresponding Raft message. +func (r *Registry) Register() *Waiter { + r.mu.Lock() + defer r.mu.Unlock() + + r.nextEventID++ + waiter := &Waiter{ + ID: r.nextEventID, + C: make(chan struct{}), + } + r.waiters[r.nextEventID] = waiter + + return waiter +} + +// AssignLSN assigns LSN to an event. LSN of an event is used to unlock obsolete proposals if Raft detects duplicated +// LSNs but with higher term. +func (r *Registry) AssignLSN(id EventID, lsn storage.LSN) { + r.mu.Lock() + defer r.mu.Unlock() + + waiter, exists := r.waiters[id] + if !exists { + return + } + waiter.LSN = lsn +} + +// UntrackSince untracks all events having LSNs greater than or equal to the input LSN. +func (r *Registry) UntrackSince(lsn storage.LSN) { + r.mu.Lock() + defer r.mu.Unlock() + + var toRemove []EventID + for id, w := range r.waiters { + if w.LSN >= lsn { + toRemove = append(toRemove, id) + } + } + for _, id := range toRemove { + close(r.waiters[id].C) + r.waiters[id].Err = ErrObsoleted + delete(r.waiters, id) + } +} + +// Untrack closes the channel associated with a given EventID and removes the waiter from the registry once the event is +// committed. This function returns if the registry is still tracking the event. +func (r *Registry) Untrack(id EventID) bool { + r.mu.Lock() + defer r.mu.Unlock() + + waiter, exists := r.waiters[id] + if !exists { + return false + } + + // Close the channel to notify any goroutines waiting on this event. + close(waiter.C) + delete(r.waiters, id) + return true +} diff --git a/internal/gitaly/storage/raftmgr/registry_test.go b/internal/gitaly/storage/raftmgr/registry_test.go new file mode 100644 index 0000000000000000000000000000000000000000..beca124804aacd2a424e9d586432f246b2095b3a --- /dev/null +++ b/internal/gitaly/storage/raftmgr/registry_test.go @@ -0,0 +1,181 @@ +package raftmgr + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" +) + +func TestRegistry(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + action func(*Registry) []*Waiter + expectedEvents []EventID + }{ + { + name: "Register and Remove single event", + action: func(r *Registry) []*Waiter { + waiter := r.Register() + require.True(t, r.Untrack(waiter.ID)) + return []*Waiter{waiter} + }, + expectedEvents: []EventID{1}, + }, + { + name: "Register multiple events and remove in order", + action: func(r *Registry) []*Waiter { + w1 := r.Register() + w2 := r.Register() + require.True(t, r.Untrack(w1.ID)) + require.True(t, r.Untrack(w2.ID)) + return []*Waiter{w1, w2} + }, + expectedEvents: []EventID{1, 2}, + }, + { + name: "Register multiple events and remove out of order", + action: func(r *Registry) []*Waiter { + w1 := r.Register() + w2 := r.Register() + require.True(t, r.Untrack(w2.ID)) // Removing the second one first + require.True(t, r.Untrack(w1.ID)) // Then the first one + return []*Waiter{w1, w2} + }, + expectedEvents: []EventID{1, 2}, + }, + { + name: "Remove non-existent event", + action: func(r *Registry) []*Waiter { + require.False(t, r.Untrack(1234)) + + c := make(chan struct{}) + close(c) + return []*Waiter{{ID: 99999, C: c}} // Non-existent event + }, + expectedEvents: []EventID{99999}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + registry := NewRegistry() + waiters := tc.action(registry) + + for _, waiter := range waiters { + select { + case <-waiter.C: + // Success, channel was closed + case <-time.After(10 * time.Second): + t.Fatalf("Expected channel for event %d to be closed", waiter.ID) + } + require.Contains(t, tc.expectedEvents, waiter.ID) + } + }) + } +} + +func TestRegistry_AssignLSN(t *testing.T) { + t.Parallel() + registry := NewRegistry() + + waiter1 := registry.Register() + waiter2 := registry.Register() + + // Assign LSN to the registered waiters + registry.AssignLSN(waiter1.ID, 10) + registry.AssignLSN(waiter2.ID, 20) + registry.AssignLSN(999, 99) + + // Verify that LSNs are assigned correctly + require.Equal(t, storage.LSN(10), waiter1.LSN) + require.Equal(t, storage.LSN(20), waiter2.LSN) +} + +func TestRegistry_UntrackSince(t *testing.T) { + t.Parallel() + registry := NewRegistry() + + waiter1 := registry.Register() + waiter2 := registry.Register() + waiter3 := registry.Register() + + // Assign LSNs + registry.AssignLSN(waiter1.ID, 10) + registry.AssignLSN(waiter2.ID, 11) + registry.AssignLSN(waiter3.ID, 12) + + // Call UntrackSince with threshold LSN + registry.UntrackSince(11) + + // Waiters with LSN > 10 should be obsoleted + select { + case <-waiter1.C: + t.Fatalf("Expected channel for event %d to remain open", waiter1.ID) + default: + // Waiter1 should not be closed + } + select { + case <-waiter2.C: + // Expected behavior, channel closed + require.Equal(t, ErrObsoleted, waiter2.Err) + default: + t.Fatalf("Expected channel for event %d to be closed", waiter2.ID) + } + + select { + case <-waiter3.C: + // Expected behavior, channel closed + require.Equal(t, ErrObsoleted, waiter3.Err) + default: + t.Fatalf("Expected channel for event %d to be closed", waiter3.ID) + } + + // Waiter1 should still be tracked + require.True(t, registry.Untrack(waiter1.ID)) + // Waiter2 and Waiter3 should not be tracked anymore + require.False(t, registry.Untrack(waiter2.ID)) + require.False(t, registry.Untrack(waiter3.ID)) +} + +func TestRegistry_ConcurrentAccess(t *testing.T) { + t.Parallel() + const numEvents = 100 + + registry := NewRegistry() + waiters := make(chan EventID, numEvents) + var producerWg, consumerWg sync.WaitGroup + + // Register events concurrently + for i := 0; i < 10; i++ { + producerWg.Add(1) + go func() { + defer producerWg.Done() + for i := 0; i < numEvents; i++ { + waiters <- registry.Register().ID + } + }() + } + + // Untrack events concurrently + for i := 0; i < 10; i++ { + consumerWg.Add(1) + go func() { + defer consumerWg.Done() + for i := range waiters { + assert.True(t, registry.Untrack(i)) + } + }() + } + + producerWg.Wait() + close(waiters) + consumerWg.Wait() + + require.Emptyf(t, registry.waiters, "waiter list must be empty") +} diff --git a/internal/gitaly/storage/raftmgr/testhelper_test.go b/internal/gitaly/storage/raftmgr/testhelper_test.go new file mode 100644 index 0000000000000000000000000000000000000000..c80ed0e3813c41269f68102742adce1594ed4520 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/testhelper_test.go @@ -0,0 +1,11 @@ +package raftmgr + +import ( + "testing" + + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" +) + +func TestMain(m *testing.M) { + testhelper.Run(m) +} diff --git a/internal/gitaly/storage/raftmgr/transport.go b/internal/gitaly/storage/raftmgr/transport.go new file mode 100644 index 0000000000000000000000000000000000000000..6f4f1f7b6b1ae0f6d890597ee61e19f830e84a14 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/transport.go @@ -0,0 +1,122 @@ +package raftmgr + +import ( + "bufio" + "bytes" + "context" + "fmt" + + "gitlab.com/gitlab-org/gitaly/v16/internal/archive" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" + "go.etcd.io/etcd/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// Transport defines the interface for sending Raft protocol messages. +type Transport interface { + // Send dispatches a batch of Raft messages. It returns an error if the sending fails. + Send(context.Context, func(storage.LSN) string, []raftpb.Message) error + + // GetRecordedMessages retrieves all recorded messages if recording is enabled. + // This is typically used in a testing environment to verify message transmission. + GetRecordedMessages() []raftpb.Message +} + +// NoopTransport is a transport implementation that logs messages and optionally records them. +// It is useful in testing environments where message delivery is non-functional but needs to be observed. +type NoopTransport struct { + logger log.Logger // Logger for outputting message information + recordTransport bool // Flag indicating whether message recording is enabled + recordedMessages []*raftpb.Message // Slice to store recorded messages +} + +// NewNoopTransport constructs a new NoopTransport instance. +// The logger is used for logging message information, and the recordTransport flag +// determines whether messages should be recorded. +func NewNoopTransport(logger log.Logger, recordTransport bool) Transport { + return &NoopTransport{ + logger: logger, + recordTransport: recordTransport, + } +} + +// Send logs each message being sent and records it if recording is enabled. +func (t *NoopTransport) Send(ctx context.Context, pathForLSN func(storage.LSN) string, messages []raftpb.Message) error { + for i := range messages { + for j := range messages[i].Entries { + if messages[i].Entries[j].Type == raftpb.EntryNormal { + var msg gitalypb.RaftMessageV1 + + if err := proto.Unmarshal(messages[i].Entries[j].Data, &msg); err != nil { + return fmt.Errorf("unmarshalling entry type: %w", err) + } + + // This is a very native implementation. Noop Transport is only used for testing + // purposes. All external messages are swallowed and stored in a recorder. It packages + // the whole log entry directory as a tar ball using an existing backup utility. The + // resulting binary data is stored inside a subfield of the message for examining + // purpose. A real implementation of Transaction will likely use an optimized method + // (such as sidechannel) to deliver the data. It does not necessarily store the data in + // the memory. + switch msg.GetLogData().(type) { + case *gitalypb.RaftMessageV1_Packed: + continue + case *gitalypb.RaftMessageV1_Referenced: + lsn := storage.LSN(messages[i].Entries[j].Index) + path := pathForLSN(lsn) + if err := t.packLogData(ctx, lsn, &msg, path); err != nil { + return fmt.Errorf("packing log data: %w", err) + } + } + data, err := proto.Marshal(&msg) + if err != nil { + return fmt.Errorf("marshaling Raft entry: %w", err) + } + messages[i].Entries[j].Data = data + } + } + + t.logger.WithFields(log.Fields{ + "raft.type": messages[i].Type, + "raft.to": messages[i].To, + "raft.from": messages[i].From, + "raft.term": messages[i].Term, + "raft.num_entries": len(messages[i].Entries), + }).Info("sending message") + + // Record messages if recording is enabled. + if t.recordTransport { + t.recordedMessages = append(t.recordedMessages, &messages[i]) + } + } + return nil +} + +func (t *NoopTransport) packLogData(ctx context.Context, lsn storage.LSN, message *gitalypb.RaftMessageV1, logEntryPath string) error { + var logData bytes.Buffer + writer := bufio.NewWriter(&logData) + if err := archive.WriteTarball(ctx, t.logger.WithFields(log.Fields{ + "raft.component": "WAL archiver", + "raft.log_entry_lsn": lsn, + "raft.log_entry_path": logEntryPath, + }), writer, logEntryPath, "."); err != nil { + return fmt.Errorf("archiving WAL log entry") + } + message.LogData = &gitalypb.RaftMessageV1_Packed{ + Packed: &gitalypb.RaftMessageV1_PackedLogData{ + Data: logData.Bytes(), + }, + } + return nil +} + +// GetRecordedMessages returns the list of recorded messages. +func (t *NoopTransport) GetRecordedMessages() []raftpb.Message { + messages := make([]raftpb.Message, 0, len(t.recordedMessages)) + for _, m := range t.recordedMessages { + messages = append(messages, *m) + } + return messages +} diff --git a/internal/gitaly/storage/raftmgr/transport_test.go b/internal/gitaly/storage/raftmgr/transport_test.go new file mode 100644 index 0000000000000000000000000000000000000000..c15bbdd8e807ed374d28025c1eb4a221e9a89372 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/transport_test.go @@ -0,0 +1,238 @@ +package raftmgr + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" + "go.etcd.io/etcd/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +func TestNoopTransport_Send(t *testing.T) { + t.Parallel() + + mustMarshalProto := func(msg proto.Message) []byte { + data, err := proto.Marshal(msg) + if err != nil { + panic(fmt.Sprintf("failed to marshal proto: %v", err)) + } + return data + } + + tests := []struct { + name string + setupFunc func(tempDir string) ([]*gitalypb.LogEntry, []raftpb.Message, map[string]string) + expectedOutput map[string]string + }{ + { + name: "No messages", + setupFunc: func(tempDir string) ([]*gitalypb.LogEntry, []raftpb.Message, map[string]string) { + return nil, []raftpb.Message{}, nil + }, + }, + { + name: "Empty Entries", + setupFunc: func(tempDir string) ([]*gitalypb.LogEntry, []raftpb.Message, map[string]string) { + return nil, []raftpb.Message{ + { + Type: raftpb.MsgApp, + From: 2, + To: 1, + Term: 1, + Entries: []raftpb.Entry{}, // Empty Entries + }, + }, nil + }, + }, + { + name: "Messages with already packed data", + setupFunc: func(tempDir string) ([]*gitalypb.LogEntry, []raftpb.Message, map[string]string) { + logEntry := &gitalypb.LogEntry{ + RelativePath: "relative-path", + Operations: []*gitalypb.LogEntry_Operation{ + { + Operation: &gitalypb.LogEntry_Operation_CreateHardLink_{ + CreateHardLink: &gitalypb.LogEntry_Operation_CreateHardLink{ + SourcePath: []byte("source"), + DestinationPath: []byte("destination"), + }, + }, + }, + }, + } + initialMessage := gitalypb.RaftMessageV1{ + Id: 1, + ClusterId: "44c58f50-0a8b-4849-bf8b-d5a56198ea7c", + AuthorityName: "sample-storage", + PartitionId: 1, + LogEntry: logEntry, + LogData: &gitalypb.RaftMessageV1_Packed{Packed: &gitalypb.RaftMessageV1_PackedLogData{Data: []byte("already packed data")}}, + } + messages := []raftpb.Message{ + { + Type: raftpb.MsgApp, + From: 2, + To: 1, + Term: 1, + Index: 1, + Entries: []raftpb.Entry{{Index: uint64(1), Type: raftpb.EntryNormal, Data: mustMarshalProto(&initialMessage)}}, + }, + } + return []*gitalypb.LogEntry{logEntry}, messages, nil + }, + }, + { + name: "Messages with referenced data", + setupFunc: func(tempDir string) ([]*gitalypb.LogEntry, []raftpb.Message, map[string]string) { + // Simulate a log entry dir with files + fileContents := map[string]string{ + "1": "file1 content", + "2": "file2 content", + "3": "file3 content", + } + for name, content := range fileContents { + require.NoError(t, os.WriteFile(filepath.Join(tempDir, name), []byte(content), 0o644)) + } + + // Construct message with ReferencedLogData + logEntry := &gitalypb.LogEntry{ + RelativePath: "relative-path", + Operations: []*gitalypb.LogEntry_Operation{ + { + Operation: &gitalypb.LogEntry_Operation_CreateHardLink_{ + CreateHardLink: &gitalypb.LogEntry_Operation_CreateHardLink{ + SourcePath: []byte("source"), + DestinationPath: []byte("destination"), + }, + }, + }, + }, + } + initialMessage := gitalypb.RaftMessageV1{ + Id: 1, + ClusterId: "44c58f50-0a8b-4849-bf8b-d5a56198ea7c", + AuthorityName: "sample-storage", + PartitionId: 1, + LogEntry: logEntry, + LogData: &gitalypb.RaftMessageV1_Referenced{Referenced: &gitalypb.RaftMessageV1_ReferencedLogData{}}, + } + + messages := []raftpb.Message{ + { + Type: raftpb.MsgApp, + From: 2, + To: 1, + Term: 1, + Index: 1, + Entries: []raftpb.Entry{ + {Index: uint64(1), Type: raftpb.EntryNormal, Data: mustMarshalProto(&initialMessage)}, + }, + }, + } + return []*gitalypb.LogEntry{logEntry}, messages, fileContents + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Create a temporary directory + tempDir := testhelper.TempDir(t) + defer func() { require.NoError(t, os.RemoveAll(tempDir)) }() + + // Execute setup function to prepare messages and any necessary file contents + entries, messages, expectedContents := tc.setupFunc(tempDir) + + // Setup logger and transport + logger := testhelper.SharedLogger(t) + transport := NewNoopTransport(logger, true) + + // Execute the Send operation + require.NoError(t, transport.Send(context.Background(), func(storage.LSN) string { return tempDir }, messages)) + + // Fetch recorded messages for verification + recordedMessages := transport.GetRecordedMessages() + + require.Len(t, recordedMessages, len(messages)) + + // Messages must be sent in order. + for i := range messages { + require.Equal(t, messages[i].Type, recordedMessages[i].Type) + require.Equal(t, messages[i].From, recordedMessages[i].From) + require.Equal(t, messages[i].To, recordedMessages[i].To) + require.Equal(t, messages[i].Term, recordedMessages[i].Term) + require.Equal(t, messages[i].Index, recordedMessages[i].Index) + + if len(messages[i].Entries) == 0 { + require.Empty(t, recordedMessages[i].Entries) + } else { + var resultMessage gitalypb.RaftMessageV1 + require.NoError(t, proto.Unmarshal(recordedMessages[i].Entries[0].Data, &resultMessage)) + + testhelper.ProtoEqual(t, entries[i], resultMessage.GetLogEntry()) + + packedData, ok := resultMessage.GetLogData().(*gitalypb.RaftMessageV1_Packed) + require.True(t, ok) + + tarballData := packedData.Packed.GetData() + require.NotEmpty(t, tarballData) + + // Optionally verify packed data if expected + if expectedContents != nil { + var resultMessage gitalypb.RaftMessageV1 + require.NoError(t, proto.Unmarshal(recordedMessages[0].Entries[0].Data, &resultMessage)) + packedData, ok := resultMessage.GetLogData().(*gitalypb.RaftMessageV1_Packed) + require.True(t, ok) + // Verify tarball content matches expectations + verifyPackedData(t, packedData.Packed.GetData(), expectedContents) + } + } + } + }) + } +} + +// verifyPackedData checks if the tarballData contains files with content as expected +func verifyPackedData(t *testing.T, tarballData []byte, expectedContents map[string]string) { + // Create a tar reader + buf := bytes.NewReader(tarballData) + tarReader := tar.NewReader(buf) + + extractedContents := make(map[string]string) + + for { + // Read the next header + header, err := tarReader.Next() + if errors.Is(err, io.EOF) { + // End of tar archive + break + } + require.NoError(t, err) + + // We skip directories since they don't contain data + if header.Typeflag == tar.TypeDir { + continue + } + + // Read file content + var extractedData bytes.Buffer + _, err = io.Copy(&extractedData, tarReader) + require.NoError(t, err) + + // Store extracted content in the map + extractedContents[header.Name] = extractedData.String() + } + + require.Equal(t, expectedContents, extractedContents) +} diff --git a/internal/gitaly/storage/storage.go b/internal/gitaly/storage/storage.go index 29cf42cd1fcafb17318f51470f7605fd79b69a64..92404d5df859cb8a7decc08196de3df79119b821 100644 --- a/internal/gitaly/storage/storage.go +++ b/internal/gitaly/storage/storage.go @@ -137,6 +137,8 @@ type BeginOptions struct { // This is a temporary workaround for some RPCs that do not work well with shared // read-only snapshots yet. ForceExclusiveSnapshot bool + // WithoutRaftReady lets the transaction skip Raft readiness waiting. + WithoutRaftReady bool } // Partition is responsible for a single partition of data. diff --git a/internal/gitaly/storage/storagemgr/partition/factory.go b/internal/gitaly/storage/storagemgr/partition/factory.go index e2fabf936fd5cb62dac94ea20439e5ca79648391..d72f306796e29f2048ca7a61fc1bed1b1717f24f 100644 --- a/internal/gitaly/storage/storagemgr/partition/factory.go +++ b/internal/gitaly/storage/storagemgr/partition/factory.go @@ -8,6 +8,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr" "gitlab.com/gitlab-org/gitaly/v16/internal/log" ) @@ -27,10 +28,11 @@ type LogConsumer interface { // Factory is factory type that can create new partitions. type Factory struct { - cmdFactory gitcmd.CommandFactory - repoFactory localrepo.Factory - metrics Metrics - logConsumer LogConsumer + cmdFactory gitcmd.CommandFactory + repoFactory localrepo.Factory + metrics Metrics + logConsumer LogConsumer + raftManagerFactory raftmgr.RaftManagerFactory } // New returns a new Partition instance. @@ -55,6 +57,14 @@ func (f Factory) New( panic(fmt.Errorf("building a partition for a non-existent storage: %q", storageName)) } + var raftManager *raftmgr.Manager + if f.raftManagerFactory != nil { + raftManager, err = f.raftManagerFactory(partitionID, storageName, db, logger) + if err != nil { + panic(fmt.Errorf("creating raft manager: %w", err)) + } + } + return NewTransactionManager( partitionID, logger, @@ -67,6 +77,7 @@ func (f Factory) New( repoFactory, f.metrics.Scope(storageName), f.logConsumer, + raftManager, ) } @@ -76,11 +87,13 @@ func NewFactory( repoFactory localrepo.Factory, metrics Metrics, logConsumer LogConsumer, + raftManagerFactory raftmgr.RaftManagerFactory, ) Factory { return Factory{ - cmdFactory: cmdFactory, - repoFactory: repoFactory, - metrics: metrics, - logConsumer: logConsumer, + cmdFactory: cmdFactory, + repoFactory: repoFactory, + metrics: metrics, + logConsumer: logConsumer, + raftManagerFactory: raftManagerFactory, } } diff --git a/internal/gitaly/storage/storagemgr/partition/testhelper_test.go b/internal/gitaly/storage/storagemgr/partition/testhelper_test.go index c7f5b00464ddb7053962cf6040b3fc21d39b8655..f38c6ab1668e3b93cae35b7937e9b942f0682505 100644 --- a/internal/gitaly/storage/storagemgr/partition/testhelper_test.go +++ b/internal/gitaly/storage/storagemgr/partition/testhelper_test.go @@ -9,11 +9,15 @@ import ( "os" "path/filepath" "reflect" + "regexp" "sort" + "strconv" "strings" "sync" "testing" + "time" + "github.com/google/uuid" "github.com/prometheus/client_golang/prometheus" io_prometheus_client "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" @@ -34,10 +38,12 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/mode" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/snapshot" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" ) func TestMain(m *testing.M) { @@ -606,7 +612,7 @@ func RequireRepositories(tb testing.TB, ctx context.Context, cfg config.Cfg, sto // relative path of the repository that failed. If the call failed the test, // print out the relative path to ease troubleshooting. if tb.Failed() { - require.Failf(tb, "unexpected repository state", "relative path: %q", relativePath) + tb.Log("unexpected repository state", "relative path: %q", relativePath) } }() @@ -656,13 +662,20 @@ func RequireDatabase(tb testing.TB, ctx context.Context, database keyvalue.Trans // Unmarshal the actual value to the same type as the expected value. actualValue := reflect.New(reflect.TypeOf(expectedValue).Elem()).Interface().(proto.Message) require.NoError(tb, proto.Unmarshal(value, actualValue)) + + if _, isAny := expectedValue.(*anypb.Any); isAny { + // Verify if the database contains the key, but the content does not matter. + actualState[string(key)] = expectedValue + continue + } + actualState[string(key)] = actualValue } return nil })) - require.Empty(tb, unexpectedKeys, "database contains unexpected keys") + require.Emptyf(tb, unexpectedKeys, "database contains unexpected keys") testhelper.ProtoEqual(tb, expectedState, actualState) } @@ -705,6 +718,8 @@ type testTransactionHooks struct { BeforeApplyLogEntry hookFunc // BeforeAppendLogEntry is called before a log entry is appended to the log. BeforeAppendLogEntry hookFunc + // BeforeAppendLogEntry is called before a log entry is marked as committed. + BeforeCommitLogEntry hookFunc // AfterDeleteLogEntry is called after a log entry is deleted. AfterDeleteLogEntry hookFunc // BeforeReadAppliedLSN is invoked before the applied LSN is read. @@ -945,6 +960,8 @@ type RepositoryAssertion struct { type StateAssertion struct { // Database is the expected state of the database. Database DatabaseState + // NotOffsetDatabaseInRaft indicates if the LSN in the database should not be shifted. + NotOffsetDatabaseInRaft bool // Directory is the expected state of the manager's state directory in the repository. Directory testhelper.DirectoryState // Repositories is the expected state of the repositories in the storage. The key is @@ -1064,11 +1081,35 @@ func runTransactionTest(t *testing.T, ctx context.Context, tc transactionTestCas ).Scope(storageName) } + // Cluster ID and recorder is kept stable in each test run. + clusterID := uuid.New().String() + var raftEntryRecorder *raftmgr.EntryRecorder + var raftManager *raftmgr.Manager + + createRaftManager := func() *raftmgr.Manager { + raftManager, err := raftmgr.NewManager( + setup.PartitionID, + setup.Config.Storages[0].Name, + config.DefaultRaftConfig(clusterID), + database, + logger, + raftmgr.WithEntryRecorder(raftEntryRecorder), + raftmgr.WithRecordTransport(), + raftmgr.WithOpTimeout(1*time.Minute), + ) + require.NoError(t, err) + return raftManager + } + if testhelper.IsRaftEnabled() { + raftEntryRecorder = raftmgr.NewEntryRecorder() + raftManager = createRaftManager() + } + var ( // managerRunning tracks whether the manager is running or closed. managerRunning bool // transactionManager is the current TransactionManager instance. - transactionManager = NewTransactionManager(setup.PartitionID, logger, database, storageName, storagePath, stateDir, stagingDir, setup.CommandFactory, storageScopedFactory, newMetrics(), setup.Consumer) + transactionManager = NewTransactionManager(setup.PartitionID, logger, database, storageName, storagePath, stateDir, stagingDir, setup.CommandFactory, storageScopedFactory, newMetrics(), setup.Consumer, raftManager) // managerErr is used for synchronizing manager closing and returning // the error from Run. managerErr chan error @@ -1115,7 +1156,12 @@ func runTransactionTest(t *testing.T, ctx context.Context, tc transactionTestCas require.NoError(t, os.RemoveAll(stagingDir)) require.NoError(t, os.Mkdir(stagingDir, mode.Directory)) - transactionManager = NewTransactionManager(setup.PartitionID, logger, database, setup.Config.Storages[0].Name, storagePath, stateDir, stagingDir, setup.CommandFactory, storageScopedFactory, newMetrics(), setup.Consumer) + if testhelper.IsRaftEnabled() { + raftManager = createRaftManager() + } else { + raftManager = nil + } + transactionManager = NewTransactionManager(setup.PartitionID, logger, database, setup.Config.Storages[0].Name, storagePath, stateDir, stagingDir, setup.CommandFactory, storageScopedFactory, newMetrics(), setup.Consumer, raftManager) installHooks(transactionManager, &inflightTransactions, step.Hooks) go func() { @@ -1158,7 +1204,11 @@ func runTransactionTest(t *testing.T, ctx context.Context, tc transactionTestCas require.NoError(t, err) tx := transaction.(*Transaction) - require.Equalf(t, step.ExpectedSnapshotLSN, tx.SnapshotLSN(), "mismatched ExpectedSnapshotLSN") + expectedSnapshotLSN := step.ExpectedSnapshotLSN + if testhelper.IsRaftEnabled() { + expectedSnapshotLSN = raftEntryRecorder.Offset(expectedSnapshotLSN) + } + require.Equal(t, expectedSnapshotLSN, tx.SnapshotLSN()) require.NotEmpty(t, tx.Root(), "empty Root") require.Contains(t, tx.Root(), transactionManager.snapshotsDir()) @@ -1442,7 +1492,11 @@ func runTransactionTest(t *testing.T, ctx context.Context, tc transactionTestCas transaction := openTransactions[step.TransactionID] transaction.WriteCommitGraphs(step.Config) case ConsumerAcknowledge: - transactionManager.AcknowledgeTransaction(step.LSN) + lsn := step.LSN + if testhelper.IsRaftEnabled() { + lsn = raftEntryRecorder.Offset(lsn) + } + transactionManager.AcknowledgeTransaction(lsn) case RepositoryAssertion: require.Contains(t, openTransactions, step.TransactionID, "test error: transaction's snapshot asserted before beginning it") transaction := openTransactions[step.TransactionID] @@ -1509,6 +1563,39 @@ func runTransactionTest(t *testing.T, ctx context.Context, tc transactionTestCas require.NoError(t, err) } + // Most of the time, persisted committedLSN is equal to appliedLSN except for some intentional tests. Thus, if + // appliedLSN is asserted, the test should backfill committedLSN if it's not there. + if appliedLSN, appliedExist := tc.expectedState.Database[string(keyAppliedLSN)]; appliedExist { + if _, committedExist := tc.expectedState.Database[string(keyCommittedLSN)]; !committedExist { + tc.expectedState.Database[string(keyCommittedLSN)] = appliedLSN + } + } + + // If singular Raft cluster is enabled, the Raft handler starts to insert raft log entries. We need to + // offset the expected LSN to account for LSN shifting. + if testhelper.IsRaftEnabled() && raftEntryRecorder.Len() > 0 { + if !tc.expectedState.NotOffsetDatabaseInRaft { + appliedLSN, exist := tc.expectedState.Database[string(keyAppliedLSN)] + if exist { + // If expected applied LSN is present, offset expected LSN. + appliedLSN := appliedLSN.(*gitalypb.LSN) + tc.expectedState.Database[string(keyAppliedLSN)] = raftEntryRecorder.Offset(storage.LSN(appliedLSN.GetValue())).ToProto() + + committedLSN := tc.expectedState.Database[string(keyCommittedLSN)].(*gitalypb.LSN) + tc.expectedState.Database[string(keyCommittedLSN)] = raftEntryRecorder.Offset(storage.LSN(committedLSN.GetValue())).ToProto() + } else { + // Otherwise, the test expects no applied log entry in cases such as invalid transactions. + // Regardless, raft log entries are applied successfully. + if tc.expectedState.Database == nil { + tc.expectedState.Database = DatabaseState{} + } + tc.expectedState.Database[string(keyAppliedLSN)] = raftEntryRecorder.Latest().ToProto() + tc.expectedState.Database[string(keyCommittedLSN)] = raftEntryRecorder.Latest().ToProto() + } + } + tc.expectedState.Database[string(raftmgr.KeyHardState)] = &anypb.Any{} + tc.expectedState.Database[string(raftmgr.KeyConfState)] = &anypb.Any{} + } RequireDatabase(t, ctx, database, tc.expectedState.Database) expectedRepositories := tc.expectedState.Repositories @@ -1544,6 +1631,17 @@ func runTransactionTest(t *testing.T, ctx context.Context, tc transactionTestCas require.NoError(t, transactionManager.CloseSnapshots()) RequireRepositories(t, ctx, setup.Config, setup.Config.Storages[0].Path, storageScopedFactory.Build, expectedRepositories) + expectedConsumers := tc.expectedState.Consumers + if testhelper.IsRaftEnabled() { + expectedConsumers.HighWaterMark = raftEntryRecorder.Offset(expectedConsumers.HighWaterMark) + // If the expected manager position is equal to 0, it means the test asserts the starting position of the + // consumer or consumer errors. Otherwise, it should be strictly positive as LSN starts at 1. + if expectedConsumers.ManagerPosition != 0 { + expectedConsumers.ManagerPosition = raftEntryRecorder.Offset(expectedConsumers.ManagerPosition) + } + } + RequireConsumer(t, transactionManager.consumer, transactionManager.consumerPos, expectedConsumers) + expectedDirectory := tc.expectedState.Directory if expectedDirectory == nil { // Set the base state as the default so we don't have to repeat it in every test case but it @@ -1554,8 +1652,7 @@ func runTransactionTest(t *testing.T, ctx context.Context, tc transactionTestCas } } - RequireConsumer(t, transactionManager.consumer, transactionManager.consumerPos, tc.expectedState.Consumers) - + expectedDirectory = modifyDirectoryStateForRaft(t, expectedDirectory, transactionManager) testhelper.RequireDirectoryState(t, stateDir, "", expectedDirectory) expectedStagingDirState := testhelper.DirectoryState{ @@ -1573,6 +1670,48 @@ func runTransactionTest(t *testing.T, ctx context.Context, tc transactionTestCas testhelper.RequireDirectoryState(t, transactionManager.stagingDirectory, "", expectedStagingDirState) } +func modifyDirectoryStateForRaft(t *testing.T, expectedDirectory testhelper.DirectoryState, transactionManager *TransactionManager) testhelper.DirectoryState { + if !testhelper.IsRaftEnabled() { + return expectedDirectory + } + + raftEntryRecorder := transactionManager.raftManager.EntryRecorder + newExpectedDirectory := testhelper.DirectoryState{} + lsnRegex := regexp.MustCompile(`/wal/(\d+)\/?.*`) + for path, state := range expectedDirectory { + matches := lsnRegex.FindStringSubmatch(path) + if len(matches) == 0 { + // If no LSN found, retain the original entry + newExpectedDirectory[path] = state + continue + } + // Extract and offset the LSN + lsnStr := matches[1] + lsn, err := strconv.ParseUint(lsnStr, 10, 64) + require.NoError(t, err) + + offsetLSN := raftEntryRecorder.Offset(storage.LSN(lsn)) + offsetPath := strings.Replace(path, fmt.Sprintf("/%s", lsnStr), fmt.Sprintf("/%s", offsetLSN.String()), 1) + newExpectedDirectory[offsetPath] = state + + if entry, isEntry := state.Content.(*gitalypb.LogEntry); isEntry { + entry.Metadata = raftEntryRecorder.Metadata(offsetLSN) + } + } + + // Insert Raft-specific log entries into the new directory structure + raftEntries := raftEntryRecorder.FromRaft() + for lsn, raftEntry := range raftEntries { + if lsn >= transactionManager.oldestLSN { + newExpectedDirectory[fmt.Sprintf("/wal/%s", lsn)] = testhelper.DirectoryEntry{Mode: mode.Directory} + newExpectedDirectory[fmt.Sprintf("/wal/%s/MANIFEST", lsn)] = manifestDirectoryEntry(raftEntry) + } + } + + // Update expected directory + return newExpectedDirectory +} + func checkManagerError(t *testing.T, ctx context.Context, managerErrChannel chan error, mgr *TransactionManager) (bool, error) { t.Helper() diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager.go index f6b7e1ab420ea55eab5167c676b7aa73489b1053..6a9e6a2c131b0ebfb9caacef93a3dbde77e5733c 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager.go @@ -34,6 +34,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/gitstorage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/mode" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/snapshot" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/wal" "gitlab.com/gitlab-org/gitaly/v16/internal/log" @@ -101,6 +102,8 @@ var ( // keyAppliedLSN is the database key storing a partition's last applied log entry's LSN. keyAppliedLSN = []byte("applied_lsn") + // keyCommittedLSN is the database key storing a partition's last committed log entry's LSN. + keyCommittedLSN = []byte("committed_lsn") ) const relativePathKeyPrefix = "r/" @@ -340,6 +343,12 @@ func (mgr *TransactionManager) Begin(ctx context.Context, opts storage.BeginOpti return nil, errWritableAllRepository } + if mgr.raftManager != nil && !opts.WithoutRaftReady { + if err := mgr.raftManager.WaitReady(); err != nil { + return nil, fmt.Errorf("waiting for Raft to be ready: %w", err) + } + } + span, _ := tracing.StartSpanIfHasParent(ctx, "transaction.Begin", nil) span.SetTag("write", opts.Write) span.SetTag("relativePath", relativePath) @@ -350,7 +359,7 @@ func (mgr *TransactionManager) Begin(ctx context.Context, opts storage.BeginOpti txn := &Transaction{ write: opts.Write, commit: mgr.commit, - snapshotLSN: mgr.appendedLSN, + snapshotLSN: mgr.committedLSN, finished: make(chan struct{}), relativePath: relativePath, metrics: mgr.metrics, @@ -385,20 +394,8 @@ func (mgr *TransactionManager) Begin(ctx context.Context, opts storage.BeginOpti mgr.mutex.Lock() removedAnyEntry = mgr.cleanCommittedEntry(entry) mgr.mutex.Unlock() - - // Signal the manager this transaction finishes. The purpose of this signaling is to wake it up - // and clean up stale entries in the database. The manager scans and removes leading empty - // entries. We signal only if the transaction modifies the in-memory committed entry. - // This signal queue is buffered. If the queue is full, the manager hasn't woken up. The - // next scan will cover the work of the prior one. So, no need to let the transaction wait. - // ┌─ 1st signal ┌─ The manager scans til here - // □ □ □ □ □ □ □ □ □ □ ■ ■ ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ■ ■ ⧅ ⧅ ⧅ ⧅ ■ - // └─ 2nd signal if removedAnyEntry { - select { - case mgr.completedQueue <- struct{}{}: - default: - } + mgr.NotifyNewCommittedEntry() } } }() @@ -925,7 +922,7 @@ func (p *consumerPosition) setPosition(pos storage.LSN) { // - The reference verification failures can be ignored instead of aborting the entire transaction. // If done, the references that failed verification are dropped from the transaction but the updates // that passed verification are still performed. -// 2. The transaction is appended to the write-ahead log. Once the write has been logged, it is effectively +// 2. The transaction is committed to the write-ahead log. Once the write has been logged, it is effectively // committed and will be applied to the repository even after restarting. // 3. The transaction is applied from the write-ahead log to the repository by actually performing the reference // changes. @@ -998,7 +995,7 @@ type TransactionManager struct { // initializationSuccessful is set if the TransactionManager initialized successfully. If it didn't, // transactions will fail to begin. initializationSuccessful bool - // mutex guards access to snapshotLocks and appendedLSN. These fields are accessed by both + // mutex guards access to snapshotLocks and committedLSN. These fields are accessed by both // Run and Begin which are ran in different goroutines. mutex sync.Mutex @@ -1008,8 +1005,18 @@ type TransactionManager struct { // snapshotManager is responsible for creation and management of file system snapshots. snapshotManager *snapshot.Manager - // appendedLSN holds the LSN of the last log entry appended to the partition's write-ahead log. + // ┌─ oldestLSN ┌─ committedLSN + // ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ■ ■ ■ ■ ■ ■ ■ ■ ■ ■ □ □ □ □ □ □ □ + // └─ appliedLSN └─ appendedLSN + // + // appendedLSN holds the LSN of the last log entry emitted by the current node but not yet acknowledged by other + // nodes. After so, the committedLSN is increment respectively and catches up with appendedLSN. If Raft is not + // enabled or functions as a single-node cluster, committedLSN is increment instantly. appendedLSN storage.LSN + // committedLSN holds the LSN of the last log entry committed to the partition's write-ahead log. A log entry is + // considered to be committed if it's accepted by the majority of cluster members. Eventually, it will be + // applied by all cluster members. + committedLSN storage.LSN // appliedLSN holds the LSN of the last log entry applied to the partition. appliedLSN storage.LSN // oldestLSN holds the LSN of the head of log entries which is still kept in the database. The manager keeps @@ -1020,7 +1027,11 @@ type TransactionManager struct { // the partition. It's keyed by the LSN the transaction is waiting to be applied and the // value is the resultChannel that is waiting the result. awaitingTransactions map[storage.LSN]resultChannel - // committedEntries keeps some latest appended log entries around. Some types of transactions, such as + + // appendedEntries keeps track of appended but not-yet committed entries. After an entry is committed, it is + // removed from this map. This provides quick reference to those entries. + appendedEntries map[storage.LSN]*gitalypb.LogEntry + // committedEntries keeps some latest committed log entries around. Some types of transactions, such as // housekeeping, operate on snapshot repository. There is a gap between transaction doing its work and the time // when it is committed. They need to verify if concurrent operations can cause conflict. These log entries are // still kept around even after they are applied. They are removed when there are no active readers accessing @@ -1041,14 +1052,33 @@ type TransactionManager struct { // metrics stores reporters which facilitate metric recording of transactional operations. metrics ManagerMetrics + + // raftManager controls cross-network replication using the Raft consensus algorithm. Raft manager follows the + // life cycle of Transaction Manager. Major interactions between Raft Manager and Transaction Manager are + // summarized in the below graph: + // + // Without Raft + // TransactionManager New Transaction ┌────┬───────┐ + // │ ▼ │ ▼ ▼ + // └─►Initialize───►txn.Commit()─►Verify─┤ Append Commit─►Apply ──...──► Stop + // │ │ ▲ ▲ │ + // ▼ ▼ │ │ ▼ + // ┌── Run()──────────...───────────Propose──┴───────┘ ───────...─────── Stop + // │ ▲ │ + // Raft Manager Conf Change │ │ + // ┌──┴─▼─────┐ + // │Raft Group├◄───► Network + // └──────────┘ + raftManager *raftmgr.Manager } type testHooks struct { beforeInitialization func() - beforeAppendLogEntry func() - beforeApplyLogEntry func() - beforeStoreAppliedLSN func() - beforeDeleteLogEntryFiles func() + beforeAppendLogEntry func(storage.LSN) + beforeCommitLogEntry func(storage.LSN) + beforeApplyLogEntry func(storage.LSN) + beforeStoreAppliedLSN func(storage.LSN) + beforeDeleteLogEntryFiles func(storage.LSN) beforeRunExiting func() } @@ -1065,6 +1095,7 @@ func NewTransactionManager( repositoryFactory localrepo.StorageScopedFactory, metrics ManagerMetrics, consumer LogConsumer, + raftManager *raftmgr.Manager, ) *TransactionManager { ctx, cancel := context.WithCancel(context.Background()) @@ -1089,18 +1120,21 @@ func NewTransactionManager( stateDirectory: stateDir, stagingDirectory: stagingDir, awaitingTransactions: make(map[storage.LSN]resultChannel), + appendedEntries: map[storage.LSN]*gitalypb.LogEntry{}, committedEntries: list.New(), metrics: metrics, consumer: consumer, consumerPos: consumerPos, acknowledgedQueue: make(chan struct{}, 1), + raftManager: raftManager, testHooks: testHooks{ beforeInitialization: func() {}, - beforeAppendLogEntry: func() {}, - beforeApplyLogEntry: func() {}, - beforeStoreAppliedLSN: func() {}, - beforeDeleteLogEntryFiles: func() {}, + beforeAppendLogEntry: func(storage.LSN) {}, + beforeCommitLogEntry: func(storage.LSN) {}, + beforeApplyLogEntry: func(storage.LSN) {}, + beforeStoreAppliedLSN: func(storage.LSN) {}, + beforeDeleteLogEntryFiles: func(storage.LSN) {}, beforeRunExiting: func() {}, }, } @@ -1115,8 +1149,6 @@ func (mgr *TransactionManager) commit(ctx context.Context, transaction *Transact span, ctx := tracing.StartSpanIfHasParent(ctx, "transaction.Commit", nil) defer span.Finish() - transaction.result = make(resultChannel, 1) - if transaction.repositoryTarget() && !transaction.repositoryExists { // Determine if the repository was created in this transaction and stage its state // for committing if so. @@ -1130,6 +1162,43 @@ func (mgr *TransactionManager) commit(ctx context.Context, transaction *Transact } } + if err := mgr.prepareCommit(ctx, transaction); err != nil { + return err + } + + if err := func() error { + defer trace.StartRegion(ctx, "commit queue").End() + transaction.metrics.commitQueueDepth.Inc() + defer transaction.metrics.commitQueueDepth.Dec() + defer prometheus.NewTimer(mgr.metrics.commitQueueWaitSeconds).ObserveDuration() + + select { + case mgr.admissionQueue <- transaction: + transaction.admitted = true + return nil + case <-ctx.Done(): + return ctx.Err() + case <-mgr.closing: + return storage.ErrTransactionProcessingStopped + } + }(); err != nil { + return err + } + + defer trace.StartRegion(ctx, "result wait").End() + select { + case err := <-transaction.result: + return unwrapExpectedError(err) + case <-ctx.Done(): + return ctx.Err() + case <-mgr.closed: + return storage.ErrTransactionProcessingStopped + } +} + +func (mgr *TransactionManager) prepareCommit(ctx context.Context, transaction *Transaction) error { + transaction.result = make(resultChannel, 1) + // Create a directory to store all staging files. if err := os.Mkdir(transaction.walFilesPath(), mode.Directory); err != nil { return fmt.Errorf("create wal files directory: %w", err) @@ -1228,7 +1297,7 @@ func (mgr *TransactionManager) commit(ctx context.Context, transaction *Transact // The reference updates are staged into the transaction when they are verified, including // the packed-refs. While the reference files would generally be small, the packed-refs file // may be large. Sync the contents of the file before entering the critical section to ensure - // we don't end up syncing the potentially very large file to disk when we're appending the + // we don't end up syncing the potentially very large file to disk when we're committing the // log entry. preImagePackedRefsInode, err := getInode(transaction.originalPackedRefsFilePath()) if err != nil { @@ -1251,34 +1320,7 @@ func (mgr *TransactionManager) commit(ctx context.Context, transaction *Transact } } - if err := func() error { - defer trace.StartRegion(ctx, "commit queue").End() - transaction.metrics.commitQueueDepth.Inc() - defer transaction.metrics.commitQueueDepth.Dec() - defer prometheus.NewTimer(mgr.metrics.commitQueueWaitSeconds).ObserveDuration() - - select { - case mgr.admissionQueue <- transaction: - transaction.admitted = true - return nil - case <-ctx.Done(): - return ctx.Err() - case <-mgr.closing: - return storage.ErrTransactionProcessingStopped - } - }(); err != nil { - return err - } - - defer trace.StartRegion(ctx, "result wait").End() - select { - case err := <-transaction.result: - return unwrapExpectedError(err) - case <-ctx.Done(): - return ctx.Err() - case <-mgr.closed: - return storage.ErrTransactionProcessingStopped - } + return nil } // replaceObjectDirectory replaces the snapshot repository's object directory @@ -1774,7 +1816,7 @@ func (mgr *TransactionManager) preparePackRefsReftable(ctx context.Context, tran Name: "pack-refs", // By using the '--auto' flag, we ensure that git uses the best heuristic // for compaction. For reftables, it currently uses a geometric progression. - // This ensures we don't keep compacting unecessarily to a single file. + // This ensures we don't keep compacting unnecessarily to a single file. Flags: []gitcmd.Option{gitcmd.Flag{Name: "--auto"}}, }, gitcmd.WithStderr(&stderr)); err != nil { return structerr.New("exec pack-refs: %w", err).WithMetadata("stderr", stderr.String()) @@ -2131,7 +2173,7 @@ func unwrapExpectedError(err error) error { return err } -// Run starts the transaction processing. On start up Run loads the indexes of the last appended and applied +// Run starts the transaction processing. On start up Run loads the indexes of the last committed and applied // log entries from the database. It will then apply any transactions that have been logged but not applied // to the repository. Once the recovery is completed, Run starts processing new transactions by verifying the // references, logging the transaction and finally applying it to the repository. The transactions are acknowledged @@ -2161,8 +2203,19 @@ func (mgr *TransactionManager) run(ctx context.Context) (returnedErr error) { return fmt.Errorf("initialize: %w", err) } + if mgr.raftManager != nil { + if err := mgr.raftManager.Run(ctx, mgr); err != nil { + return fmt.Errorf("starting raft manager: %w", err) + } + } + for { - if mgr.appliedLSN < mgr.appendedLSN { + // We prioritize applying committed log entries to the partition first. + mgr.mutex.Lock() + shouldApply := mgr.appliedLSN < mgr.committedLSN + mgr.mutex.Unlock() + + if shouldApply { lsn := mgr.appliedLSN + 1 if err := mgr.applyLogEntry(ctx, lsn); err != nil { @@ -2184,7 +2237,9 @@ func (mgr *TransactionManager) run(ctx context.Context) (returnedErr error) { if err := mgr.deleteLogEntry(ctx, mgr.oldestLSN); err != nil { return fmt.Errorf("deleting log entry: %w", err) } + mgr.mutex.Lock() mgr.oldestLSN++ + mgr.mutex.Unlock() continue } @@ -2198,6 +2253,12 @@ func (mgr *TransactionManager) run(ctx context.Context) (returnedErr error) { // logging it. func (mgr *TransactionManager) processTransaction(ctx context.Context) (returnedErr error) { var transaction *Transaction + + var raftDone <-chan struct{} + if mgr.raftManager != nil { + raftDone = mgr.raftManager.Done() + } + select { case transaction = <-mgr.admissionQueue: defer trace.StartRegion(ctx, "processTransaction").End() @@ -2216,6 +2277,7 @@ func (mgr *TransactionManager) processTransaction(ctx context.Context) (returned case <-mgr.acknowledgedQueue: return nil case <-ctx.Done(): + case <-raftDone: } // Return if the manager was stopped. The select is indeterministic so this guarantees @@ -2224,98 +2286,114 @@ func (mgr *TransactionManager) processTransaction(ctx context.Context) (returned return err } + // If Raft stops, we should not continue processing more entries. + if mgr.raftManager != nil { + if err := mgr.raftManager.Err(); err != nil { + return fmt.Errorf("raft error: %w", err) + } + } + span, ctx := tracing.StartSpanIfHasParent(ctx, "transaction.processTransaction", nil) defer span.Finish() - if err := func() (commitErr error) { - repositoryExists, err := mgr.doesRepositoryExist(ctx, transaction.relativePath) + if err := func() error { + logEntry, logEntryPath, err := mgr.packageLogEntry(ctx, transaction) if err != nil { - return fmt.Errorf("does repository exist: %w", err) + return err } - logEntry := &gitalypb.LogEntry{ - RelativePath: transaction.relativePath, - } + return mgr.proposeLogEntry(ctx, transaction.objectDependencies, logEntry, logEntryPath) + }(); err != nil { + transaction.result <- err + return nil + } - if transaction.repositoryCreation != nil && repositoryExists { - return ErrRepositoryAlreadyExists - } else if transaction.repositoryCreation == nil && !repositoryExists { - return storage.ErrRepositoryNotFound - } + mgr.awaitingTransactions[mgr.committedLSN] = transaction.result - alternateRelativePath, err := mgr.verifyAlternateUpdate(ctx, transaction) - if err != nil { - return fmt.Errorf("verify alternate update: %w", err) - } + return nil +} - if err := mgr.setupStagingRepository(ctx, transaction, alternateRelativePath); err != nil { - return fmt.Errorf("setup staging snapshot: %w", err) - } +// packageLogEntry verifies the transaction and packaged its content to respective log entry. +func (mgr *TransactionManager) packageLogEntry(ctx context.Context, transaction *Transaction) (*gitalypb.LogEntry, string, error) { + repositoryExists, err := mgr.doesRepositoryExist(ctx, transaction.relativePath) + if err != nil { + return nil, "", fmt.Errorf("does repository exist: %w", err) + } - // Verify that all objects this transaction depends on are present in the repository. The dependency - // objects are the reference tips set in the transaction and the objects the transaction's packfile - // is based on. If an object dependency is missing, the transaction is aborted as applying it would - // result in repository corruption. - if err := mgr.verifyObjectsExist(ctx, transaction.stagingRepository, transaction.objectDependencies); err != nil { - return fmt.Errorf("verify object dependencies: %w", err) - } + logEntry := &gitalypb.LogEntry{ + RelativePath: transaction.relativePath, + } - if transaction.repositoryCreation == nil && transaction.runHousekeeping == nil && !transaction.deleteRepository { - logEntry.ReferenceTransactions, err = mgr.verifyReferences(ctx, transaction) - if err != nil { - return fmt.Errorf("verify references: %w", err) - } - } + if transaction.repositoryCreation != nil && repositoryExists { + return nil, "", ErrRepositoryAlreadyExists + } else if transaction.repositoryCreation == nil && !repositoryExists { + return nil, "", storage.ErrRepositoryNotFound + } - if transaction.customHooksUpdated { - // Log a deletion of the existing custom hooks so they are removed before the - // new ones are put in place. - if err := transaction.walEntry.RecordDirectoryRemoval( - mgr.storagePath, - filepath.Join(transaction.relativePath, repoutil.CustomHooksDir), - ); err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("record custom hook removal: %w", err) - } - } + alternateRelativePath, err := mgr.verifyAlternateUpdate(ctx, transaction) + if err != nil { + return nil, "", fmt.Errorf("verify alternate update: %w", err) + } - if transaction.deleteRepository { - logEntry.RepositoryDeletion = &gitalypb.LogEntry_RepositoryDeletion{} + if err := mgr.setupStagingRepository(ctx, transaction, alternateRelativePath); err != nil { + return nil, "", fmt.Errorf("setup staging snapshot: %w", err) + } - if err := transaction.walEntry.RecordDirectoryRemoval( - mgr.storagePath, - transaction.relativePath, - ); err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("record repository removal: %w", err) - } + // Verify that all objects this transaction depends on are present in the repository. The dependency + // objects are the reference tips set in the transaction and the objects the transaction's packfile + // is based on. If an object dependency is missing, the transaction is aborted as applying it would + // result in repository corruption. + if err := mgr.verifyObjectsExist(ctx, transaction.stagingRepository, transaction.objectDependencies); err != nil { + return nil, "", fmt.Errorf("verify object dependencies: %w", err) + } - if err := transaction.KV().Delete(relativePathKey(transaction.relativePath)); err != nil { - return fmt.Errorf("delete relative path: %w", err) - } + if transaction.repositoryCreation == nil && transaction.runHousekeeping == nil && !transaction.deleteRepository { + logEntry.ReferenceTransactions, err = mgr.verifyReferences(ctx, transaction) + if err != nil { + return nil, "", fmt.Errorf("verify references: %w", err) } + } - if transaction.runHousekeeping != nil { - housekeepingEntry, err := mgr.verifyHousekeeping(ctx, transaction) - if err != nil { - return fmt.Errorf("verifying pack refs: %w", err) - } - logEntry.Housekeeping = housekeepingEntry + if transaction.customHooksUpdated { + // Log a deletion of the existing custom hooks so they are removed before the + // new ones are put in place. + if err := transaction.walEntry.RecordDirectoryRemoval( + mgr.storagePath, + filepath.Join(transaction.relativePath, repoutil.CustomHooksDir), + ); err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, "", fmt.Errorf("record custom hook removal: %w", err) } + } - if err := mgr.verifyKeyValueOperations(ctx, transaction); err != nil { - return fmt.Errorf("verify key-value operations: %w", err) + if transaction.deleteRepository { + logEntry.RepositoryDeletion = &gitalypb.LogEntry_RepositoryDeletion{} + + if err := transaction.walEntry.RecordDirectoryRemoval( + mgr.storagePath, + transaction.relativePath, + ); err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, "", fmt.Errorf("record repository removal: %w", err) } - logEntry.Operations = transaction.walEntry.Operations() + if err := transaction.KV().Delete(relativePathKey(transaction.relativePath)); err != nil { + return nil, "", fmt.Errorf("delete relative path: %w", err) + } + } - return mgr.appendLogEntry(ctx, transaction.objectDependencies, logEntry, transaction.walFilesPath()) - }(); err != nil { - transaction.result <- err - return nil + if transaction.runHousekeeping != nil { + housekeepingEntry, err := mgr.verifyHousekeeping(ctx, transaction) + if err != nil { + return nil, "", fmt.Errorf("verifying pack refs: %w", err) + } + logEntry.Housekeeping = housekeepingEntry } - mgr.awaitingTransactions[mgr.appendedLSN] = transaction.result + if err := mgr.verifyKeyValueOperations(ctx, transaction); err != nil { + return nil, "", fmt.Errorf("verify key-value operations: %w", err) + } - return nil + logEntry.Operations = transaction.walEntry.Operations() + return logEntry, transaction.walFilesPath(), nil } // verifyKeyValueOperations checks the key-value operations of the transaction for conflicts and includes @@ -2410,7 +2488,12 @@ func (mgr *TransactionManager) verifyObjectsExist(ctx context.Context, repositor } // Close stops the transaction processing causing Run to return. -func (mgr *TransactionManager) Close() { mgr.close() } +func (mgr *TransactionManager) Close() { + if mgr.raftManager != nil { + mgr.raftManager.Stop() + } + mgr.close() +} // CloseSnapshots closes any remaining snapshots in the cache. Caller of Run() should // call it after there are no more active transactions and no new transactions will be @@ -2429,7 +2512,7 @@ func (mgr *TransactionManager) snapshotsDir() string { return filepath.Join(mgr.stagingDirectory, "snapshots") } -// initialize initializes the TransactionManager's state from the database. It loads the appended and the applied +// initialize initializes the TransactionManager's state from the database. It loads the committed and the applied // LSNs and initializes the notification channels that synchronize transaction beginning with log entry applying. func (mgr *TransactionManager) initialize(ctx context.Context) error { defer trace.StartRegion(ctx, "initialize").End() @@ -2456,38 +2539,70 @@ func (mgr *TransactionManager) initialize(ctx context.Context) error { return fmt.Errorf("new snapshot manager: %w", err) } - // The LSN of the last appended log entry is determined from the LSN of the latest entry in the log and + // The LSN of the last committed log entry is determined from the LSN of the latest entry in the log and // the latest applied log entry. The manager also keeps track of committed entries and reserves them until there // is no transaction refers them. It's possible there are some left-over entries in the database because a // transaction can hold the entry stubbornly. So, the manager could not clean them up in the last session. // - // ┌─ oldestLSN ┌─ appendedLSN - // ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ■ ■ ■ ■ ■ ■ ■ ■ ■ ■ - // └─ appliedLSN + // ┌─ oldestLSN ┌─ committedLSN + // ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ■ ■ ■ ■ ■ ■ ■ ■ ■ ■ □ □ □ □ □ □ □ + // └─ appliedLSN └─ appendedLSN // // // oldestLSN is initialized to appliedLSN + 1. If there are no log entries in the log, then everything has been // pruned already or there has not been any log entries yet. Setting this +1 avoids trying to clean up log entries // that do not exist. If there are some, we'll set oldestLSN to the head of the log below. mgr.oldestLSN = mgr.appliedLSN + 1 - // appendedLSN is initialized to appliedLSN. If there are no log entries, then there has been no transaction yet, or - // all log entries have been applied and have been already pruned. If there are some in the log, we'll update this - // below to match. - mgr.appendedLSN = mgr.appliedLSN + + // CommittedLSN is loaded from DB. A log entry is appended first and marked as committed later. There's a chance + // that log entry is never marked as committed. After a restart, especially after a crash, the manager won't be + // able to tell if it's committed or not. Thus, we need to persist this index. + // Because index persistence is introduced later, we need to fallback to appliedLSN if that key does not exist + // in the DB. + var committedLSN gitalypb.LSN + if err := mgr.readKey(keyCommittedLSN, &committedLSN); err != nil { + if !errors.Is(err, badger.ErrKeyNotFound) { + return fmt.Errorf("read committed LSN: %w", err) + } + mgr.committedLSN = mgr.appliedLSN + } else { + mgr.committedLSN = storage.LSN(committedLSN.GetValue()) + } + + // appendedLSN is always set to committedLSN after starting. If a log entry hasn't been committed, it could be + // discarded. Its caller never received the acknowledgement. + mgr.appendedLSN = mgr.committedLSN if logEntries, err := os.ReadDir(walFilesPath(mgr.stateDirectory)); err != nil { return fmt.Errorf("read wal directory: %w", err) } else if len(logEntries) > 0 { - if mgr.oldestLSN, err = storage.ParseLSN(logEntries[0].Name()); err != nil { - return fmt.Errorf("parse oldest LSN: %w", err) - } - if mgr.appendedLSN, err = storage.ParseLSN(logEntries[len(logEntries)-1].Name()); err != nil { - return fmt.Errorf("parse appended LSN: %w", err) + // All log entries starting from mgr.committedLSN + 1 are not committed. No reason to keep them around. + // Returned log entries are sorted in ascending order. We iterate backward and break when the iterating + // LSN drops below committedLSN. + for i := len(logEntries) - 1; i >= 0; i-- { + logEntry := logEntries[i] + + lsn, err := storage.ParseLSN(logEntry.Name()) + if err != nil { + return fmt.Errorf("parse LSN: %w", err) + } + if lsn <= mgr.committedLSN { + // Found some on-disk log entries older than or equal to committedLSN. They might be + // referenced by other transactions before restart. Eventually, they'll be removed in + // the main loop. + if mgr.oldestLSN, err = storage.ParseLSN(logEntries[0].Name()); err != nil { + return fmt.Errorf("parse oldest LSN: %w", err) + } + break + } + if err := mgr.deleteLogEntry(mgr.ctx, lsn); err != nil { + return fmt.Errorf("cleaning uncommitted log entry: %w", err) + } } } if mgr.consumer != nil { - mgr.consumer.NotifyNewTransactions(mgr.storageName, mgr.partitionID, mgr.oldestLSN, mgr.appendedLSN) + mgr.consumer.NotifyNewTransactions(mgr.storageName, mgr.partitionID, mgr.oldestLSN, mgr.committedLSN) } // Create a snapshot lock for the applied LSN as it is used for synchronizing @@ -2497,11 +2612,11 @@ func (mgr *TransactionManager) initialize(ctx context.Context) error { // Each unapplied log entry should have a snapshot lock as they are created in normal // operation when committing a log entry. Recover these entries. - for i := mgr.appliedLSN + 1; i <= mgr.appendedLSN; i++ { + for i := mgr.appliedLSN + 1; i <= mgr.committedLSN; i++ { mgr.snapshotLocks[i] = &snapshotLock{applied: make(chan struct{})} } - if err := mgr.removeStaleWALFiles(ctx, mgr.oldestLSN, mgr.appendedLSN); err != nil { + if err := mgr.removeStaleWALFiles(ctx, mgr.oldestLSN, mgr.committedLSN); err != nil { return fmt.Errorf("remove stale packs: %w", err) } @@ -2597,15 +2712,15 @@ func (mgr *TransactionManager) removePackedRefsLocks(ctx context.Context, reposi // but the manager was interrupted before successfully persisting the log entry itself. // If the manager deletes a log entry successfully from the database but is interrupted before it cleans // up the associated files, such a directory can also be left at the head of the log. -func (mgr *TransactionManager) removeStaleWALFiles(ctx context.Context, oldestLSN, appendedLSN storage.LSN) error { +func (mgr *TransactionManager) removeStaleWALFiles(ctx context.Context, oldestLSN, committedLSN storage.LSN) error { needsFsync := false for _, possibleStaleFilesPath := range []string{ // Log entries are pruned one by one. If a write is interrupted, the only possible stale files would be // for the log entry preceding the oldest log entry. walFilesPathForLSN(mgr.stateDirectory, oldestLSN-1), - // Log entries are appended one by one to the log. If a write is interrupted, the only possible stale + // Log entries are committed one by one to the log. If a write is interrupted, the only possible stale // files would be for the next LSN. Remove the files if they exist. - walFilesPathForLSN(mgr.stateDirectory, appendedLSN+1), + walFilesPathForLSN(mgr.stateDirectory, committedLSN+1), } { if _, err := os.Stat(possibleStaleFilesPath); err != nil { @@ -2654,6 +2769,11 @@ func packFilePath(walFiles string) string { return filepath.Join(walFiles, "transaction.pack") } +// LSNDirPath returns the path to WAL dir of a particular LSN. +func (mgr *TransactionManager) LSNDirPath(lsn storage.LSN) string { + return walFilesPathForLSN(mgr.stateDirectory, lsn) +} + // verifyAlternateUpdate verifies the staged alternate update. func (mgr *TransactionManager) verifyAlternateUpdate(ctx context.Context, transaction *Transaction) (string, error) { defer trace.StartRegion(ctx, "verifyAlternateUpdate").End() @@ -2857,7 +2977,7 @@ func (mgr *TransactionManager) verifyReferences(ctx context.Context, transaction // to transaction operations. // // To ensure that we don't modify existing tables and autocompact, we lock the existing tables -// before applying the updates. This way the reftable backend willl only create new tables +// before applying the updates. This way the reftable backend will only create new tables func (mgr *TransactionManager) verifyReferencesWithGitForReftables( ctx context.Context, referenceTransactions []*gitalypb.LogEntry_ReferenceTransaction, @@ -3016,7 +3136,7 @@ func (mgr *TransactionManager) stagePackedRefs(ctx context.Context, tx *Transact } if containsReferenceDeletions { - if err := mgr.walkCommittedEntries(tx, func(entry *gitalypb.LogEntry, dependencies map[git.ObjectID]struct{}) error { + if err := mgr.walkCommittedEntries(tx, func(entry *gitalypb.LogEntry, _ map[git.ObjectID]struct{}) error { if entry.GetHousekeeping().GetPackRefs() != nil { return errConcurrentReferencePacking } @@ -3139,8 +3259,8 @@ func (mgr *TransactionManager) verifyHousekeeping(ctx context.Context, transacti mgr.mutex.Lock() defer mgr.mutex.Unlock() - // Check for any concurrent housekeeping between this transaction's snapshot LSN and the latest appended LSN. - if err := mgr.walkCommittedEntries(transaction, func(entry *gitalypb.LogEntry, objectDependencies map[git.ObjectID]struct{}) error { + // Check for any concurrent housekeeping between this transaction's snapshot LSN and the latest committed LSN. + if err := mgr.walkCommittedEntries(transaction, func(entry *gitalypb.LogEntry, _ map[git.ObjectID]struct{}) error { if entry.GetHousekeeping() != nil { return errHousekeepingConflictConcurrent } @@ -3272,7 +3392,7 @@ func (mgr *TransactionManager) verifyPackRefsFiles(ctx context.Context, transact packRefs := transaction.runHousekeeping.packRefs // Check for any concurrent ref deletion between this transaction's snapshot LSN to the end. - if err := mgr.walkCommittedEntries(transaction, func(entry *gitalypb.LogEntry, objectDependencies map[git.ObjectID]struct{}) error { + if err := mgr.walkCommittedEntries(transaction, func(entry *gitalypb.LogEntry, _ map[git.ObjectID]struct{}) error { for _, refTransaction := range entry.GetReferenceTransactions() { for _, change := range refTransaction.GetChanges() { // We handle HEAD updates through the git-update-ref, but since @@ -3506,12 +3626,128 @@ func (mgr *TransactionManager) applyReferenceTransaction(ctx context.Context, ch return nil } -// appendLogEntry appends the transaction to the write-ahead log. It first writes the transaction's manifest file -// into the log entry's directory. Afterwards it moves the log entry's directory from the staging area to its final -// place in the write-ahead log. -func (mgr *TransactionManager) appendLogEntry(ctx context.Context, objectDependencies map[git.ObjectID]struct{}, logEntry *gitalypb.LogEntry, logEntryPath string) error { +// proposeLogEntry proposes a log etnry of a transaction to the write-ahead log. It first writes the transaction's +// manifest file into the log entry's directory. Second, it sends the log entry to other cluster members if needed. +// Afterwards it moves the log entry's directory from the staging area to its final place in the write-ahead log. +func (mgr *TransactionManager) proposeLogEntry(ctx context.Context, objectDependencies map[git.ObjectID]struct{}, logEntry *gitalypb.LogEntry, logEntryPath string) error { + if mgr.raftManager != nil { + if err := mgr.raftManager.Propose(ctx, logEntry, logEntryPath); err != nil { + return err + } + } else { + nextLSN := mgr.committedLSN + 1 + if err := mgr.AppendLogEntry(ctx, nextLSN, logEntry, logEntryPath); err != nil { + return fmt.Errorf("append log entry: %w", err) + } + if err := mgr.CommitLogEntry(ctx, nextLSN); err != nil { + return fmt.Errorf("commit log entry: %w", err) + } + } + + mgr.mutex.Lock() + // Tracking object dependencies is only relevant to repacking task that runs in parallel with other operations. + // When the task finishes, the Transaction Manager needs to verify the existence of all dependent objects of + // entries since the time the repacking task started. The current node must be primary when the task started. + // This node need to backfill object dependencies back to the committed entries list in this code path. In + // contrast, replicas don't need to do so. If this node wins an election, the next log entry uses the snapshot + // from at the latest committed entry; hence prior object dependencies are not relevant. + if elm := mgr.committedEntries.Back(); elm != nil { + entry := elm.Value.(*committedEntry) + entry.objectDependencies = objectDependencies + } + mgr.mutex.Unlock() + + return nil +} + +// InsertLogEntry inserts a KV log entry into the WAL at the specified LSN. The caller can manipulate the data +// by passing a modification function. It's responsibility of the caller NOT TO append/insert any log entries +// while this function is running. All adjacent log entries at and after inserting position are wiped. +func (mgr *TransactionManager) InsertLogEntry( + ctx context.Context, + lsn storage.LSN, + txnFunc func(keyvalue.ReadWriter) error, + metadata *gitalypb.LogEntry_Metadata, +) (_ *gitalypb.LogEntry, returnedErr error) { + txn, err := mgr.Begin(ctx, storage.BeginOptions{ + Write: true, + RelativePaths: []string{}, + WithoutRaftReady: true, + }) + if err != nil { + return nil, fmt.Errorf("initializing transaction: %w", err) + } + + if txnFunc != nil { + if err := txnFunc(txn.KV()); err != nil { + defer func() { + if rollbackErr := txn.Rollback(ctx); rollbackErr != nil { + returnedErr = errors.Join(returnedErr, fmt.Errorf("rollback transaction: %w", rollbackErr)) + } + }() + return nil, fmt.Errorf("mutate KV transaction: %w", err) + } + } + + transaction := txn.(*Transaction) + defer func() { + if err := transaction.finish(); err != nil && returnedErr == nil { + returnedErr = fmt.Errorf("finish transaction: %w", err) + } + }() + defer trace.StartRegion(ctx, "commit").End() + defer prometheus.NewTimer(transaction.metrics.commitDuration(transaction.write)).ObserveDuration() + + if err = transaction.updateState(transactionStateCommit); err != nil { + return nil, err + } + transaction.admitted = true + + if err = mgr.prepareCommit(ctx, transaction); err != nil { + return nil, fmt.Errorf("preparing commit: %w", err) + } + + logEntry, logEntryPath, err := mgr.packageLogEntry(ctx, transaction) + if err != nil { + return nil, fmt.Errorf("packaging log entry: %w", err) + } + + if err := mgr.AppendLogEntry(ctx, lsn, logEntry, logEntryPath); err != nil { + return nil, fmt.Errorf("appending log entry: %w", err) + } + return logEntry, nil +} + +// AppendLogEntry appends the transaction to the write-ahead log. +func (mgr *TransactionManager) AppendLogEntry(ctx context.Context, lsn storage.LSN, logEntry *gitalypb.LogEntry, logEntryPath string) (err error) { defer trace.StartRegion(ctx, "appendLogEntry").End() + select { + case <-mgr.ctx.Done(): + return mgr.ctx.Err() + default: + } + + // ┌─ committedLSN + // ■ ■ ■ ■ ■ ■ ■ ■ ■ ■ □ □ ⧅ ⧅ ⧅ ⧅ ⧅ + // └─ appliedLSN └─ lsn └─ appendedLSN + // + // When a new log entry is appended, it needs to be confirmed by peers. If the number of confirmations + // (including this node) reach quorum, the log entry is moved to the committed phase. In contrast, that + // log entry and following ones are discarded in favor of more up-to-date entries. + var removeRange storage.LSN + mgr.mutex.Lock() + if lsn <= mgr.committedLSN { + return fmt.Errorf("out-of-order log entry appending, position %s taken", lsn.String()) + } + if lsn <= mgr.appendedLSN { + removeRange = mgr.appendedLSN + } + mgr.mutex.Unlock() + if err := mgr.removeOsboleteLogEntries(lsn, removeRange); err != nil { + return fmt.Errorf("removing obsolete log entries: %w", err) + } + manifestBytes, err := proto.Marshal(logEntry) if err != nil { return fmt.Errorf("marshal manifest: %w", err) @@ -3538,11 +3774,10 @@ func (mgr *TransactionManager) appendLogEntry(ctx context.Context, objectDepende return fmt.Errorf("synchronizing WAL directory: %w", err) } - mgr.testHooks.beforeAppendLogEntry() + mgr.testHooks.beforeAppendLogEntry(lsn) - nextLSN := mgr.appendedLSN + 1 // Move the log entry from the staging directory into its place in the log. - destinationPath := walFilesPathForLSN(mgr.stateDirectory, nextLSN) + destinationPath := walFilesPathForLSN(mgr.stateDirectory, lsn) if err := os.Rename(logEntryPath, destinationPath); err != nil { return fmt.Errorf("move wal files: %w", err) } @@ -3566,20 +3801,119 @@ func (mgr *TransactionManager) appendLogEntry(ctx context.Context, objectDepende // After this latch block, the transaction is committed and all subsequent transactions // are guaranteed to read it. mgr.mutex.Lock() - mgr.appendedLSN = nextLSN - mgr.snapshotLocks[nextLSN] = &snapshotLock{applied: make(chan struct{})} - mgr.committedEntries.PushBack(&committedEntry{ - lsn: nextLSN, - objectDependencies: objectDependencies, - }) + mgr.appendedLSN = lsn + mgr.appendedEntries[mgr.appendedLSN] = logEntry + mgr.mutex.Unlock() + + return nil +} + +// ReadLogEntry reads and returns the log entry stored at the specified LSN within the WAL. It first looks into the +// appended entries before attempting to load it from disk. +func (mgr *TransactionManager) ReadLogEntry(lsn storage.LSN) (*gitalypb.LogEntry, error) { + mgr.mutex.Lock() + defer mgr.mutex.Unlock() + + if lsn < mgr.oldestLSN || lsn > mgr.appendedLSN { + return nil, fmt.Errorf("log entry not found") + } else if lsn > mgr.committedLSN { + // Quick-access from appended entries map. + return mgr.appendedEntries[lsn], nil + } + + return mgr.readLogEntry(lsn) +} + +// FirstLSN retrieves the first LSN that is still accessible in the WAL. +func (mgr *TransactionManager) FirstLSN() storage.LSN { + mgr.mutex.Lock() + defer mgr.mutex.Unlock() + + return mgr.oldestLSN +} + +// LastLSN returns the last LSN that has been appended to the WAL, indicating the latest position. +func (mgr *TransactionManager) LastLSN() storage.LSN { + mgr.mutex.Lock() + defer mgr.mutex.Unlock() + + return mgr.appendedLSN +} + +func (mgr *TransactionManager) removeOsboleteLogEntries(from storage.LSN, to storage.LSN) error { + // Remove log entries reversely to prevent holes in between. + for lsn := to; lsn >= from; lsn-- { + path := walFilesPathForLSN(mgr.stateDirectory, lsn) + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + continue + } + return err + } + if err := os.RemoveAll(path); err != nil { + return err + } + } + return nil +} + +// CommitLogEntry commits the transaction to the write-ahead log. +func (mgr *TransactionManager) CommitLogEntry(ctx context.Context, lsn storage.LSN) (err error) { + defer trace.StartRegion(ctx, "commitLogEntry").End() + mgr.testHooks.beforeCommitLogEntry(lsn) + + select { + case <-mgr.ctx.Done(): + return mgr.ctx.Err() + default: + } + + // Persist committed LSN before updating internal states. + if err := mgr.storeCommittedLSN(lsn); err != nil { + return fmt.Errorf("persisting committed entry: %w", err) + } + + if lsn <= mgr.committedLSN { + return fmt.Errorf("out-of-order log entry committing, position %s taken", lsn.String()) + } + + mgr.mutex.Lock() + mgr.committedLSN = lsn + mgr.snapshotLocks[lsn] = &snapshotLock{applied: make(chan struct{})} + if _, exist := mgr.appendedEntries[lsn]; !exist { + mgr.mutex.Unlock() + return fmt.Errorf("log entry %s not found in the appended list", lsn) + } + delete(mgr.appendedEntries, lsn) + mgr.committedEntries.PushBack(&committedEntry{lsn: lsn}) mgr.mutex.Unlock() if mgr.consumer != nil { - mgr.consumer.NotifyNewTransactions(mgr.storageName, mgr.partitionID, mgr.lowWaterMark(), nextLSN) + mgr.consumer.NotifyNewTransactions(mgr.storageName, mgr.partitionID, mgr.lowWaterMark(), lsn) } + return nil } +// NotifyNewCommittedEntry notifies the Transaction Manager of a new committed log entry. Most of the time, the +// processing loop of the transaction manager receives completion signal from admission queue. After processing, +// committing, then applying a log entry, the processing goroutine is put to idle state until there a new transaction +// commits. This flow works fine if Raft is not enabled. In contrast, when this node acts as a replica, Raft needs to +// notify the processing goroutine manually. +func (mgr *TransactionManager) NotifyNewCommittedEntry() { + // The purpose of this signaling is to wake it up and clean up stale entries in the database. The manager scans and + // removes leading empty entries. We signal only if the transaction modifies the in-memory committed entry. This + // signal queue is buffered. If the queue is full, the manager hasn't woken up. The next scan will cover the work + // of the prior one. So, no need to let the transaction wait. + // ┌─ 1st signal ┌─ The manager scans til here + // □ □ □ □ □ □ □ □ □ □ ■ ■ ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ■ ■ ⧅ ⧅ ⧅ ⧅ ■ + // └─ 2nd signal + select { + case mgr.completedQueue <- struct{}{}: + default: + } +} + // applyLogEntry reads a log entry at the given LSN and applies it to the repository. func (mgr *TransactionManager) applyLogEntry(ctx context.Context, lsn storage.LSN) error { defer trace.StartRegion(ctx, "applyLogEntry").End() @@ -3600,7 +3934,7 @@ func (mgr *TransactionManager) applyLogEntry(ctx context.Context, lsn storage.LS delete(mgr.snapshotLocks, previousLSN) mgr.mutex.Unlock() - mgr.testHooks.beforeApplyLogEntry() + mgr.testHooks.beforeApplyLogEntry(lsn) if err := applyOperations(ctx, safe.NewSyncer().Sync, mgr.storagePath, walFilesPathForLSN(mgr.stateDirectory, lsn), logEntry, mgr.db); err != nil { return fmt.Errorf("apply operations: %w", err) @@ -3615,7 +3949,7 @@ func (mgr *TransactionManager) applyLogEntry(ctx context.Context, lsn storage.LS } } - mgr.testHooks.beforeStoreAppliedLSN() + mgr.testHooks.beforeStoreAppliedLSN(lsn) if err := mgr.storeAppliedLSN(lsn); err != nil { return fmt.Errorf("set applied LSN: %w", err) } @@ -4005,6 +4339,11 @@ func (mgr *TransactionManager) storeAppliedLSN(lsn storage.LSN) error { return mgr.setKey(keyAppliedLSN, lsn.ToProto()) } +// storeCommittedLSN stores the partition's committed LSN in the database. +func (mgr *TransactionManager) storeCommittedLSN(lsn storage.LSN) error { + return mgr.setKey(keyCommittedLSN, lsn.ToProto()) +} + // setKey marshals and stores a given protocol buffer message into the database under the given key. func (mgr *TransactionManager) setKey(key []byte, value proto.Message) error { marshaledValue, err := proto.Marshal(value) diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_alternate_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_alternate_test.go index dbe1843e39f6e3ae6cc7fe3dfd9e6629cfbd90b4..bbd632c432bbe4c1a5eaf27b51a6bbf600a8808f 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_alternate_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_alternate_test.go @@ -1124,9 +1124,7 @@ func generateAlternateTests(t *testing.T, setup testTransactionSetup) []transact CloseManager{}, StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -1203,9 +1201,7 @@ func generateAlternateTests(t *testing.T, setup testTransactionSetup) []transact CloseManager{}, StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_default_branch_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_default_branch_test.go index 5bfd1c5fc969996e08a368dd90627742d1df717d..07ab97957c8e22e1fec21c94d7872832391e8208 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_default_branch_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_default_branch_test.go @@ -464,9 +464,7 @@ func generateDefaultBranchTests(t *testing.T, setup testTransactionSetup) []tran steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookCtx hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go index 3434a94f40824a46259154d08693fc368026b464..a791926902b4d989cebed9ab2bedbf2f485acd5e 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go @@ -8,6 +8,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/mode" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" ) @@ -17,18 +18,18 @@ type hookFunc func(hookContext) // hookContext are the control toggles available in a hook. type hookContext struct { - // closeManager calls the calls Close on the TransactionManager. - closeManager func() + // manager points to the subject TransactionManager. + manager *TransactionManager + // lsn stores the LSN context when the hook is triggered. + lsn storage.LSN + // raftManager allows the access to the Raft manager inside TransactionManager. + raftManager *raftmgr.Manager } // installHooks takes the hooks in the test setup and configures them in the TransactionManager. func installHooks(mgr *TransactionManager, inflightTransactions *sync.WaitGroup, hooks testTransactionHooks) { for destination, source := range map[*func()]hookFunc{ - &mgr.testHooks.beforeInitialization: hooks.BeforeReadAppliedLSN, - &mgr.testHooks.beforeAppendLogEntry: hooks.BeforeAppendLogEntry, - &mgr.testHooks.beforeApplyLogEntry: hooks.BeforeApplyLogEntry, - &mgr.testHooks.beforeStoreAppliedLSN: hooks.BeforeStoreAppliedLSN, - &mgr.testHooks.beforeDeleteLogEntryFiles: hooks.AfterDeleteLogEntry, + &mgr.testHooks.beforeInitialization: hooks.BeforeReadAppliedLSN, &mgr.testHooks.beforeRunExiting: func(hookContext) { if hooks.WaitForTransactionsWhenClosing { inflightTransactions.Wait() @@ -41,7 +42,26 @@ func installHooks(mgr *TransactionManager, inflightTransactions *sync.WaitGroup, runHook := source *destination = func() { runHook(hookContext{ - closeManager: mgr.Close, + manager: mgr, + raftManager: mgr.raftManager, + }) + } + } + } + for destination, source := range map[*func(storage.LSN)]hookFunc{ + &mgr.testHooks.beforeApplyLogEntry: hooks.BeforeApplyLogEntry, + &mgr.testHooks.beforeAppendLogEntry: hooks.BeforeAppendLogEntry, + &mgr.testHooks.beforeCommitLogEntry: hooks.BeforeCommitLogEntry, + &mgr.testHooks.beforeStoreAppliedLSN: hooks.BeforeStoreAppliedLSN, + &mgr.testHooks.beforeDeleteLogEntryFiles: hooks.AfterDeleteLogEntry, + } { + if source != nil { + runHook := source + *destination = func(lsn storage.LSN) { + runHook(hookContext{ + manager: mgr, + lsn: lsn, + raftManager: mgr.raftManager, }) } } @@ -89,9 +109,7 @@ func generateCustomHooksTests(t *testing.T, setup testTransactionSetup) []transa steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -139,9 +157,7 @@ func generateCustomHooksTests(t *testing.T, setup testTransactionSetup) []transa steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeApplyLogEntry: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeApplyLogEntry: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -483,9 +499,7 @@ func generateCustomHooksTests(t *testing.T, setup testTransactionSetup) []transa steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookCtx hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -577,9 +591,7 @@ func generateCustomHooksTests(t *testing.T, setup testTransactionSetup) []transa steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeApplyLogEntry: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeApplyLogEntry: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -671,9 +683,7 @@ func generateCustomHooksTests(t *testing.T, setup testTransactionSetup) []transa steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeApplyLogEntry: func(hookCtx hookContext) { - panic(errSimulatedCrash) - }, + BeforeApplyLogEntry: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_housekeeping_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_housekeeping_test.go index f0e36bd571e21bcb5063f35960618e12387fbf51..5af8d077c473ea9f45319b173f81b4a192ac540d 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_housekeeping_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_housekeeping_test.go @@ -1335,9 +1335,7 @@ func generateHousekeepingPackRefsTests(t *testing.T, ctx context.Context, testPa CloseManager{}, StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_key_value_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_key_value_test.go index b39cc1b2b534ade2356a77f76a317e254e600fc4..25f2055cb69b03980aea29ffa1d9f20a118b1fb4 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_key_value_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_key_value_test.go @@ -1,11 +1,13 @@ package partition import ( + "testing" + "github.com/dgraph-io/badger/v4" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" ) -func generateKeyValueTests(setup testTransactionSetup) []transactionTestCase { +func generateKeyValueTests(t *testing.T, setup testTransactionSetup) []transactionTestCase { return []transactionTestCase{ { desc: "set keys with values", @@ -615,9 +617,7 @@ func generateKeyValueTests(setup testTransactionSetup) []transactionTestCase { steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeApplyLogEntry: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeApplyLogEntry: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_repo_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_repo_test.go index 785204ee3381a73025bcade13af8416aab579165..1f356bc4d6a3dcc3bd1281db672dfbdb29bd1ab2 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_repo_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_repo_test.go @@ -364,9 +364,7 @@ func generateCreateRepositoryTests(t *testing.T, setup testTransactionSetup) []t RemoveRepository{}, StartManager{ Hooks: testTransactionHooks{ - BeforeApplyLogEntry: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeApplyLogEntry: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -408,9 +406,7 @@ func generateCreateRepositoryTests(t *testing.T, setup testTransactionSetup) []t RemoveRepository{}, StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -910,9 +906,7 @@ func generateDeleteRepositoryTests(t *testing.T, setup testTransactionSetup) []t steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeApplyLogEntry: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeApplyLogEntry: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -954,9 +948,7 @@ func generateDeleteRepositoryTests(t *testing.T, setup testTransactionSetup) []t steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go index 021f6adddc322ac4afee2435f6d72fa49ba55e5c..de03e9128aa7d7540e14b3311643172f584cada2 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go @@ -28,17 +28,30 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/mode" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/snapshot" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" ) // errSimulatedCrash is used in the tests to simulate a crash at a certain point during // TransactionManager.Run execution. var errSimulatedCrash = errors.New("simulated crash") +// simulateCrashHook returns a hook function that panics with errSimulatedCrash. +var simulateCrashHook = func() func(hookContext) { + return func(c hookContext) { + if !testhelper.IsRaftEnabled() || + c.raftManager == nil || + !c.raftManager.EntryRecorder.IsFromRaft(c.lsn) { + panic(errSimulatedCrash) + } + } +} + func manifestDirectoryEntry(expected *gitalypb.LogEntry) testhelper.DirectoryEntry { return testhelper.DirectoryEntry{ Mode: mode.File, @@ -339,6 +352,7 @@ func TestTransactionManager(t *testing.T) { subTests := map[string][]transactionTestCase{ "Common": generateCommonTests(t, ctx, setup), "CommittedEntries": generateCommittedEntriesTests(t, setup), + "AppendedEntries": generateAppendedEntriesTests(t, setup), "ModifyReferences": generateModifyReferencesTests(t, setup), "CreateRepository": generateCreateRepositoryTests(t, setup), "DeleteRepository": generateDeleteRepositoryTests(t, setup), @@ -350,7 +364,7 @@ func TestTransactionManager(t *testing.T) { "Housekeeping/RepackingConcurrent": generateHousekeepingRepackingConcurrentTests(t, ctx, setup), "Housekeeping/CommitGraphs": generateHousekeepingCommitGraphsTests(t, ctx, setup), "Consumer": generateConsumerTests(t, setup), - "KeyValue": generateKeyValueTests(setup), + "KeyValue": generateKeyValueTests(t, setup), } for desc, tests := range subTests { @@ -433,59 +447,74 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio ExpectedError: context.Canceled, }, }, - expectedState: StateAssertion{ - Database: DatabaseState{ - string(keyAppliedLSN): storage.LSN(1).ToProto(), + expectedState: testhelper.WithOrWithoutRaft( + StateAssertion{ + Database: DatabaseState{ + string(keyAppliedLSN): storage.LSN(1).ToProto(), + string(keyCommittedLSN): storage.LSN(1).ToProto(), + }, + NotOffsetDatabaseInRaft: true, }, - Repositories: RepositoryStates{ - setup.RelativePath: { - DefaultBranch: "refs/heads/main", - References: gittest.FilesOrReftables( - &ReferencesState{ - FilesBackend: &FilesBackendState{ - LooseReferences: map[git.ReferenceName]git.ObjectID{ - "refs/heads/main": setup.Commits.First.OID, + StateAssertion{ + Database: DatabaseState{ + string(keyAppliedLSN): storage.LSN(1).ToProto(), + string(keyCommittedLSN): storage.LSN(1).ToProto(), + }, + Repositories: RepositoryStates{ + setup.RelativePath: { + DefaultBranch: "refs/heads/main", + References: gittest.FilesOrReftables( + &ReferencesState{ + FilesBackend: &FilesBackendState{ + LooseReferences: map[git.ReferenceName]git.ObjectID{ + "refs/heads/main": setup.Commits.First.OID, + }, }, - }, - }, &ReferencesState{ - ReftableBackend: &ReftableBackendState{ - Tables: []ReftableTable{ - { - MinIndex: 1, - MaxIndex: 1, - References: []git.Reference{ - { - Name: "HEAD", - Target: "refs/heads/main", - IsSymbolic: true, + }, &ReferencesState{ + ReftableBackend: &ReftableBackendState{ + Tables: []ReftableTable{ + { + MinIndex: 1, + MaxIndex: 1, + References: []git.Reference{ + { + Name: "HEAD", + Target: "refs/heads/main", + IsSymbolic: true, + }, }, }, - }, - { - MinIndex: 2, - MaxIndex: 2, - References: []git.Reference{ - { - Name: "refs/heads/main", - Target: setup.Commits.First.OID.String(), + { + MinIndex: 2, + MaxIndex: 2, + References: []git.Reference{ + { + Name: "refs/heads/main", + Target: setup.Commits.First.OID.String(), + }, }, }, }, }, }, - }, - ), + ), + }, }, }, - }, + ), } }(), { - desc: "commit returns if transaction processing stops before transaction acceptance", + desc: "commit returns if context is canceled before admission", + skip: func(t *testing.T) { + testhelper.SkipWithRaft(t, `The hook is installed before appending log entry, before + recorder is activated. Hence, it's not feasible to differentiate between normal + entries and Raft internal entries`) + }, steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeAppendLogEntry: func(hookContext hookContext) { hookContext.closeManager() }, + BeforeAppendLogEntry: func(hookContext hookContext) { hookContext.manager.Close() }, // This ensures we are testing the context cancellation errors being unwrapped properly // to an storage.ErrTransactionProcessingStopped instead of hitting the general case when // runDone is closed. @@ -500,15 +529,17 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio ExpectedError: storage.ErrTransactionProcessingStopped, }, }, + expectedState: StateAssertion{ + Database: DatabaseState{}, + NotOffsetDatabaseInRaft: true, + }, }, { desc: "commit returns if transaction processing stops after transaction acceptance", steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeApplyLogEntry: func(hookCtx hookContext) { - panic(errSimulatedCrash) - }, + BeforeApplyLogEntry: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -526,6 +557,18 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio }, }, expectedState: StateAssertion{ + Database: testhelper.WithOrWithoutRaft( + // The process crashes before apply but after it's committed. So, only + // committedLSN is persisted. + DatabaseState{ + string(keyAppliedLSN): storage.LSN(2).ToProto(), + string(keyCommittedLSN): storage.LSN(3).ToProto(), + }, + DatabaseState{ + string(keyCommittedLSN): storage.LSN(1).ToProto(), + }, + ), + NotOffsetDatabaseInRaft: true, Directory: gittest.FilesOrReftables(testhelper.DirectoryState{ "/": {Mode: mode.Directory}, "/wal": {Mode: mode.Directory}, @@ -865,9 +908,7 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio Prune{}, StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -1132,7 +1173,8 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio }, expectedState: StateAssertion{ Database: DatabaseState{ - string(keyAppliedLSN): storage.LSN(1).ToProto(), + string(keyAppliedLSN): storage.LSN(1).ToProto(), + string(keyCommittedLSN): storage.LSN(1).ToProto(), }, Repositories: RepositoryStates{ setup.RelativePath: { @@ -1196,7 +1238,8 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio }, expectedState: StateAssertion{ Database: DatabaseState{ - string(keyAppliedLSN): storage.LSN(1).ToProto(), + string(keyAppliedLSN): storage.LSN(1).ToProto(), + string(keyCommittedLSN): storage.LSN(1).ToProto(), }, Repositories: RepositoryStates{ setup.RelativePath: { @@ -1438,13 +1481,11 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeReadAppliedLSN: func(hookContext) { - // Raise a panic when the manager is about to read the applied log - // index when initializing. In reality this would crash the server but - // in tests it serves as a way to abort the initialization in correct - // location. - panic(errSimulatedCrash) - }, + // Raise a panic when the manager is about to read the applied log + // index when initializing. In reality this would crash the server but + // in tests it serves as a way to abort the initialization in correct + // location. + BeforeReadAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -1713,13 +1754,18 @@ type expectedCommittedEntry struct { } func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []transactionTestCase { - assertCommittedEntries := func(t *testing.T, manager *TransactionManager, expected []*expectedCommittedEntry, actualList *list.List) { + assertCommittedEntries := func(t *testing.T, manager *TransactionManager, expected []*expectedCommittedEntry, actualList *list.List, offsetLSN bool) { require.Equal(t, len(expected), actualList.Len()) i := 0 for elm := actualList.Front(); elm != nil; elm = elm.Next() { actual := elm.Value.(*committedEntry) - require.Equal(t, expected[i].lsn, actual.lsn) + if testhelper.IsRaftEnabled() && offsetLSN { + recorder := manager.raftManager.EntryRecorder + require.Equal(t, recorder.Offset(expected[i].lsn), actual.lsn) + } else { + require.Equal(t, expected[i].lsn, actual.lsn) + } require.Equal(t, expected[i].snapshotReaders, actual.snapshotReaders) if expected[i].entry != nil { @@ -1745,11 +1791,17 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t } } } + if testhelper.IsRaftEnabled() { + recorder := manager.raftManager.EntryRecorder + expectedEntry.Metadata = recorder.Metadata(recorder.Offset(expected[i].lsn)) + } testhelper.ProtoEqual(t, expectedEntry, actualEntry) } i++ } + + require.Empty(t, manager.appendedEntries) } return []transactionTestCase{ @@ -1758,7 +1810,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t steps: steps{ StartManager{}, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries) + assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries, true) }), }, }, @@ -1776,7 +1828,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t lsn: 0, snapshotReaders: 1, }, - }, tm.committedEntries) + }, tm.committedEntries, true) }), Commit{ TransactionID: 1, @@ -1785,7 +1837,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t }, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries) + assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries, true) }), Begin{ TransactionID: 2, @@ -1798,7 +1850,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t lsn: 1, snapshotReaders: 1, }, - }, tm.committedEntries) + }, tm.committedEntries, true) }), Commit{ TransactionID: 2, @@ -1807,7 +1859,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t }, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries) + assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries, true) }), }, expectedState: StateAssertion{ @@ -1897,7 +1949,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t lsn: 1, snapshotReaders: 2, }, - }, tm.committedEntries) + }, tm.committedEntries, true) }), Commit{ TransactionID: 2, @@ -1915,7 +1967,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t lsn: 2, entry: refChangeLogEntry(setup, "refs/heads/branch-1", setup.Commits.First.OID), }, - }, tm.committedEntries) + }, tm.committedEntries, true) }), Begin{ TransactionID: 4, @@ -1933,7 +1985,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t snapshotReaders: 1, entry: refChangeLogEntry(setup, "refs/heads/branch-1", setup.Commits.First.OID), }, - }, tm.committedEntries) + }, tm.committedEntries, true) }), Commit{ TransactionID: 3, @@ -1952,13 +2004,13 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t lsn: 3, entry: refChangeLogEntry(setup, "refs/heads/branch-2", setup.Commits.First.OID), }, - }, tm.committedEntries) + }, tm.committedEntries, true) }), Rollback{ TransactionID: 4, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries) + assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries, true) }), }, expectedState: StateAssertion{ @@ -2042,7 +2094,15 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t TransactionID: 1, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries) + assertCommittedEntries( + t, tm, + testhelper.WithOrWithoutRaft( + []*expectedCommittedEntry{{lsn: 1}}, + []*expectedCommittedEntry{}, + ), + tm.committedEntries, + false, + ) }), Begin{ TransactionID: 2, @@ -2053,7 +2113,15 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t TransactionID: 2, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries) + assertCommittedEntries( + t, tm, + testhelper.WithOrWithoutRaft( + []*expectedCommittedEntry{{lsn: 1}}, + []*expectedCommittedEntry{}, + ), + tm.committedEntries, + false, + ) }), }, expectedState: StateAssertion{ @@ -2066,7 +2134,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t }, }, { - desc: "transaction manager cleans up left-over committed entries when appliedLSN == appendedLSN", + desc: "transaction manager cleans up left-over committed entries when appliedLSN == committedLSN", steps: steps{ StartManager{}, Begin{ @@ -2112,12 +2180,23 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t ExpectedError: storage.ErrTransactionProcessingStopped, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - RequireDatabase(t, ctx, tm.db, DatabaseState{ - string(keyAppliedLSN): storage.LSN(3).ToProto(), - }) + RequireDatabase(t, ctx, tm.db, + testhelper.WithOrWithoutRaft( + DatabaseState{ + string(keyAppliedLSN): storage.LSN(5).ToProto(), + string(keyCommittedLSN): storage.LSN(5).ToProto(), + string(raftmgr.KeyHardState): &anypb.Any{}, + string(raftmgr.KeyConfState): &anypb.Any{}, + }, + DatabaseState{ + string(keyAppliedLSN): storage.LSN(3).ToProto(), + string(keyCommittedLSN): storage.LSN(3).ToProto(), + }, + ), + ) // Transaction 2 and 3 are left-over. testhelper.RequireDirectoryState(t, tm.stateDirectory, "", - gittest.FilesOrReftables(testhelper.DirectoryState{ + gittest.FilesOrReftables(modifyDirectoryStateForRaft(t, testhelper.DirectoryState{ "/": {Mode: mode.Directory}, "/wal": {Mode: mode.Directory}, "/wal/0000000000002": {Mode: mode.Directory}, @@ -2126,7 +2205,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t "/wal/0000000000003": {Mode: mode.Directory}, "/wal/0000000000003/MANIFEST": manifestDirectoryEntry(refChangeLogEntry(setup, "refs/heads/branch-2", setup.Commits.First.OID)), "/wal/0000000000003/1": {Mode: mode.File, Content: []byte(setup.Commits.First.OID + "\n")}, - }, buildReftableDirectory(map[int][]git.ReferenceUpdates{ + }, tm), buildReftableDirectory(map[int][]git.ReferenceUpdates{ 2: {{"refs/heads/branch-1": git.ReferenceUpdate{NewOID: setup.Commits.First.OID}}}, 3: {{"refs/heads/branch-2": git.ReferenceUpdate{NewOID: setup.Commits.First.OID}}}, }))) @@ -2136,11 +2215,25 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { // When the manager finishes initialization, the left-over log entries are // cleaned up. - RequireDatabase(t, ctx, tm.db, DatabaseState{ - string(keyAppliedLSN): storage.LSN(3).ToProto(), - }) - require.Equal(t, tm.appliedLSN, storage.LSN(3)) - require.Equal(t, tm.appendedLSN, storage.LSN(3)) + RequireDatabase(t, ctx, tm.db, + testhelper.WithOrWithoutRaft( + DatabaseState{ + string(keyAppliedLSN): storage.LSN(5).ToProto(), + string(keyCommittedLSN): storage.LSN(5).ToProto(), + string(raftmgr.KeyHardState): &anypb.Any{}, + string(raftmgr.KeyConfState): &anypb.Any{}, + }, + DatabaseState{ + string(keyAppliedLSN): storage.LSN(3).ToProto(), + string(keyCommittedLSN): storage.LSN(3).ToProto(), + }, + ), + ) + + finalLSN := testhelper.WithOrWithoutRaft(storage.LSN(5), storage.LSN(3)) + require.Equal(t, tm.appliedLSN, finalLSN) + require.Equal(t, tm.committedLSN, finalLSN) + require.Equal(t, tm.appendedLSN, finalLSN) }), }, expectedState: StateAssertion{ @@ -2212,9 +2305,10 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t }, }, { - desc: "transaction manager cleans up left-over committed entries when appliedLSN < appendedLSN", + desc: "transaction manager cleans up left-over committed entries when appliedLSN < committedLSN", skip: func(t *testing.T) { testhelper.SkipWithReftable(t, "test requires manual log addition") + testhelper.SkipWithRaft(t, "test requires manual log addition without going through Raft") }, steps: steps{ StartManager{}, @@ -2261,16 +2355,24 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t ExpectedError: storage.ErrTransactionProcessingStopped, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + backupCtx := tm.ctx + tm.ctx = testhelper.Context(t) + // Insert an out-of-band log-entry directly into the database for easier test // setup. It's a bit tricky to simulate committed log entries and un-processed - // appended log entries at the same time. + // committed log entries at the same time. logEntryPath := filepath.Join(t.TempDir(), "log_entry") require.NoError(t, os.Mkdir(logEntryPath, mode.Directory)) require.NoError(t, os.WriteFile(filepath.Join(logEntryPath, "1"), []byte(setup.Commits.First.OID+"\n"), mode.File)) - require.NoError(t, tm.appendLogEntry(ctx, map[git.ObjectID]struct{}{setup.Commits.First.OID: {}}, refChangeLogEntry(setup, "refs/heads/branch-3", setup.Commits.First.OID), logEntryPath)) + require.NoError(t, tm.proposeLogEntry(ctx, map[git.ObjectID]struct{}{setup.Commits.First.OID: {}}, refChangeLogEntry(setup, "refs/heads/branch-3", setup.Commits.First.OID), logEntryPath)) + + tm.ctx = backupCtx RequireDatabase(t, ctx, tm.db, DatabaseState{ string(keyAppliedLSN): storage.LSN(3).ToProto(), + // Because this is an out-of-band insertion, the committedLSN is updated + // but new log entry is not applied until waken up. + string(keyCommittedLSN): storage.LSN(4).ToProto(), }) // Transaction 2 and 3 are left-over. testhelper.RequireDirectoryState(t, tm.stateDirectory, "", testhelper.DirectoryState{ @@ -2293,15 +2395,18 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t // When the manager finishes initialization, the left-over log entries are // cleaned up. RequireDatabase(t, ctx, tm.db, DatabaseState{ - string(keyAppliedLSN): storage.LSN(4).ToProto(), + string(keyAppliedLSN): storage.LSN(4).ToProto(), + string(keyCommittedLSN): storage.LSN(4).ToProto(), }) require.Equal(t, tm.appliedLSN, storage.LSN(4)) + require.Equal(t, tm.committedLSN, storage.LSN(4)) require.Equal(t, tm.appendedLSN, storage.LSN(4)) }), }, expectedState: StateAssertion{ Database: DatabaseState{ - string(keyAppliedLSN): storage.LSN(4).ToProto(), + string(keyAppliedLSN): storage.LSN(4).ToProto(), + string(keyCommittedLSN): storage.LSN(4).ToProto(), }, Repositories: RepositoryStates{ setup.RelativePath: { @@ -2323,6 +2428,317 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t } } +func generateAppendedEntriesTests(t *testing.T, setup testTransactionSetup) []transactionTestCase { + assertAppendedEntries := func(t *testing.T, manager *TransactionManager, expectedList []storage.LSN) { + actual := manager.appendedEntries + assert.Equalf(t, len(expectedList), len(actual), "appended entries not matched") + + for _, lsn := range expectedList { + expectedEntry, err := manager.readLogEntry(lsn) + assert.NoError(t, err) + testhelper.ProtoEqualAssert(t, expectedEntry, actual[lsn]) + } + } + + offsetIfNeeded := func(tm *TransactionManager, lsn storage.LSN) storage.LSN { + if !testhelper.IsRaftEnabled() || tm.raftManager == nil { + return lsn + } + recorder := tm.raftManager.EntryRecorder + return recorder.Offset(lsn) + } + + return []transactionTestCase{ + { + desc: "manager has just initialized", + steps: steps{ + StartManager{}, + AssertManager{}, + AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + require.Equal(t, offsetIfNeeded(tm, 0), tm.appendedLSN) + assertAppendedEntries(t, tm, nil) + }), + CloseManager{}, + }, + }, + { + desc: "appended entries are removed after committed", + skip: func(t *testing.T) { + testhelper.SkipWithRaft(t, "this test is not table if Raft inserts internal entries") + }, + steps: steps{ + StartManager{Hooks: testTransactionHooks{ + BeforeCommitLogEntry: func(c hookContext) { + // It is not the best solution. But as we intercept to assert an + // intermediate state, there is no cleaner way than using hooks. + lsn := c.lsn + manager := c.manager + switch lsn { + case storage.LSN(1): + assert.Equal(t, storage.LSN(1), manager.appendedLSN) + assert.Equal(t, storage.LSN(0), manager.committedLSN) + assertAppendedEntries(t, manager, []storage.LSN{1}) + case storage.LSN(2): + assert.Equal(t, storage.LSN(2), manager.appendedLSN) + assert.Equal(t, storage.LSN(1), manager.committedLSN) + assertAppendedEntries(t, manager, []storage.LSN{2}) + default: + assert.Fail(t, "there shouldn't be another committed entry") + } + }, + }}, + Begin{ + TransactionID: 1, + RelativePaths: []string{setup.RelativePath}, + }, + Commit{ + TransactionID: 1, + ReferenceUpdates: git.ReferenceUpdates{ + "refs/heads/branch-1": {OldOID: setup.ObjectHash.ZeroOID, NewOID: setup.Commits.First.OID}, + }, + }, + AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + require.Equal(t, offsetIfNeeded(tm, 1), tm.appendedLSN) + require.Equal(t, offsetIfNeeded(tm, 1), tm.committedLSN) + assertAppendedEntries(t, tm, nil) + }), + Begin{ + TransactionID: 2, + RelativePaths: []string{setup.RelativePath}, + ExpectedSnapshotLSN: 1, + }, + AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + require.Equal(t, offsetIfNeeded(tm, 1), tm.appendedLSN) + require.Equal(t, offsetIfNeeded(tm, 1), tm.committedLSN) + assertAppendedEntries(t, tm, nil) + }), + Commit{ + TransactionID: 2, + ReferenceUpdates: git.ReferenceUpdates{ + "refs/heads/main": {OldOID: setup.ObjectHash.ZeroOID, NewOID: setup.Commits.First.OID}, + }, + }, + AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + require.Equal(t, offsetIfNeeded(tm, 2), tm.appendedLSN) + require.Equal(t, offsetIfNeeded(tm, 2), tm.committedLSN) + assertAppendedEntries(t, tm, nil) + }), + CloseManager{}, + }, + expectedState: StateAssertion{ + Database: DatabaseState{ + string(keyAppliedLSN): storage.LSN(2).ToProto(), + }, + // The appended-but-not-committed log entry from transaction 1 is also removed. + Directory: testhelper.DirectoryState{ + "/": {Mode: mode.Directory}, + "/wal": {Mode: mode.Directory}, + }, + Repositories: RepositoryStates{ + setup.RelativePath: { + DefaultBranch: "refs/heads/main", + References: gittest.FilesOrReftables( + &ReferencesState{ + FilesBackend: &FilesBackendState{ + LooseReferences: map[git.ReferenceName]git.ObjectID{ + "refs/heads/main": setup.Commits.First.OID, + "refs/heads/branch-1": setup.Commits.First.OID, + }, + }, + }, &ReferencesState{ + ReftableBackend: &ReftableBackendState{ + Tables: []ReftableTable{ + { + MinIndex: 1, + MaxIndex: 1, + References: []git.Reference{ + { + Name: "HEAD", + Target: "refs/heads/main", + IsSymbolic: true, + }, + }, + }, + { + MinIndex: 2, + MaxIndex: 2, + References: []git.Reference{ + { + Name: "refs/heads/branch-1", + Target: setup.Commits.First.OID.String(), + }, + }, + }, + { + MinIndex: 3, + MaxIndex: 3, + References: []git.Reference{ + { + Name: "refs/heads/main", + Target: setup.Commits.First.OID.String(), + }, + }, + }, + }, + }, + }, + ), + }, + }, + }, + }, + { + desc: "transaction manager crashes after appending", + skip: func(t *testing.T) { + testhelper.SkipWithRaft(t, "this test is not table if Raft inserts internal entries") + }, + steps: steps{ + StartManager{ + Hooks: testTransactionHooks{ + BeforeCommitLogEntry: func(c hookContext) { + assert.Equal(t, storage.LSN(1), c.manager.appendedLSN) + assertAppendedEntries(t, c.manager, []storage.LSN{1}) + simulateCrashHook()(c) + }, + WaitForTransactionsWhenClosing: true, + }, + ExpectedError: errSimulatedCrash, + }, + Begin{ + TransactionID: 1, + RelativePaths: []string{setup.RelativePath}, + }, + AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + require.Equal(t, offsetIfNeeded(tm, 0), tm.appendedLSN) + require.Equal(t, offsetIfNeeded(tm, 0), tm.committedLSN) + assertAppendedEntries(t, tm, nil) + }), + Commit{ + TransactionID: 1, + ReferenceUpdates: git.ReferenceUpdates{ + "refs/heads/branch-1": {OldOID: setup.ObjectHash.ZeroOID, NewOID: setup.Commits.First.OID}, + }, + ExpectedError: storage.ErrTransactionProcessingStopped, + }, + AssertManager{ + ExpectedError: errSimulatedCrash, + }, + AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + // Appended entries are persisted. + testhelper.RequireDirectoryState(t, tm.stateDirectory, "", gittest.FilesOrReftables(modifyDirectoryStateForRaft(t, testhelper.DirectoryState{ + "/": {Mode: mode.Directory}, + "/wal": {Mode: mode.Directory}, + "/wal/0000000000001": {Mode: mode.Directory}, + "/wal/0000000000001/MANIFEST": manifestDirectoryEntry(&gitalypb.LogEntry{ + RelativePath: setup.RelativePath, + ReferenceTransactions: []*gitalypb.LogEntry_ReferenceTransaction{ + { + Changes: []*gitalypb.LogEntry_ReferenceTransaction_Change{ + { + ReferenceName: []byte("refs/heads/branch-1"), + NewOid: []byte(setup.Commits.First.OID), + }, + }, + }, + }, + Operations: []*gitalypb.LogEntry_Operation{ + { + Operation: &gitalypb.LogEntry_Operation_CreateHardLink_{ + CreateHardLink: &gitalypb.LogEntry_Operation_CreateHardLink{ + SourcePath: []byte("1"), + DestinationPath: []byte(filepath.Join(setup.RelativePath, "refs/heads/branch-1")), + }, + }, + }, + }, + }), + "/wal/0000000000001/1": {Mode: mode.File, Content: []byte(setup.Commits.First.OID + "\n")}, + }, tm), buildReftableDirectory(map[int][]git.ReferenceUpdates{ + 1: {{"refs/heads/branch-1": git.ReferenceUpdate{NewOID: setup.Commits.First.OID}}}, + }))) + }), + StartManager{}, + AssertManager{}, + // No-op, just to ensure the manager is initialized. + AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + // Both of them are set to 0 now. + require.Equal(t, offsetIfNeeded(tm, 0), tm.appendedLSN) + require.Equal(t, offsetIfNeeded(tm, 0), tm.committedLSN) + // Appended entries are removed now. + testhelper.RequireDirectoryState(t, tm.stateDirectory, "", modifyDirectoryStateForRaft(t, + testhelper.DirectoryState{ + "/": {Mode: mode.Directory}, + "/wal": {Mode: mode.Directory}, + }, + tm)) + assertAppendedEntries(t, tm, nil) + }), + Begin{ + TransactionID: 2, + RelativePaths: []string{setup.RelativePath}, + }, + Commit{ + TransactionID: 2, + ReferenceUpdates: git.ReferenceUpdates{ + "refs/heads/branch-1": {OldOID: setup.ObjectHash.ZeroOID, NewOID: setup.Commits.Second.OID}, + }, + }, + AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + // Apply new commit only. The prior commit was rejected. + require.Equal(t, offsetIfNeeded(tm, 1), tm.appendedLSN) + require.Equal(t, offsetIfNeeded(tm, 1), tm.committedLSN) + assertAppendedEntries(t, tm, nil) + }), + }, + expectedState: StateAssertion{ + Database: DatabaseState{ + string(keyAppliedLSN): storage.LSN(1).ToProto(), + }, + Repositories: RepositoryStates{ + setup.RelativePath: { + DefaultBranch: "refs/heads/main", + References: gittest.FilesOrReftables( + &ReferencesState{ + FilesBackend: &FilesBackendState{ + LooseReferences: map[git.ReferenceName]git.ObjectID{ + "refs/heads/branch-1": setup.Commits.Second.OID, + }, + }, + }, &ReferencesState{ + ReftableBackend: &ReftableBackendState{ + Tables: []ReftableTable{ + { + MinIndex: 1, + MaxIndex: 1, + References: []git.Reference{ + { + Name: "HEAD", + Target: "refs/heads/main", + IsSymbolic: true, + }, + }, + }, + { + MinIndex: 2, + MaxIndex: 2, + References: []git.Reference{ + { + Name: "refs/heads/branch-1", + Target: setup.Commits.Second.OID.String(), + }, + }, + }, + }, + }, + }, + ), + }, + }, + }, + }, + } +} + // BenchmarkTransactionManager benchmarks the transaction throughput of the TransactionManager at various levels // of concurrency and transaction sizes. func BenchmarkTransactionManager(b *testing.B) { @@ -2422,7 +2838,7 @@ func BenchmarkTransactionManager(b *testing.B) { // Valid partition IDs are >=1. testPartitionID := storage.PartitionID(i + 1) - manager := NewTransactionManager(testPartitionID, logger, database, storageName, storagePath, stateDir, stagingDir, cmdFactory, repositoryFactory, m, nil) + manager := NewTransactionManager(testPartitionID, logger, database, storageName, storagePath, stateDir, stagingDir, cmdFactory, repositoryFactory, m, nil, nil) managers = append(managers, manager) diff --git a/internal/gitaly/storage/wal.go b/internal/gitaly/storage/wal.go new file mode 100644 index 0000000000000000000000000000000000000000..b74b494af9bc4cb8f40b3cac774e3001f6aa907c --- /dev/null +++ b/internal/gitaly/storage/wal.go @@ -0,0 +1,42 @@ +package storage + +import ( + "context" + + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" +) + +// WriteAheadLog is an abstract interface for Gitaly's WAL. +// It defines methods to manage log entries in a write-ahead log system for use with Raft consensus. +type WriteAheadLog interface { + // AppendLogEntry appends a new log entry at the specified LSN (Log Sequence Number), + // using the provided log entry data and file path, ensuring the entry is durably written. + AppendLogEntry(ctx context.Context, lsn LSN, entry *gitalypb.LogEntry, filePath string) error + + // CommitLogEntry marks the log entry at the given LSN as committed, which means it has been accepted + // by a majority of Raft members and can be safely applied to the state machine. + CommitLogEntry(ctx context.Context, lsn LSN) error + + // NotifyNewCommittedEntry notifies WAL that a log entry has been committed, allowing them to proceed with + // processing. The caller should only notify new commits if WAL is idle or the new committed entries are + // issued by Raft itself. Otherwise, WAL is waken up too frequently. + NotifyNewCommittedEntry() + + // InsertLogEntry inserts a KV log entry into the WAL at the specified LSN. The caller can manipulate the data + // by passing a modification function. It's responsibility of the caller NOT TO append/insert any log entries + // while this function is running. All adjacent log entries at and after inserting position are wiped. + InsertLogEntry(ctx context.Context, lsn LSN, txnFunc func(keyvalue.ReadWriter) error, metadata *gitalypb.LogEntry_Metadata) (*gitalypb.LogEntry, error) + + // ReadLogEntry reads and returns the log entry stored at the specified LSN within the WAL. + ReadLogEntry(lsn LSN) (*gitalypb.LogEntry, error) + + // LastLSN returns the last LSN that has been appended to the WAL, indicating the latest position. + LastLSN() LSN + + // FirstLSN retrieves the first LSN available in the WAL, useful for identifying the oldest log entry retained. + FirstLSN() LSN + + // LSNDirPath provides the directory path where files corresponding to a specific LSN are stored. + LSNDirPath(lsn LSN) string +} diff --git a/internal/testhelper/logger.go b/internal/testhelper/logger.go index eb6dc86685a47ec590516543761cfa1a55d1ffa2..6fdced6c75806ea82e71df3e881b59dfde9874ad 100644 --- a/internal/testhelper/logger.go +++ b/internal/testhelper/logger.go @@ -85,7 +85,8 @@ func (b *syncBuffer) String() string { } type loggerOptions struct { - name string + name string + level logrus.Level } // LoggerOption configures a logger. @@ -99,16 +100,24 @@ func WithLoggerName(name string) LoggerOption { } } +// WithLevel sets the level of the logger. It's useful when we would like +// to capture debug logs. +func WithLevel(level logrus.Level) LoggerOption { + return func(opts *loggerOptions) { + opts.level = level + } +} + // NewLogger returns a logger that records the log output and // prints it out only if the test fails. func NewLogger(tb testing.TB, options ...LoggerOption) log.LogrusLogger { - logger, logOutput := NewCapturedLogger() - var opts loggerOptions for _, apply := range options { apply(&opts) } + logger, logOutput := NewCapturedLogger(opts) + tb.Cleanup(func() { if !tb.Failed() || logOutput.Len() == 0 { return @@ -126,10 +135,13 @@ func NewLogger(tb testing.TB, options ...LoggerOption) log.LogrusLogger { // NewCapturedLogger returns a logger that records the log outputs in a buffer. The caller // decides how and when the logs are dumped out. -func NewCapturedLogger() (log.LogrusLogger, *syncBuffer) { +func NewCapturedLogger(opts loggerOptions) (log.LogrusLogger, *syncBuffer) { logOutput := &syncBuffer{} logger := logrus.New() //nolint:forbidigo logger.Out = logOutput + if opts.level != 0 { + logger.SetLevel(opts.level) + } return log.FromLogrusEntry(logrus.NewEntry(logger)), logOutput } diff --git a/internal/testhelper/testcfg/gitaly.go b/internal/testhelper/testcfg/gitaly.go index 726092838afe7827659953c276b92f5ad8fd214c..d11d478518b3bd81a4fcae863cd8cba135f83561 100644 --- a/internal/testhelper/testcfg/gitaly.go +++ b/internal/testhelper/testcfg/gitaly.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/google/uuid" "github.com/pelletier/go-toml/v2" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" @@ -172,6 +173,10 @@ func (gc *GitalyCfgBuilder) Build(tb testing.TB) config.Cfg { } cfg.Transactions.Enabled = testhelper.IsWALEnabled() + if testhelper.IsRaftEnabled() { + cfg.Transactions.Enabled = true + cfg.Raft = config.DefaultRaftConfig(uuid.New().String()) + } // We force the use of bundled (embedded) binaries unless we're specifically executing tests // against a custom version of Git. diff --git a/internal/testhelper/testhelper.go b/internal/testhelper/testhelper.go index 452fe9843e0ef3f52f40b3b2d705d46e75e0fdcc..57f74b68961209e2814539b5afb8d16a23dbfc9e 100644 --- a/internal/testhelper/testhelper.go +++ b/internal/testhelper/testhelper.go @@ -57,6 +57,30 @@ func IsWALEnabled() bool { return ok } +// IsRaftEnabled returns whether Raft single cluster is enabled in this testing run. +func IsRaftEnabled() bool { + _, ok := os.LookupEnv("GITALY_TEST_RAFT") + if ok && !IsWALEnabled() { + panic("GITALY_TEST_WAL must be enabled") + } + return ok +} + +// WithOrWithoutRaft returns a value correspondingly to if Raft is enabled or not. +func WithOrWithoutRaft[T any](raftVal, noRaftVal T) T { + if IsRaftEnabled() { + return raftVal + } + return noRaftVal +} + +// SkipWithRaft skips the test if Raft is enabled in this testing run. +func SkipWithRaft(tb testing.TB, reason string) { + if IsRaftEnabled() { + tb.Skip(reason) + } +} + // SkipWithWAL skips the test if write-ahead logging is enabled in this testing run. A reason // should be provided either as a description or a link to an issue to explain why the test is // skipped. diff --git a/internal/testhelper/testserver/gitaly.go b/internal/testhelper/testserver/gitaly.go index 5a582d14111425c77f9eb120b0dd604af4e414d6..3ee0773f94603667537aa078c52007015f36dacf 100644 --- a/internal/testhelper/testserver/gitaly.go +++ b/internal/testhelper/testserver/gitaly.go @@ -6,6 +6,7 @@ import ( "os" "testing" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" gitalyauth "gitlab.com/gitlab-org/gitaly/v16/auth" @@ -30,6 +31,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue/databasemgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/mode" nodeimpl "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/node" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/partition" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/snapshot" @@ -348,6 +350,13 @@ func (gsd *gitalyServerDeps) createDependencies(tb testing.TB, ctx context.Conte require.NoError(tb, err) tb.Cleanup(dbMgr.Close) + var raftManagerFactory raftmgr.RaftManagerFactory + if testhelper.IsRaftEnabled() { + cfg.Raft = config.DefaultRaftConfig(uuid.New().String()) + raftManagerFactory = func(ptnID storage.PartitionID, storageName string, db keyvalue.Transactioner, logger log.Logger) (*raftmgr.Manager, error) { + return raftmgr.NewManager(ptnID, storageName, cfg.Raft, db, logger) + } + } nodeMgr, err := nodeimpl.NewManager( cfg.Storages, storagemgr.NewFactory( @@ -361,6 +370,7 @@ func (gsd *gitalyServerDeps) createDependencies(tb testing.TB, ctx context.Conte snapshot.NewMetrics(), ), nil, + raftManagerFactory, ), storagemgr.DefaultMaxInactivePartitions, storagemgr.NewMetrics(cfg.Prometheus), diff --git a/proto/cluster.proto b/proto/cluster.proto new file mode 100644 index 0000000000000000000000000000000000000000..6e39ac1015d2889bd4c016f1055c7cc6867000f8 --- /dev/null +++ b/proto/cluster.proto @@ -0,0 +1,107 @@ +syntax = "proto3"; + +package gitaly; +import "log.proto"; + +option go_package = "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"; + +// RaftMessageType defines the types of messages that can be used within the Raft protocol. +// These types help in identifying the nature of the message being processed. +enum RaftMessageType { + // UNSPECIFIED is used to indicate unspecified message type enum. + UNSPECIFIED = 0; // protolint:disable:this ENUM_FIELD_NAMES_PREFIX + // NORMAL represents a standard Raft log entry proposed by the application. + NORMAL = 1; // protolint:disable:this ENUM_FIELD_NAMES_PREFIX + // VERIFICATION refers to a special type of entry used for verifying state + // without altering it, ensuring the integrity of operations and logs. + VERIFICATION = 2; // protolint:disable:this ENUM_FIELD_NAMES_PREFIX + // CONFIG_CHANGE signifies a change in the configuration of the Raft cluster, + // typically involving node additions or removals. + CONFIG_CHANGE = 3; // protolint:disable:this ENUM_FIELD_NAMES_PREFIX +} + +// RaftMessageV1 serves as a wrapper for messages exchanged in the Raft protocol, +// encapsulating essential information such as the log entry and related metadata. +message RaftMessageV1 { + // ReferencedLogData holds a reference path to the log data stored externally, + // which can be used to access large log entries without including them directly. + message ReferencedLogData { + // path represents the external storage location of the log data. + bytes path = 1; + } + // PackedLogData contains serialized log data including log entry itself and + // all attached files in the log entry directory. Those data are exchanged at + // the Transport layer before sending after after receiving messages. Hence, + // they are transparent to the core Raft engine. + message PackedLogData { + // data is serialized form of the log entry data. Transport implementations + // can choose to populate this data or read the data directly on disk. The + // latter approach is recommended. + bytes data = 1; + } + + // id is unique identifier for the Raft message. This ID is generated by an + // in-memory revent registry. Raft uses this ID to notify the committment + // status of a log entry. + uint64 id = 1; + + // cluster_id is the identifier of the Raft cluster to which this message belongs. + string cluster_id = 2; + + // authority_name is the storage name of storage that creates a partition. + string authority_name = 3; + + // partition_id is the local incremental ID of the specific partition within a + // storage. (authority_name, partition_id) can be used as a unique identifier + // of a partition across the cluster. + uint64 partition_id = 4; + + // log_entry is the actual log entry being processed or transmitted. + LogEntry log_entry = 5; + + // log_data holds files inside log entry dir in one of two possible forms: + // referenced or packed. + oneof log_data { + // referenced represents reference to on-disk log data. + ReferencedLogData referenced = 6; + + // packed represents packed and serialized log data. + PackedLogData packed = 7; + } +} + +// RaftHardStateV1 is a wrapper for raftpb.HardState. The upstream uses proto2 +// syntax while Gitaly uses proto3. In addition, the protobuf package in +// upstream is outdated. The generated structs are not compatible with Gitaly's +// protobuf utilities. +// Source: +// https://github.com/etcd-io/raft/blob/12f0e5dc1b5bfff9bc6886ef1be4cba19495d6f2/raftpb/raft.proto#L110-114 +message RaftHardStateV1 { + // term represents the current term of the raft group. + uint64 term = 1; + // vote represents the vote of the raft group. + uint64 vote = 2; + // commit represents the latest commit index of the raft group. + uint64 commit = 3; +} + +// RaftConfStateV1 is a wrapper for raftpb.ConfState. For more information, +// please refer to RaftHardStateV1. Source: +// https://github.com/etcd-io/raft/blob/12f0e5dc1b5bfff9bc6886ef1be4cba19495d6f2/raftpb/raft.proto#L136 +message RaftConfStateV1 { + // voters in the incoming config. (If the configuration is not joint, + // then the outgoing config is empty). + repeated uint64 voters = 1; + // learners in the incoming config. + repeated uint64 learners = 2; + // voters_outgoing in the outgoing config. + repeated uint64 voters_outgoing = 3; // protolint:disable:this REPEATED_FIELD_NAMES_PLURALIZED + // learners_next is the nodes that will become learners when the outgoing + // config is removed. These nodes are necessarily currently in nodes_joint (or + // they would have been added to the incoming config right away). + repeated uint64 learners_next = 4; // protolint:disable:this REPEATED_FIELD_NAMES_PLURALIZED + // auto_leave is set when the config is joint and Raft will automatically + // transition into the final config (i.e. remove the outgoing config) when + // this is safe. + bool auto_leave = 5; +} diff --git a/proto/go/gitalypb/cluster.pb.go b/proto/go/gitalypb/cluster.pb.go new file mode 100644 index 0000000000000000000000000000000000000000..c77541b260fc0d3a7f878f86f27238fd69f1031e --- /dev/null +++ b/proto/go/gitalypb/cluster.pb.go @@ -0,0 +1,600 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.35.1 +// protoc v4.23.1 +// source: cluster.proto + +package gitalypb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// RaftMessageType defines the types of messages that can be used within the Raft protocol. +// These types help in identifying the nature of the message being processed. +type RaftMessageType int32 + +const ( + // UNSPECIFIED is used to indicate unspecified message type enum. + RaftMessageType_UNSPECIFIED RaftMessageType = 0 // protolint:disable:this ENUM_FIELD_NAMES_PREFIX + // NORMAL represents a standard Raft log entry proposed by the application. + RaftMessageType_NORMAL RaftMessageType = 1 // protolint:disable:this ENUM_FIELD_NAMES_PREFIX + // VERIFICATION refers to a special type of entry used for verifying state + // without altering it, ensuring the integrity of operations and logs. + RaftMessageType_VERIFICATION RaftMessageType = 2 // protolint:disable:this ENUM_FIELD_NAMES_PREFIX + // CONFIG_CHANGE signifies a change in the configuration of the Raft cluster, + // typically involving node additions or removals. + RaftMessageType_CONFIG_CHANGE RaftMessageType = 3 // protolint:disable:this ENUM_FIELD_NAMES_PREFIX +) + +// Enum value maps for RaftMessageType. +var ( + RaftMessageType_name = map[int32]string{ + 0: "UNSPECIFIED", + 1: "NORMAL", + 2: "VERIFICATION", + 3: "CONFIG_CHANGE", + } + RaftMessageType_value = map[string]int32{ + "UNSPECIFIED": 0, + "NORMAL": 1, + "VERIFICATION": 2, + "CONFIG_CHANGE": 3, + } +) + +func (x RaftMessageType) Enum() *RaftMessageType { + p := new(RaftMessageType) + *p = x + return p +} + +func (x RaftMessageType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (RaftMessageType) Descriptor() protoreflect.EnumDescriptor { + return file_cluster_proto_enumTypes[0].Descriptor() +} + +func (RaftMessageType) Type() protoreflect.EnumType { + return &file_cluster_proto_enumTypes[0] +} + +func (x RaftMessageType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use RaftMessageType.Descriptor instead. +func (RaftMessageType) EnumDescriptor() ([]byte, []int) { + return file_cluster_proto_rawDescGZIP(), []int{0} +} + +// RaftMessageV1 serves as a wrapper for messages exchanged in the Raft protocol, +// encapsulating essential information such as the log entry and related metadata. +type RaftMessageV1 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // id is unique identifier for the Raft message. This ID is generated by an + // in-memory revent registry. Raft uses this ID to notify the committment + // status of a log entry. + Id uint64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + // cluster_id is the identifier of the Raft cluster to which this message belongs. + ClusterId string `protobuf:"bytes,2,opt,name=cluster_id,json=clusterId,proto3" json:"cluster_id,omitempty"` + // authority_name is the storage name of storage that creates a partition. + AuthorityName string `protobuf:"bytes,3,opt,name=authority_name,json=authorityName,proto3" json:"authority_name,omitempty"` + // partition_id is the local incremental ID of the specific partition within a + // storage. (authority_name, partition_id) can be used as a unique identifier + // of a partition across the cluster. + PartitionId uint64 `protobuf:"varint,4,opt,name=partition_id,json=partitionId,proto3" json:"partition_id,omitempty"` + // log_entry is the actual log entry being processed or transmitted. + LogEntry *LogEntry `protobuf:"bytes,5,opt,name=log_entry,json=logEntry,proto3" json:"log_entry,omitempty"` + // log_data holds files inside log entry dir in one of two possible forms: + // referenced or packed. + // + // Types that are assignable to LogData: + // + // *RaftMessageV1_Referenced + // *RaftMessageV1_Packed + LogData isRaftMessageV1_LogData `protobuf_oneof:"log_data"` +} + +func (x *RaftMessageV1) Reset() { + *x = RaftMessageV1{} + mi := &file_cluster_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RaftMessageV1) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaftMessageV1) ProtoMessage() {} + +func (x *RaftMessageV1) ProtoReflect() protoreflect.Message { + mi := &file_cluster_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaftMessageV1.ProtoReflect.Descriptor instead. +func (*RaftMessageV1) Descriptor() ([]byte, []int) { + return file_cluster_proto_rawDescGZIP(), []int{0} +} + +func (x *RaftMessageV1) GetId() uint64 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *RaftMessageV1) GetClusterId() string { + if x != nil { + return x.ClusterId + } + return "" +} + +func (x *RaftMessageV1) GetAuthorityName() string { + if x != nil { + return x.AuthorityName + } + return "" +} + +func (x *RaftMessageV1) GetPartitionId() uint64 { + if x != nil { + return x.PartitionId + } + return 0 +} + +func (x *RaftMessageV1) GetLogEntry() *LogEntry { + if x != nil { + return x.LogEntry + } + return nil +} + +func (m *RaftMessageV1) GetLogData() isRaftMessageV1_LogData { + if m != nil { + return m.LogData + } + return nil +} + +func (x *RaftMessageV1) GetReferenced() *RaftMessageV1_ReferencedLogData { + if x, ok := x.GetLogData().(*RaftMessageV1_Referenced); ok { + return x.Referenced + } + return nil +} + +func (x *RaftMessageV1) GetPacked() *RaftMessageV1_PackedLogData { + if x, ok := x.GetLogData().(*RaftMessageV1_Packed); ok { + return x.Packed + } + return nil +} + +type isRaftMessageV1_LogData interface { + isRaftMessageV1_LogData() +} + +type RaftMessageV1_Referenced struct { + // referenced represents reference to on-disk log data. + Referenced *RaftMessageV1_ReferencedLogData `protobuf:"bytes,6,opt,name=referenced,proto3,oneof"` +} + +type RaftMessageV1_Packed struct { + // packed represents packed and serialized log data. + Packed *RaftMessageV1_PackedLogData `protobuf:"bytes,7,opt,name=packed,proto3,oneof"` +} + +func (*RaftMessageV1_Referenced) isRaftMessageV1_LogData() {} + +func (*RaftMessageV1_Packed) isRaftMessageV1_LogData() {} + +// RaftHardStateV1 is a wrapper for raftpb.HardState. The upstream uses proto2 +// syntax while Gitaly uses proto3. In addition, the protobuf package in +// upstream is outdated. The generated structs are not compatible with Gitaly's +// protobuf utilities. +// Source: +// https://github.com/etcd-io/raft/blob/12f0e5dc1b5bfff9bc6886ef1be4cba19495d6f2/raftpb/raft.proto#L110-114 +type RaftHardStateV1 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // term represents the current term of the raft group. + Term uint64 `protobuf:"varint,1,opt,name=term,proto3" json:"term,omitempty"` + // vote represents the vote of the raft group. + Vote uint64 `protobuf:"varint,2,opt,name=vote,proto3" json:"vote,omitempty"` + // commit represents the latest commit index of the raft group. + Commit uint64 `protobuf:"varint,3,opt,name=commit,proto3" json:"commit,omitempty"` +} + +func (x *RaftHardStateV1) Reset() { + *x = RaftHardStateV1{} + mi := &file_cluster_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RaftHardStateV1) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaftHardStateV1) ProtoMessage() {} + +func (x *RaftHardStateV1) ProtoReflect() protoreflect.Message { + mi := &file_cluster_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaftHardStateV1.ProtoReflect.Descriptor instead. +func (*RaftHardStateV1) Descriptor() ([]byte, []int) { + return file_cluster_proto_rawDescGZIP(), []int{1} +} + +func (x *RaftHardStateV1) GetTerm() uint64 { + if x != nil { + return x.Term + } + return 0 +} + +func (x *RaftHardStateV1) GetVote() uint64 { + if x != nil { + return x.Vote + } + return 0 +} + +func (x *RaftHardStateV1) GetCommit() uint64 { + if x != nil { + return x.Commit + } + return 0 +} + +// RaftConfStateV1 is a wrapper for raftpb.ConfState. For more information, +// please refer to RaftHardStateV1. Source: +// https://github.com/etcd-io/raft/blob/12f0e5dc1b5bfff9bc6886ef1be4cba19495d6f2/raftpb/raft.proto#L136 +type RaftConfStateV1 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // voters in the incoming config. (If the configuration is not joint, + // then the outgoing config is empty). + Voters []uint64 `protobuf:"varint,1,rep,packed,name=voters,proto3" json:"voters,omitempty"` + // learners in the incoming config. + Learners []uint64 `protobuf:"varint,2,rep,packed,name=learners,proto3" json:"learners,omitempty"` + // voters_outgoing in the outgoing config. + VotersOutgoing []uint64 `protobuf:"varint,3,rep,packed,name=voters_outgoing,json=votersOutgoing,proto3" json:"voters_outgoing,omitempty"` // protolint:disable:this REPEATED_FIELD_NAMES_PLURALIZED + // learners_next is the nodes that will become learners when the outgoing + // config is removed. These nodes are necessarily currently in nodes_joint (or + // they would have been added to the incoming config right away). + LearnersNext []uint64 `protobuf:"varint,4,rep,packed,name=learners_next,json=learnersNext,proto3" json:"learners_next,omitempty"` // protolint:disable:this REPEATED_FIELD_NAMES_PLURALIZED + // auto_leave is set when the config is joint and Raft will automatically + // transition into the final config (i.e. remove the outgoing config) when + // this is safe. + AutoLeave bool `protobuf:"varint,5,opt,name=auto_leave,json=autoLeave,proto3" json:"auto_leave,omitempty"` +} + +func (x *RaftConfStateV1) Reset() { + *x = RaftConfStateV1{} + mi := &file_cluster_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RaftConfStateV1) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaftConfStateV1) ProtoMessage() {} + +func (x *RaftConfStateV1) ProtoReflect() protoreflect.Message { + mi := &file_cluster_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaftConfStateV1.ProtoReflect.Descriptor instead. +func (*RaftConfStateV1) Descriptor() ([]byte, []int) { + return file_cluster_proto_rawDescGZIP(), []int{2} +} + +func (x *RaftConfStateV1) GetVoters() []uint64 { + if x != nil { + return x.Voters + } + return nil +} + +func (x *RaftConfStateV1) GetLearners() []uint64 { + if x != nil { + return x.Learners + } + return nil +} + +func (x *RaftConfStateV1) GetVotersOutgoing() []uint64 { + if x != nil { + return x.VotersOutgoing + } + return nil +} + +func (x *RaftConfStateV1) GetLearnersNext() []uint64 { + if x != nil { + return x.LearnersNext + } + return nil +} + +func (x *RaftConfStateV1) GetAutoLeave() bool { + if x != nil { + return x.AutoLeave + } + return false +} + +// ReferencedLogData holds a reference path to the log data stored externally, +// which can be used to access large log entries without including them directly. +type RaftMessageV1_ReferencedLogData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // path represents the external storage location of the log data. + Path []byte `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` +} + +func (x *RaftMessageV1_ReferencedLogData) Reset() { + *x = RaftMessageV1_ReferencedLogData{} + mi := &file_cluster_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RaftMessageV1_ReferencedLogData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaftMessageV1_ReferencedLogData) ProtoMessage() {} + +func (x *RaftMessageV1_ReferencedLogData) ProtoReflect() protoreflect.Message { + mi := &file_cluster_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaftMessageV1_ReferencedLogData.ProtoReflect.Descriptor instead. +func (*RaftMessageV1_ReferencedLogData) Descriptor() ([]byte, []int) { + return file_cluster_proto_rawDescGZIP(), []int{0, 0} +} + +func (x *RaftMessageV1_ReferencedLogData) GetPath() []byte { + if x != nil { + return x.Path + } + return nil +} + +// PackedLogData contains serialized log data including log entry itself and +// all attached files in the log entry directory. Those data are exchanged at +// the Transport layer before sending after after receiving messages. Hence, +// they are transparent to the core Raft engine. +type RaftMessageV1_PackedLogData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // data is serialized form of the log entry data. Transport implementations + // can choose to populate this data or read the data directly on disk. The + // latter approach is recommended. + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` +} + +func (x *RaftMessageV1_PackedLogData) Reset() { + *x = RaftMessageV1_PackedLogData{} + mi := &file_cluster_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RaftMessageV1_PackedLogData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaftMessageV1_PackedLogData) ProtoMessage() {} + +func (x *RaftMessageV1_PackedLogData) ProtoReflect() protoreflect.Message { + mi := &file_cluster_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaftMessageV1_PackedLogData.ProtoReflect.Descriptor instead. +func (*RaftMessageV1_PackedLogData) Descriptor() ([]byte, []int) { + return file_cluster_proto_rawDescGZIP(), []int{0, 1} +} + +func (x *RaftMessageV1_PackedLogData) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +var File_cluster_proto protoreflect.FileDescriptor + +var file_cluster_proto_rawDesc = []byte{ + 0x0a, 0x0d, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x06, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x1a, 0x09, 0x6c, 0x6f, 0x67, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x22, 0x9b, 0x03, 0x0a, 0x0d, 0x52, 0x61, 0x66, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x56, 0x31, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x61, 0x75, 0x74, + 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61, + 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x2d, 0x0a, + 0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x10, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x49, 0x0a, 0x0a, + 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x27, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x56, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x64, 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, 0x61, 0x48, 0x00, 0x52, 0x0a, 0x72, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x64, 0x12, 0x3d, 0x0a, 0x06, 0x70, 0x61, 0x63, 0x6b, 0x65, + 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, + 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x56, 0x31, 0x2e, 0x50, + 0x61, 0x63, 0x6b, 0x65, 0x64, 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, 0x61, 0x48, 0x00, 0x52, 0x06, + 0x70, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x1a, 0x27, 0x0a, 0x11, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, + 0x6e, 0x63, 0x65, 0x64, 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x70, + 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x1a, + 0x23, 0x0a, 0x0d, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, 0x61, + 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, + 0x64, 0x61, 0x74, 0x61, 0x42, 0x0a, 0x0a, 0x08, 0x6c, 0x6f, 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, + 0x22, 0x51, 0x0a, 0x0f, 0x52, 0x61, 0x66, 0x74, 0x48, 0x61, 0x72, 0x64, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x56, 0x31, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x65, 0x72, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x04, 0x74, 0x65, 0x72, 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x76, 0x6f, 0x74, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x76, 0x6f, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, + 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x63, 0x6f, 0x6d, + 0x6d, 0x69, 0x74, 0x22, 0xb2, 0x01, 0x0a, 0x0f, 0x52, 0x61, 0x66, 0x74, 0x43, 0x6f, 0x6e, 0x66, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x56, 0x31, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x6f, 0x74, 0x65, 0x72, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x04, 0x52, 0x06, 0x76, 0x6f, 0x74, 0x65, 0x72, 0x73, 0x12, + 0x1a, 0x0a, 0x08, 0x6c, 0x65, 0x61, 0x72, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x04, 0x52, 0x08, 0x6c, 0x65, 0x61, 0x72, 0x6e, 0x65, 0x72, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x76, + 0x6f, 0x74, 0x65, 0x72, 0x73, 0x5f, 0x6f, 0x75, 0x74, 0x67, 0x6f, 0x69, 0x6e, 0x67, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x04, 0x52, 0x0e, 0x76, 0x6f, 0x74, 0x65, 0x72, 0x73, 0x4f, 0x75, 0x74, 0x67, + 0x6f, 0x69, 0x6e, 0x67, 0x12, 0x23, 0x0a, 0x0d, 0x6c, 0x65, 0x61, 0x72, 0x6e, 0x65, 0x72, 0x73, + 0x5f, 0x6e, 0x65, 0x78, 0x74, 0x18, 0x04, 0x20, 0x03, 0x28, 0x04, 0x52, 0x0c, 0x6c, 0x65, 0x61, + 0x72, 0x6e, 0x65, 0x72, 0x73, 0x4e, 0x65, 0x78, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x75, 0x74, + 0x6f, 0x5f, 0x6c, 0x65, 0x61, 0x76, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, + 0x75, 0x74, 0x6f, 0x4c, 0x65, 0x61, 0x76, 0x65, 0x2a, 0x53, 0x0a, 0x0f, 0x52, 0x61, 0x66, 0x74, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x55, + 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, + 0x4e, 0x4f, 0x52, 0x4d, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x56, 0x45, 0x52, 0x49, + 0x46, 0x49, 0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x43, 0x4f, + 0x4e, 0x46, 0x49, 0x47, 0x5f, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x10, 0x03, 0x42, 0x34, 0x5a, + 0x32, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, + 0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, + 0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, + 0x79, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_cluster_proto_rawDescOnce sync.Once + file_cluster_proto_rawDescData = file_cluster_proto_rawDesc +) + +func file_cluster_proto_rawDescGZIP() []byte { + file_cluster_proto_rawDescOnce.Do(func() { + file_cluster_proto_rawDescData = protoimpl.X.CompressGZIP(file_cluster_proto_rawDescData) + }) + return file_cluster_proto_rawDescData +} + +var file_cluster_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_cluster_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_cluster_proto_goTypes = []any{ + (RaftMessageType)(0), // 0: gitaly.RaftMessageType + (*RaftMessageV1)(nil), // 1: gitaly.RaftMessageV1 + (*RaftHardStateV1)(nil), // 2: gitaly.RaftHardStateV1 + (*RaftConfStateV1)(nil), // 3: gitaly.RaftConfStateV1 + (*RaftMessageV1_ReferencedLogData)(nil), // 4: gitaly.RaftMessageV1.ReferencedLogData + (*RaftMessageV1_PackedLogData)(nil), // 5: gitaly.RaftMessageV1.PackedLogData + (*LogEntry)(nil), // 6: gitaly.LogEntry +} +var file_cluster_proto_depIdxs = []int32{ + 6, // 0: gitaly.RaftMessageV1.log_entry:type_name -> gitaly.LogEntry + 4, // 1: gitaly.RaftMessageV1.referenced:type_name -> gitaly.RaftMessageV1.ReferencedLogData + 5, // 2: gitaly.RaftMessageV1.packed:type_name -> gitaly.RaftMessageV1.PackedLogData + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_cluster_proto_init() } +func file_cluster_proto_init() { + if File_cluster_proto != nil { + return + } + file_log_proto_init() + file_cluster_proto_msgTypes[0].OneofWrappers = []any{ + (*RaftMessageV1_Referenced)(nil), + (*RaftMessageV1_Packed)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_cluster_proto_rawDesc, + NumEnums: 1, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_cluster_proto_goTypes, + DependencyIndexes: file_cluster_proto_depIdxs, + EnumInfos: file_cluster_proto_enumTypes, + MessageInfos: file_cluster_proto_msgTypes, + }.Build() + File_cluster_proto = out.File + file_cluster_proto_rawDesc = nil + file_cluster_proto_goTypes = nil + file_cluster_proto_depIdxs = nil +} diff --git a/proto/go/gitalypb/log.pb.go b/proto/go/gitalypb/log.pb.go index 01f99c4e6abed4c1ba7118987991280e253fffcf..5acd783dafbfffe8bdc8867fc7ddc8be16a0cdff 100644 --- a/proto/go/gitalypb/log.pb.go +++ b/proto/go/gitalypb/log.pb.go @@ -44,6 +44,8 @@ type LogEntry struct { // operations is an ordered list of operations to run in order to apply // this log entry. Operations []*LogEntry_Operation `protobuf:"bytes,10,rep,name=operations,proto3" json:"operations,omitempty"` + // metadata contains some extra information, mostly Raft-related metadata. + Metadata *LogEntry_Metadata `protobuf:"bytes,11,opt,name=metadata,proto3" json:"metadata,omitempty"` } func (x *LogEntry) Reset() { @@ -111,6 +113,13 @@ func (x *LogEntry) GetOperations() []*LogEntry_Operation { return nil } +func (x *LogEntry) GetMetadata() *LogEntry_Metadata { + if x != nil { + return x.Metadata + } + return nil +} + // LSN serializes a log sequence number. It's used for storing a partition's // applied LSN in the database. // @@ -440,6 +449,64 @@ func (*LogEntry_Operation_SetKey_) isLogEntry_Operation_Operation() {} func (*LogEntry_Operation_DeleteKey_) isLogEntry_Operation_Operation() {} +// Metadata contains some metadata data of the log entry. +type LogEntry_Metadata struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // raft_term is the term sequence of the Raft group finalized this + // log entry. + RaftTerm uint64 `protobuf:"varint,1,opt,name=raft_term,json=raftTerm,proto3" json:"raft_term,omitempty"` + // raft_type is the type of sequence. It's essentially equal to + // raftpb.EntryType. + RaftType uint64 `protobuf:"varint,2,opt,name=raft_type,json=raftType,proto3" json:"raft_type,omitempty"` +} + +func (x *LogEntry_Metadata) Reset() { + *x = LogEntry_Metadata{} + mi := &file_log_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LogEntry_Metadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LogEntry_Metadata) ProtoMessage() {} + +func (x *LogEntry_Metadata) ProtoReflect() protoreflect.Message { + mi := &file_log_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LogEntry_Metadata.ProtoReflect.Descriptor instead. +func (*LogEntry_Metadata) Descriptor() ([]byte, []int) { + return file_log_proto_rawDescGZIP(), []int{0, 4} +} + +func (x *LogEntry_Metadata) GetRaftTerm() uint64 { + if x != nil { + return x.RaftTerm + } + return 0 +} + +func (x *LogEntry_Metadata) GetRaftType() uint64 { + if x != nil { + return x.RaftType + } + return 0 +} + // Change models a single reference change. type LogEntry_ReferenceTransaction_Change struct { state protoimpl.MessageState @@ -461,7 +528,7 @@ type LogEntry_ReferenceTransaction_Change struct { func (x *LogEntry_ReferenceTransaction_Change) Reset() { *x = LogEntry_ReferenceTransaction_Change{} - mi := &file_log_proto_msgTypes[6] + mi := &file_log_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -473,7 +540,7 @@ func (x *LogEntry_ReferenceTransaction_Change) String() string { func (*LogEntry_ReferenceTransaction_Change) ProtoMessage() {} func (x *LogEntry_ReferenceTransaction_Change) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[6] + mi := &file_log_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -524,7 +591,7 @@ type LogEntry_Housekeeping_PackRefs struct { func (x *LogEntry_Housekeeping_PackRefs) Reset() { *x = LogEntry_Housekeeping_PackRefs{} - mi := &file_log_proto_msgTypes[7] + mi := &file_log_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -536,7 +603,7 @@ func (x *LogEntry_Housekeeping_PackRefs) String() string { func (*LogEntry_Housekeeping_PackRefs) ProtoMessage() {} func (x *LogEntry_Housekeeping_PackRefs) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[7] + mi := &file_log_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -578,7 +645,7 @@ type LogEntry_Housekeeping_Repack struct { func (x *LogEntry_Housekeeping_Repack) Reset() { *x = LogEntry_Housekeeping_Repack{} - mi := &file_log_proto_msgTypes[8] + mi := &file_log_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -590,7 +657,7 @@ func (x *LogEntry_Housekeeping_Repack) String() string { func (*LogEntry_Housekeeping_Repack) ProtoMessage() {} func (x *LogEntry_Housekeeping_Repack) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[8] + mi := &file_log_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -637,7 +704,7 @@ type LogEntry_Housekeeping_WriteCommitGraphs struct { func (x *LogEntry_Housekeeping_WriteCommitGraphs) Reset() { *x = LogEntry_Housekeeping_WriteCommitGraphs{} - mi := &file_log_proto_msgTypes[9] + mi := &file_log_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -649,7 +716,7 @@ func (x *LogEntry_Housekeeping_WriteCommitGraphs) String() string { func (*LogEntry_Housekeeping_WriteCommitGraphs) ProtoMessage() {} func (x *LogEntry_Housekeeping_WriteCommitGraphs) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[9] + mi := &file_log_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -684,7 +751,7 @@ type LogEntry_Operation_CreateHardLink struct { func (x *LogEntry_Operation_CreateHardLink) Reset() { *x = LogEntry_Operation_CreateHardLink{} - mi := &file_log_proto_msgTypes[10] + mi := &file_log_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -696,7 +763,7 @@ func (x *LogEntry_Operation_CreateHardLink) String() string { func (*LogEntry_Operation_CreateHardLink) ProtoMessage() {} func (x *LogEntry_Operation_CreateHardLink) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[10] + mi := &file_log_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -747,7 +814,7 @@ type LogEntry_Operation_RemoveDirectoryEntry struct { func (x *LogEntry_Operation_RemoveDirectoryEntry) Reset() { *x = LogEntry_Operation_RemoveDirectoryEntry{} - mi := &file_log_proto_msgTypes[11] + mi := &file_log_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -759,7 +826,7 @@ func (x *LogEntry_Operation_RemoveDirectoryEntry) String() string { func (*LogEntry_Operation_RemoveDirectoryEntry) ProtoMessage() {} func (x *LogEntry_Operation_RemoveDirectoryEntry) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[11] + mi := &file_log_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -796,7 +863,7 @@ type LogEntry_Operation_CreateDirectory struct { func (x *LogEntry_Operation_CreateDirectory) Reset() { *x = LogEntry_Operation_CreateDirectory{} - mi := &file_log_proto_msgTypes[12] + mi := &file_log_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -808,7 +875,7 @@ func (x *LogEntry_Operation_CreateDirectory) String() string { func (*LogEntry_Operation_CreateDirectory) ProtoMessage() {} func (x *LogEntry_Operation_CreateDirectory) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[12] + mi := &file_log_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -852,7 +919,7 @@ type LogEntry_Operation_SetKey struct { func (x *LogEntry_Operation_SetKey) Reset() { *x = LogEntry_Operation_SetKey{} - mi := &file_log_proto_msgTypes[13] + mi := &file_log_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -864,7 +931,7 @@ func (x *LogEntry_Operation_SetKey) String() string { func (*LogEntry_Operation_SetKey) ProtoMessage() {} func (x *LogEntry_Operation_SetKey) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[13] + mi := &file_log_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -906,7 +973,7 @@ type LogEntry_Operation_DeleteKey struct { func (x *LogEntry_Operation_DeleteKey) Reset() { *x = LogEntry_Operation_DeleteKey{} - mi := &file_log_proto_msgTypes[14] + mi := &file_log_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -918,7 +985,7 @@ func (x *LogEntry_Operation_DeleteKey) String() string { func (*LogEntry_Operation_DeleteKey) ProtoMessage() {} func (x *LogEntry_Operation_DeleteKey) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[14] + mi := &file_log_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -945,7 +1012,7 @@ var File_log_proto protoreflect.FileDescriptor var file_log_proto_rawDesc = []byte{ 0x0a, 0x09, 0x6c, 0x6f, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x67, 0x69, 0x74, - 0x61, 0x6c, 0x79, 0x22, 0xe7, 0x0d, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x61, 0x6c, 0x79, 0x22, 0xe4, 0x0e, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x5c, 0x0a, 0x16, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, @@ -967,101 +1034,109 @@ var file_log_proto_rawDesc = []byte{ 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x6f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0xc7, 0x01, 0x0a, 0x14, 0x52, 0x65, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x46, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x54, 0x72, - 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, - 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x1a, 0x67, 0x0a, 0x06, 0x43, 0x68, 0x61, - 0x6e, 0x67, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x72, 0x65, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x65, - 0x77, 0x5f, 0x6f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x6e, 0x65, 0x77, - 0x4f, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x65, 0x77, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6e, 0x65, 0x77, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x1a, 0x14, 0x0a, 0x12, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, - 0x44, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xa6, 0x03, 0x0a, 0x0c, 0x48, 0x6f, 0x75, - 0x73, 0x65, 0x6b, 0x65, 0x65, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x43, 0x0a, 0x09, 0x70, 0x61, 0x63, - 0x6b, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x67, - 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, - 0x6f, 0x75, 0x73, 0x65, 0x6b, 0x65, 0x65, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x63, 0x6b, - 0x52, 0x65, 0x66, 0x73, 0x52, 0x08, 0x70, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x66, 0x73, 0x12, 0x3c, - 0x0a, 0x06, 0x72, 0x65, 0x70, 0x61, 0x63, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, - 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x2e, 0x48, 0x6f, 0x75, 0x73, 0x65, 0x6b, 0x65, 0x65, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, - 0x70, 0x61, 0x63, 0x6b, 0x52, 0x06, 0x72, 0x65, 0x70, 0x61, 0x63, 0x6b, 0x12, 0x5f, 0x0a, 0x13, - 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x5f, 0x67, 0x72, 0x61, - 0x70, 0x68, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x67, 0x69, 0x74, 0x61, - 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x6f, 0x75, 0x73, - 0x65, 0x6b, 0x65, 0x65, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x6f, - 0x6d, 0x6d, 0x69, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, 0x73, 0x52, 0x11, 0x77, 0x72, 0x69, 0x74, - 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, 0x73, 0x1a, 0x2b, 0x0a, - 0x08, 0x50, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x66, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x72, 0x75, - 0x6e, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, - 0x70, 0x72, 0x75, 0x6e, 0x65, 0x64, 0x52, 0x65, 0x66, 0x73, 0x1a, 0x70, 0x0a, 0x06, 0x52, 0x65, - 0x70, 0x61, 0x63, 0x6b, 0x12, 0x1b, 0x0a, 0x09, 0x6e, 0x65, 0x77, 0x5f, 0x66, 0x69, 0x6c, 0x65, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6e, 0x65, 0x77, 0x46, 0x69, 0x6c, 0x65, - 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, - 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x69, 0x73, 0x5f, 0x66, 0x75, 0x6c, - 0x6c, 0x5f, 0x72, 0x65, 0x70, 0x61, 0x63, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, - 0x69, 0x73, 0x46, 0x75, 0x6c, 0x6c, 0x52, 0x65, 0x70, 0x61, 0x63, 0x6b, 0x1a, 0x13, 0x0a, 0x11, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x35, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x69, 0x74, + 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x1a, + 0xc7, 0x01, 0x0a, 0x14, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x46, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, + 0x67, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, 0x69, 0x74, 0x61, + 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, + 0x1a, 0x67, 0x0a, 0x06, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, + 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x0d, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x65, 0x77, 0x5f, 0x6f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x06, 0x6e, 0x65, 0x77, 0x4f, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x65, + 0x77, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, + 0x6e, 0x65, 0x77, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x1a, 0x14, 0x0a, 0x12, 0x52, 0x65, 0x70, + 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x1a, + 0xa6, 0x03, 0x0a, 0x0c, 0x48, 0x6f, 0x75, 0x73, 0x65, 0x6b, 0x65, 0x65, 0x70, 0x69, 0x6e, 0x67, + 0x12, 0x43, 0x0a, 0x09, 0x70, 0x61, 0x63, 0x6b, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x6f, 0x75, 0x73, 0x65, 0x6b, 0x65, 0x65, 0x70, 0x69, + 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x66, 0x73, 0x52, 0x08, 0x70, 0x61, 0x63, + 0x6b, 0x52, 0x65, 0x66, 0x73, 0x12, 0x3c, 0x0a, 0x06, 0x72, 0x65, 0x70, 0x61, 0x63, 0x6b, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, + 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x6f, 0x75, 0x73, 0x65, 0x6b, 0x65, 0x65, + 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, 0x61, 0x63, 0x6b, 0x52, 0x06, 0x72, 0x65, 0x70, + 0x61, 0x63, 0x6b, 0x12, 0x5f, 0x0a, 0x13, 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x6d, + 0x6d, 0x69, 0x74, 0x5f, 0x67, 0x72, 0x61, 0x70, 0x68, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x2f, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x48, 0x6f, 0x75, 0x73, 0x65, 0x6b, 0x65, 0x65, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, - 0x73, 0x1a, 0xf9, 0x05, 0x0a, 0x09, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x55, 0x0a, 0x10, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x6c, - 0x69, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x67, 0x69, 0x74, 0x61, - 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x48, 0x61, 0x72, 0x64, - 0x4c, 0x69, 0x6e, 0x6b, 0x48, 0x00, 0x52, 0x0e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x48, 0x61, - 0x72, 0x64, 0x4c, 0x69, 0x6e, 0x6b, 0x12, 0x67, 0x0a, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, - 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, - 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x48, 0x00, 0x52, 0x14, 0x72, 0x65, 0x6d, 0x6f, 0x76, - 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x57, 0x0a, 0x10, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x67, 0x69, 0x74, 0x61, - 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x69, 0x72, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x79, 0x48, 0x00, 0x52, 0x0f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, - 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x3c, 0x0a, 0x07, 0x73, 0x65, 0x74, 0x5f, - 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x67, 0x69, 0x74, 0x61, - 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x48, 0x00, 0x52, 0x06, - 0x73, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x45, 0x0a, 0x0a, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x69, 0x74, - 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4b, 0x65, 0x79, - 0x48, 0x00, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x1a, 0x88, 0x01, - 0x0a, 0x0e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x48, 0x61, 0x72, 0x64, 0x4c, 0x69, 0x6e, 0x6b, - 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, 0x61, 0x74, - 0x68, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x69, 0x6e, 0x5f, 0x73, - 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x12, 0x29, 0x0a, - 0x10, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x61, 0x74, - 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0f, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x74, 0x68, 0x1a, 0x2a, 0x0a, 0x14, 0x52, 0x65, 0x6d, 0x6f, - 0x76, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, - 0x70, 0x61, 0x74, 0x68, 0x1a, 0x39, 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x69, - 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6d, - 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x1a, - 0x30, 0x0a, 0x06, 0x53, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x1a, 0x1d, 0x0a, 0x09, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x42, 0x0b, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x1b, 0x0a, - 0x03, 0x4c, 0x53, 0x4e, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x69, - 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2d, - 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x36, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x70, 0x62, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x52, 0x11, 0x77, 0x72, 0x69, 0x74, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x47, 0x72, + 0x61, 0x70, 0x68, 0x73, 0x1a, 0x2b, 0x0a, 0x08, 0x50, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x66, 0x73, + 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x72, 0x75, 0x6e, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, 0x70, 0x72, 0x75, 0x6e, 0x65, 0x64, 0x52, 0x65, 0x66, + 0x73, 0x1a, 0x70, 0x0a, 0x06, 0x52, 0x65, 0x70, 0x61, 0x63, 0x6b, 0x12, 0x1b, 0x0a, 0x09, 0x6e, + 0x65, 0x77, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, + 0x6e, 0x65, 0x77, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x65, 0x6c, 0x65, + 0x74, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x0c, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x24, 0x0a, + 0x0e, 0x69, 0x73, 0x5f, 0x66, 0x75, 0x6c, 0x6c, 0x5f, 0x72, 0x65, 0x70, 0x61, 0x63, 0x6b, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x73, 0x46, 0x75, 0x6c, 0x6c, 0x52, 0x65, 0x70, + 0x61, 0x63, 0x6b, 0x1a, 0x13, 0x0a, 0x11, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x6f, 0x6d, 0x6d, + 0x69, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, 0x73, 0x1a, 0xf9, 0x05, 0x0a, 0x09, 0x4f, 0x70, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x55, 0x0a, 0x10, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x5f, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x29, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x48, 0x61, 0x72, 0x64, 0x4c, 0x69, 0x6e, 0x6b, 0x48, 0x00, 0x52, 0x0e, 0x63, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x48, 0x61, 0x72, 0x64, 0x4c, 0x69, 0x6e, 0x6b, 0x12, 0x67, 0x0a, + 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x79, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, + 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, + 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, + 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x48, 0x00, + 0x52, 0x14, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x57, 0x0a, 0x10, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x2a, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x48, 0x00, 0x52, 0x0f, + 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, + 0x3c, 0x0a, 0x07, 0x73, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x21, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, + 0x4b, 0x65, 0x79, 0x48, 0x00, 0x52, 0x06, 0x73, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x45, 0x0a, + 0x0a, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x44, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x48, 0x00, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x4b, 0x65, 0x79, 0x1a, 0x88, 0x01, 0x0a, 0x0e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x48, + 0x61, 0x72, 0x64, 0x4c, 0x69, 0x6e, 0x6b, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x5f, 0x69, 0x6e, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x53, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0f, + 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x74, 0x68, 0x1a, + 0x2a, 0x0a, 0x14, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x1a, 0x39, 0x0a, 0x0f, 0x43, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x12, + 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x70, 0x61, + 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x1a, 0x30, 0x0a, 0x06, 0x53, 0x65, 0x74, 0x4b, 0x65, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x1d, 0x0a, 0x09, 0x44, 0x65, 0x6c, 0x65, + 0x74, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x42, 0x0b, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x44, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x74, 0x65, 0x72, 0x6d, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x08, 0x72, 0x61, 0x66, 0x74, 0x54, 0x65, 0x72, 0x6d, 0x12, 0x1b, 0x0a, + 0x09, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x08, 0x72, 0x61, 0x66, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0x1b, 0x0a, 0x03, 0x4c, 0x53, + 0x4e, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x6c, 0x61, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, + 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x70, 0x62, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1076,7 +1151,7 @@ func file_log_proto_rawDescGZIP() []byte { return file_log_proto_rawDescData } -var file_log_proto_msgTypes = make([]protoimpl.MessageInfo, 15) +var file_log_proto_msgTypes = make([]protoimpl.MessageInfo, 16) var file_log_proto_goTypes = []any{ (*LogEntry)(nil), // 0: gitaly.LogEntry (*LSN)(nil), // 1: gitaly.LSN @@ -1084,35 +1159,37 @@ var file_log_proto_goTypes = []any{ (*LogEntry_RepositoryDeletion)(nil), // 3: gitaly.LogEntry.RepositoryDeletion (*LogEntry_Housekeeping)(nil), // 4: gitaly.LogEntry.Housekeeping (*LogEntry_Operation)(nil), // 5: gitaly.LogEntry.Operation - (*LogEntry_ReferenceTransaction_Change)(nil), // 6: gitaly.LogEntry.ReferenceTransaction.Change - (*LogEntry_Housekeeping_PackRefs)(nil), // 7: gitaly.LogEntry.Housekeeping.PackRefs - (*LogEntry_Housekeeping_Repack)(nil), // 8: gitaly.LogEntry.Housekeeping.Repack - (*LogEntry_Housekeeping_WriteCommitGraphs)(nil), // 9: gitaly.LogEntry.Housekeeping.WriteCommitGraphs - (*LogEntry_Operation_CreateHardLink)(nil), // 10: gitaly.LogEntry.Operation.CreateHardLink - (*LogEntry_Operation_RemoveDirectoryEntry)(nil), // 11: gitaly.LogEntry.Operation.RemoveDirectoryEntry - (*LogEntry_Operation_CreateDirectory)(nil), // 12: gitaly.LogEntry.Operation.CreateDirectory - (*LogEntry_Operation_SetKey)(nil), // 13: gitaly.LogEntry.Operation.SetKey - (*LogEntry_Operation_DeleteKey)(nil), // 14: gitaly.LogEntry.Operation.DeleteKey + (*LogEntry_Metadata)(nil), // 6: gitaly.LogEntry.Metadata + (*LogEntry_ReferenceTransaction_Change)(nil), // 7: gitaly.LogEntry.ReferenceTransaction.Change + (*LogEntry_Housekeeping_PackRefs)(nil), // 8: gitaly.LogEntry.Housekeeping.PackRefs + (*LogEntry_Housekeeping_Repack)(nil), // 9: gitaly.LogEntry.Housekeeping.Repack + (*LogEntry_Housekeeping_WriteCommitGraphs)(nil), // 10: gitaly.LogEntry.Housekeeping.WriteCommitGraphs + (*LogEntry_Operation_CreateHardLink)(nil), // 11: gitaly.LogEntry.Operation.CreateHardLink + (*LogEntry_Operation_RemoveDirectoryEntry)(nil), // 12: gitaly.LogEntry.Operation.RemoveDirectoryEntry + (*LogEntry_Operation_CreateDirectory)(nil), // 13: gitaly.LogEntry.Operation.CreateDirectory + (*LogEntry_Operation_SetKey)(nil), // 14: gitaly.LogEntry.Operation.SetKey + (*LogEntry_Operation_DeleteKey)(nil), // 15: gitaly.LogEntry.Operation.DeleteKey } var file_log_proto_depIdxs = []int32{ 2, // 0: gitaly.LogEntry.reference_transactions:type_name -> gitaly.LogEntry.ReferenceTransaction 3, // 1: gitaly.LogEntry.repository_deletion:type_name -> gitaly.LogEntry.RepositoryDeletion 4, // 2: gitaly.LogEntry.housekeeping:type_name -> gitaly.LogEntry.Housekeeping 5, // 3: gitaly.LogEntry.operations:type_name -> gitaly.LogEntry.Operation - 6, // 4: gitaly.LogEntry.ReferenceTransaction.changes:type_name -> gitaly.LogEntry.ReferenceTransaction.Change - 7, // 5: gitaly.LogEntry.Housekeeping.pack_refs:type_name -> gitaly.LogEntry.Housekeeping.PackRefs - 8, // 6: gitaly.LogEntry.Housekeeping.repack:type_name -> gitaly.LogEntry.Housekeeping.Repack - 9, // 7: gitaly.LogEntry.Housekeeping.write_commit_graphs:type_name -> gitaly.LogEntry.Housekeeping.WriteCommitGraphs - 10, // 8: gitaly.LogEntry.Operation.create_hard_link:type_name -> gitaly.LogEntry.Operation.CreateHardLink - 11, // 9: gitaly.LogEntry.Operation.remove_directory_entry:type_name -> gitaly.LogEntry.Operation.RemoveDirectoryEntry - 12, // 10: gitaly.LogEntry.Operation.create_directory:type_name -> gitaly.LogEntry.Operation.CreateDirectory - 13, // 11: gitaly.LogEntry.Operation.set_key:type_name -> gitaly.LogEntry.Operation.SetKey - 14, // 12: gitaly.LogEntry.Operation.delete_key:type_name -> gitaly.LogEntry.Operation.DeleteKey - 13, // [13:13] is the sub-list for method output_type - 13, // [13:13] is the sub-list for method input_type - 13, // [13:13] is the sub-list for extension type_name - 13, // [13:13] is the sub-list for extension extendee - 0, // [0:13] is the sub-list for field type_name + 6, // 4: gitaly.LogEntry.metadata:type_name -> gitaly.LogEntry.Metadata + 7, // 5: gitaly.LogEntry.ReferenceTransaction.changes:type_name -> gitaly.LogEntry.ReferenceTransaction.Change + 8, // 6: gitaly.LogEntry.Housekeeping.pack_refs:type_name -> gitaly.LogEntry.Housekeeping.PackRefs + 9, // 7: gitaly.LogEntry.Housekeeping.repack:type_name -> gitaly.LogEntry.Housekeeping.Repack + 10, // 8: gitaly.LogEntry.Housekeeping.write_commit_graphs:type_name -> gitaly.LogEntry.Housekeeping.WriteCommitGraphs + 11, // 9: gitaly.LogEntry.Operation.create_hard_link:type_name -> gitaly.LogEntry.Operation.CreateHardLink + 12, // 10: gitaly.LogEntry.Operation.remove_directory_entry:type_name -> gitaly.LogEntry.Operation.RemoveDirectoryEntry + 13, // 11: gitaly.LogEntry.Operation.create_directory:type_name -> gitaly.LogEntry.Operation.CreateDirectory + 14, // 12: gitaly.LogEntry.Operation.set_key:type_name -> gitaly.LogEntry.Operation.SetKey + 15, // 13: gitaly.LogEntry.Operation.delete_key:type_name -> gitaly.LogEntry.Operation.DeleteKey + 14, // [14:14] is the sub-list for method output_type + 14, // [14:14] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name } func init() { file_log_proto_init() } @@ -1133,7 +1210,7 @@ func file_log_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_log_proto_rawDesc, NumEnums: 0, - NumMessages: 15, + NumMessages: 16, NumExtensions: 0, NumServices: 0, }, diff --git a/proto/go/gitalypb/protolist.go b/proto/go/gitalypb/protolist.go index 3f7817e7827c658e0d044e6e1ec564fc3c8e67c0..2819219565fc5e69d42fd6a195318e531d6928a9 100644 --- a/proto/go/gitalypb/protolist.go +++ b/proto/go/gitalypb/protolist.go @@ -7,6 +7,7 @@ var GitalyProtos = []string{ "analysis.proto", "blob.proto", "cleanup.proto", + "cluster.proto", "commit.proto", "conflicts.proto", "diff.proto", diff --git a/proto/log.proto b/proto/log.proto index fbf7879b79c664a5552a2faf42abd92d6fed16a4..595a36064e84d3b5becd8034ac34d36c160e9a74 100644 --- a/proto/log.proto +++ b/proto/log.proto @@ -146,9 +146,22 @@ message LogEntry { }; } + // Metadata contains some metadata data of the log entry. + message Metadata { + // raft_term is the term sequence of the Raft group finalized this + // log entry. + uint64 raft_term = 1; + // raft_type is the type of sequence. It's essentially equal to + // raftpb.EntryType. + uint64 raft_type = 2; + } + // operations is an ordered list of operations to run in order to apply // this log entry. repeated Operation operations = 10; + + // metadata contains some extra information, mostly Raft-related metadata. + Metadata metadata = 11; } // LSN serializes a log sequence number. It's used for storing a partition's