From 330c7f483041649d7bd767e90171e01e951fba7f Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Tue, 22 Oct 2024 17:10:33 +0700 Subject: [PATCH 01/19] storagemgr: Add LSN to test hooks The transaction manager has a set of test hooks. They are triggered before performing certain critical actions. The test suite uses those hooks to inject undesirable events, such as crashes or unexpected manager closings. Currently, they don't accept any arguments since those events are applied widely. In some later commits, the test suite needs to trigger hooks conditionally. This is due to the fact that Raft injects some internal log entries, such as config changes or empty log entries, for verification. The tests fail prematurely if the hook is triggered while applying those entries. While the transaction manager resumes the processing successfully afterward, crashing Raft's internal log entries should be covered elsewhere. The existing tests only cover the crashing while committing "normal" log entries. This commit adds LSN to test hooks for that use case. --- .../partition/transaction_manager.go | 24 +++++++++---------- .../transaction_manager_hook_test.go | 24 +++++++++++++++---- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager.go index f6b7e1ab420..7293d285f36 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager.go @@ -1045,10 +1045,10 @@ type TransactionManager struct { type testHooks struct { beforeInitialization func() - beforeAppendLogEntry func() - beforeApplyLogEntry func() - beforeStoreAppliedLSN func() - beforeDeleteLogEntryFiles func() + beforeAppendLogEntry func(storage.LSN) + beforeApplyLogEntry func(storage.LSN) + beforeStoreAppliedLSN func(storage.LSN) + beforeDeleteLogEntryFiles func(storage.LSN) beforeRunExiting func() } @@ -1097,10 +1097,10 @@ func NewTransactionManager( testHooks: testHooks{ beforeInitialization: func() {}, - beforeAppendLogEntry: func() {}, - beforeApplyLogEntry: func() {}, - beforeStoreAppliedLSN: func() {}, - beforeDeleteLogEntryFiles: func() {}, + beforeAppendLogEntry: func(storage.LSN) {}, + beforeApplyLogEntry: func(storage.LSN) {}, + beforeStoreAppliedLSN: func(storage.LSN) {}, + beforeDeleteLogEntryFiles: func(storage.LSN) {}, beforeRunExiting: func() {}, }, } @@ -3538,9 +3538,9 @@ func (mgr *TransactionManager) appendLogEntry(ctx context.Context, objectDepende return fmt.Errorf("synchronizing WAL directory: %w", err) } - mgr.testHooks.beforeAppendLogEntry() - nextLSN := mgr.appendedLSN + 1 + mgr.testHooks.beforeAppendLogEntry(nextLSN) + // Move the log entry from the staging directory into its place in the log. destinationPath := walFilesPathForLSN(mgr.stateDirectory, nextLSN) if err := os.Rename(logEntryPath, destinationPath); err != nil { @@ -3600,7 +3600,7 @@ func (mgr *TransactionManager) applyLogEntry(ctx context.Context, lsn storage.LS delete(mgr.snapshotLocks, previousLSN) mgr.mutex.Unlock() - mgr.testHooks.beforeApplyLogEntry() + mgr.testHooks.beforeApplyLogEntry(lsn) if err := applyOperations(ctx, safe.NewSyncer().Sync, mgr.storagePath, walFilesPathForLSN(mgr.stateDirectory, lsn), logEntry, mgr.db); err != nil { return fmt.Errorf("apply operations: %w", err) @@ -3615,7 +3615,7 @@ func (mgr *TransactionManager) applyLogEntry(ctx context.Context, lsn storage.LS } } - mgr.testHooks.beforeStoreAppliedLSN() + mgr.testHooks.beforeStoreAppliedLSN(lsn) if err := mgr.storeAppliedLSN(lsn); err != nil { return fmt.Errorf("set applied LSN: %w", err) } diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go index 3434a94f408..a9c26394437 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go @@ -19,16 +19,14 @@ type hookFunc func(hookContext) type hookContext struct { // closeManager calls the calls Close on the TransactionManager. closeManager func() + // lsn stores the LSN context when the hook is triggered. + lsn storage.LSN } // installHooks takes the hooks in the test setup and configures them in the TransactionManager. func installHooks(mgr *TransactionManager, inflightTransactions *sync.WaitGroup, hooks testTransactionHooks) { for destination, source := range map[*func()]hookFunc{ - &mgr.testHooks.beforeInitialization: hooks.BeforeReadAppliedLSN, - &mgr.testHooks.beforeAppendLogEntry: hooks.BeforeAppendLogEntry, - &mgr.testHooks.beforeApplyLogEntry: hooks.BeforeApplyLogEntry, - &mgr.testHooks.beforeStoreAppliedLSN: hooks.BeforeStoreAppliedLSN, - &mgr.testHooks.beforeDeleteLogEntryFiles: hooks.AfterDeleteLogEntry, + &mgr.testHooks.beforeInitialization: hooks.BeforeReadAppliedLSN, &mgr.testHooks.beforeRunExiting: func(hookContext) { if hooks.WaitForTransactionsWhenClosing { inflightTransactions.Wait() @@ -46,6 +44,22 @@ func installHooks(mgr *TransactionManager, inflightTransactions *sync.WaitGroup, } } } + for destination, source := range map[*func(storage.LSN)]hookFunc{ + &mgr.testHooks.beforeApplyLogEntry: hooks.BeforeApplyLogEntry, + &mgr.testHooks.beforeAppendLogEntry: hooks.BeforeAppendLogEntry, + &mgr.testHooks.beforeStoreAppliedLSN: hooks.BeforeStoreAppliedLSN, + &mgr.testHooks.beforeDeleteLogEntryFiles: hooks.AfterDeleteLogEntry, + } { + if source != nil { + runHook := source + *destination = func(lsn storage.LSN) { + runHook(hookContext{ + closeManager: mgr.Close, + lsn: lsn, + }) + } + } + } } func generateCustomHooksTests(t *testing.T, setup testTransactionSetup) []transactionTestCase { -- GitLab From fae2fced9f24ffe7e607cd4cb21ba5542eccd47e Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Fri, 1 Nov 2024 10:48:39 +0700 Subject: [PATCH 02/19] storagemgr: Store TransactionManager in the hook context TransactionManager's test suite installs some hooks so that tests can access certain stages of the transaction's life cycle. Those hooks are fed with a hook context containing LSN and a close function. In some later commits, some tests need to access some data of TransactionManager. Adding arbitrary data to the hook context continuously is tedious. Thus, this commit embeds the pointer to TransactionManager in the hook context. This should not be a concern. Outsiders could not access the internal state. Only internal package tests can install such hooks. --- .../partition/transaction_manager_hook_test.go | 10 +++++----- .../storagemgr/partition/transaction_manager_test.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go index a9c26394437..c8db3837963 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go @@ -17,8 +17,8 @@ type hookFunc func(hookContext) // hookContext are the control toggles available in a hook. type hookContext struct { - // closeManager calls the calls Close on the TransactionManager. - closeManager func() + // manager points to the subject TransactionManager. + manager *TransactionManager // lsn stores the LSN context when the hook is triggered. lsn storage.LSN } @@ -39,7 +39,7 @@ func installHooks(mgr *TransactionManager, inflightTransactions *sync.WaitGroup, runHook := source *destination = func() { runHook(hookContext{ - closeManager: mgr.Close, + manager: mgr, }) } } @@ -54,8 +54,8 @@ func installHooks(mgr *TransactionManager, inflightTransactions *sync.WaitGroup, runHook := source *destination = func(lsn storage.LSN) { runHook(hookContext{ - closeManager: mgr.Close, - lsn: lsn, + manager: mgr, + lsn: lsn, }) } } diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go index 021f6adddc3..b3b10111857 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go @@ -485,7 +485,7 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeAppendLogEntry: func(hookContext hookContext) { hookContext.closeManager() }, + BeforeAppendLogEntry: func(hookContext hookContext) { hookContext.manager.Close() }, // This ensures we are testing the context cancellation errors being unwrapped properly // to an storage.ErrTransactionProcessingStopped instead of hitting the general case when // runDone is closed. -- GitLab From 95ca17ac14af9c7a1357e2b9262dbbea12a4a162 Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Tue, 22 Oct 2024 17:37:25 +0700 Subject: [PATCH 03/19] storagemgr: Centralize errSimulateCrash emitter The test suite of storage inserts a bunch of test hooks that panic with errSimulateCrash. Those hooks are unconditional. This commit creates a helper function that returns a hook function that emits the panic error. This is a preparation step for later commits where the crashing occurs for certain entries only. --- .../transaction_manager_alternate_test.go | 8 ++--- ...transaction_manager_default_branch_test.go | 4 +-- .../transaction_manager_hook_test.go | 20 ++++--------- .../transaction_manager_housekeeping_test.go | 4 +-- .../transaction_manager_key_value_test.go | 8 ++--- .../transaction_manager_repo_test.go | 16 +++------- .../partition/transaction_manager_test.go | 29 ++++++++++--------- 7 files changed, 32 insertions(+), 57 deletions(-) diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_alternate_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_alternate_test.go index dbe1843e39f..bbd632c432b 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_alternate_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_alternate_test.go @@ -1124,9 +1124,7 @@ func generateAlternateTests(t *testing.T, setup testTransactionSetup) []transact CloseManager{}, StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -1203,9 +1201,7 @@ func generateAlternateTests(t *testing.T, setup testTransactionSetup) []transact CloseManager{}, StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_default_branch_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_default_branch_test.go index 5bfd1c5fc96..07ab97957c8 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_default_branch_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_default_branch_test.go @@ -464,9 +464,7 @@ func generateDefaultBranchTests(t *testing.T, setup testTransactionSetup) []tran steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookCtx hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go index c8db3837963..218f09d48e8 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go @@ -103,9 +103,7 @@ func generateCustomHooksTests(t *testing.T, setup testTransactionSetup) []transa steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -153,9 +151,7 @@ func generateCustomHooksTests(t *testing.T, setup testTransactionSetup) []transa steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeApplyLogEntry: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeApplyLogEntry: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -497,9 +493,7 @@ func generateCustomHooksTests(t *testing.T, setup testTransactionSetup) []transa steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookCtx hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -591,9 +585,7 @@ func generateCustomHooksTests(t *testing.T, setup testTransactionSetup) []transa steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeApplyLogEntry: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeApplyLogEntry: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -685,9 +677,7 @@ func generateCustomHooksTests(t *testing.T, setup testTransactionSetup) []transa steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeApplyLogEntry: func(hookCtx hookContext) { - panic(errSimulatedCrash) - }, + BeforeApplyLogEntry: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_housekeeping_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_housekeeping_test.go index f0e36bd571e..5af8d077c47 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_housekeeping_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_housekeeping_test.go @@ -1335,9 +1335,7 @@ func generateHousekeepingPackRefsTests(t *testing.T, ctx context.Context, testPa CloseManager{}, StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_key_value_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_key_value_test.go index b39cc1b2b53..25f2055cb69 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_key_value_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_key_value_test.go @@ -1,11 +1,13 @@ package partition import ( + "testing" + "github.com/dgraph-io/badger/v4" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" ) -func generateKeyValueTests(setup testTransactionSetup) []transactionTestCase { +func generateKeyValueTests(t *testing.T, setup testTransactionSetup) []transactionTestCase { return []transactionTestCase{ { desc: "set keys with values", @@ -615,9 +617,7 @@ func generateKeyValueTests(setup testTransactionSetup) []transactionTestCase { steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeApplyLogEntry: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeApplyLogEntry: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_repo_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_repo_test.go index 785204ee338..1f356bc4d6a 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_repo_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_repo_test.go @@ -364,9 +364,7 @@ func generateCreateRepositoryTests(t *testing.T, setup testTransactionSetup) []t RemoveRepository{}, StartManager{ Hooks: testTransactionHooks{ - BeforeApplyLogEntry: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeApplyLogEntry: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -408,9 +406,7 @@ func generateCreateRepositoryTests(t *testing.T, setup testTransactionSetup) []t RemoveRepository{}, StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -910,9 +906,7 @@ func generateDeleteRepositoryTests(t *testing.T, setup testTransactionSetup) []t steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeApplyLogEntry: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeApplyLogEntry: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -954,9 +948,7 @@ func generateDeleteRepositoryTests(t *testing.T, setup testTransactionSetup) []t steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go index b3b10111857..3fadb588822 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go @@ -39,6 +39,13 @@ import ( // TransactionManager.Run execution. var errSimulatedCrash = errors.New("simulated crash") +// simulateCrashHook returns a hook function that panics with errSimulatedCrash. +var simulateCrashHook = func() func(hookContext) { + return func(hookContext) { + panic(errSimulatedCrash) + } +} + func manifestDirectoryEntry(expected *gitalypb.LogEntry) testhelper.DirectoryEntry { return testhelper.DirectoryEntry{ Mode: mode.File, @@ -350,7 +357,7 @@ func TestTransactionManager(t *testing.T) { "Housekeeping/RepackingConcurrent": generateHousekeepingRepackingConcurrentTests(t, ctx, setup), "Housekeeping/CommitGraphs": generateHousekeepingCommitGraphsTests(t, ctx, setup), "Consumer": generateConsumerTests(t, setup), - "KeyValue": generateKeyValueTests(setup), + "KeyValue": generateKeyValueTests(t, setup), } for desc, tests := range subTests { @@ -506,9 +513,7 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeApplyLogEntry: func(hookCtx hookContext) { - panic(errSimulatedCrash) - }, + BeforeApplyLogEntry: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -865,9 +870,7 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio Prune{}, StartManager{ Hooks: testTransactionHooks{ - BeforeStoreAppliedLSN: func(hookContext) { - panic(errSimulatedCrash) - }, + BeforeStoreAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, @@ -1438,13 +1441,11 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio steps: steps{ StartManager{ Hooks: testTransactionHooks{ - BeforeReadAppliedLSN: func(hookContext) { - // Raise a panic when the manager is about to read the applied log - // index when initializing. In reality this would crash the server but - // in tests it serves as a way to abort the initialization in correct - // location. - panic(errSimulatedCrash) - }, + // Raise a panic when the manager is about to read the applied log + // index when initializing. In reality this would crash the server but + // in tests it serves as a way to abort the initialization in correct + // location. + BeforeReadAppliedLSN: simulateCrashHook(), }, ExpectedError: errSimulatedCrash, }, -- GitLab From 736b0c6da2d31a8b4c6c1ddb3e6db471b343013d Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Wed, 30 Oct 2024 10:23:17 +0700 Subject: [PATCH 04/19] storagemgr: Rename appendedLSN to committedLSN Recently, the TransactionManager uses two indices which are appendedLSN and appliedLSN. It increases the value of appendedLSN after it accepts a transaction. When the log entry is applied, the transaction manager increases appliedLSN accordingly. Any log entries prior to appliedLSN could be removed if there are no pending transactions referring to them. We are working on adding Raft consensus to the TransactionManager. When a log entry is accepted by the current node, it also needs to be transferred to and accepted by the majority of cluster members. To make it easier for later steps, the TransactionManager needs to track two indices independently: * appendedLSN: this index tracks the LSN of the log entry accepted by the current node, but not yet accepted by others. The transaction manager are allowed to override entries from committedLSN + 1 to appendedLSN. * committedLSN: this index tracks the committed log entries. They are safe to be applied to the repositories. Eventually, other nodes apply those entries in the same order. As the current appendedLSN index is taking the role of committedLSN, this commit renames it to commitedLSN. --- .../partition/transaction_manager.go | 74 ++++++++++--------- .../partition/transaction_manager_test.go | 12 +-- 2 files changed, 46 insertions(+), 40 deletions(-) diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager.go index 7293d285f36..534586b807a 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager.go @@ -350,7 +350,7 @@ func (mgr *TransactionManager) Begin(ctx context.Context, opts storage.BeginOpti txn := &Transaction{ write: opts.Write, commit: mgr.commit, - snapshotLSN: mgr.appendedLSN, + snapshotLSN: mgr.committedLSN, finished: make(chan struct{}), relativePath: relativePath, metrics: mgr.metrics, @@ -925,7 +925,7 @@ func (p *consumerPosition) setPosition(pos storage.LSN) { // - The reference verification failures can be ignored instead of aborting the entire transaction. // If done, the references that failed verification are dropped from the transaction but the updates // that passed verification are still performed. -// 2. The transaction is appended to the write-ahead log. Once the write has been logged, it is effectively +// 2. The transaction is committed to the write-ahead log. Once the write has been logged, it is effectively // committed and will be applied to the repository even after restarting. // 3. The transaction is applied from the write-ahead log to the repository by actually performing the reference // changes. @@ -998,7 +998,7 @@ type TransactionManager struct { // initializationSuccessful is set if the TransactionManager initialized successfully. If it didn't, // transactions will fail to begin. initializationSuccessful bool - // mutex guards access to snapshotLocks and appendedLSN. These fields are accessed by both + // mutex guards access to snapshotLocks and committedLSN. These fields are accessed by both // Run and Begin which are ran in different goroutines. mutex sync.Mutex @@ -1008,8 +1008,14 @@ type TransactionManager struct { // snapshotManager is responsible for creation and management of file system snapshots. snapshotManager *snapshot.Manager - // appendedLSN holds the LSN of the last log entry appended to the partition's write-ahead log. - appendedLSN storage.LSN + // ┌─ oldestLSN ┌─ committedLSN + // ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ■ ■ ■ ■ ■ ■ ■ ■ ■ ■ □ □ □ □ □ □ □ + // └─ appliedLSN └─ appendedLSN + // + // committedLSN holds the LSN of the last log entry committed to the partition's write-ahead log. A log entry is + // considered to be committed if it's accepted by the majority of cluster members. Eventually, it will be + // applied by all cluster members. + committedLSN storage.LSN // appliedLSN holds the LSN of the last log entry applied to the partition. appliedLSN storage.LSN // oldestLSN holds the LSN of the head of log entries which is still kept in the database. The manager keeps @@ -1020,7 +1026,7 @@ type TransactionManager struct { // the partition. It's keyed by the LSN the transaction is waiting to be applied and the // value is the resultChannel that is waiting the result. awaitingTransactions map[storage.LSN]resultChannel - // committedEntries keeps some latest appended log entries around. Some types of transactions, such as + // committedEntries keeps some latest committed log entries around. Some types of transactions, such as // housekeeping, operate on snapshot repository. There is a gap between transaction doing its work and the time // when it is committed. They need to verify if concurrent operations can cause conflict. These log entries are // still kept around even after they are applied. They are removed when there are no active readers accessing @@ -1228,7 +1234,7 @@ func (mgr *TransactionManager) commit(ctx context.Context, transaction *Transact // The reference updates are staged into the transaction when they are verified, including // the packed-refs. While the reference files would generally be small, the packed-refs file // may be large. Sync the contents of the file before entering the critical section to ensure - // we don't end up syncing the potentially very large file to disk when we're appending the + // we don't end up syncing the potentially very large file to disk when we're committing the // log entry. preImagePackedRefsInode, err := getInode(transaction.originalPackedRefsFilePath()) if err != nil { @@ -1774,7 +1780,7 @@ func (mgr *TransactionManager) preparePackRefsReftable(ctx context.Context, tran Name: "pack-refs", // By using the '--auto' flag, we ensure that git uses the best heuristic // for compaction. For reftables, it currently uses a geometric progression. - // This ensures we don't keep compacting unecessarily to a single file. + // This ensures we don't keep compacting unnecessarily to a single file. Flags: []gitcmd.Option{gitcmd.Flag{Name: "--auto"}}, }, gitcmd.WithStderr(&stderr)); err != nil { return structerr.New("exec pack-refs: %w", err).WithMetadata("stderr", stderr.String()) @@ -2131,7 +2137,7 @@ func unwrapExpectedError(err error) error { return err } -// Run starts the transaction processing. On start up Run loads the indexes of the last appended and applied +// Run starts the transaction processing. On start up Run loads the indexes of the last committed and applied // log entries from the database. It will then apply any transactions that have been logged but not applied // to the repository. Once the recovery is completed, Run starts processing new transactions by verifying the // references, logging the transaction and finally applying it to the repository. The transactions are acknowledged @@ -2162,7 +2168,7 @@ func (mgr *TransactionManager) run(ctx context.Context) (returnedErr error) { } for { - if mgr.appliedLSN < mgr.appendedLSN { + if mgr.appliedLSN < mgr.committedLSN { lsn := mgr.appliedLSN + 1 if err := mgr.applyLogEntry(ctx, lsn); err != nil { @@ -2307,13 +2313,13 @@ func (mgr *TransactionManager) processTransaction(ctx context.Context) (returned logEntry.Operations = transaction.walEntry.Operations() - return mgr.appendLogEntry(ctx, transaction.objectDependencies, logEntry, transaction.walFilesPath()) + return mgr.commitLogEntry(ctx, transaction.objectDependencies, logEntry, transaction.walFilesPath()) }(); err != nil { transaction.result <- err return nil } - mgr.awaitingTransactions[mgr.appendedLSN] = transaction.result + mgr.awaitingTransactions[mgr.committedLSN] = transaction.result return nil } @@ -2429,7 +2435,7 @@ func (mgr *TransactionManager) snapshotsDir() string { return filepath.Join(mgr.stagingDirectory, "snapshots") } -// initialize initializes the TransactionManager's state from the database. It loads the appended and the applied +// initialize initializes the TransactionManager's state from the database. It loads the committed and the applied // LSNs and initializes the notification channels that synchronize transaction beginning with log entry applying. func (mgr *TransactionManager) initialize(ctx context.Context) error { defer trace.StartRegion(ctx, "initialize").End() @@ -2456,24 +2462,24 @@ func (mgr *TransactionManager) initialize(ctx context.Context) error { return fmt.Errorf("new snapshot manager: %w", err) } - // The LSN of the last appended log entry is determined from the LSN of the latest entry in the log and + // The LSN of the last committed log entry is determined from the LSN of the latest entry in the log and // the latest applied log entry. The manager also keeps track of committed entries and reserves them until there // is no transaction refers them. It's possible there are some left-over entries in the database because a // transaction can hold the entry stubbornly. So, the manager could not clean them up in the last session. // - // ┌─ oldestLSN ┌─ appendedLSN - // ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ■ ■ ■ ■ ■ ■ ■ ■ ■ ■ - // └─ appliedLSN + // ┌─ oldestLSN ┌─ committedLSN + // ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ■ ■ ■ ■ ■ ■ ■ ■ ■ ■ □ □ □ □ □ □ □ + // └─ appliedLSN └─ appendedLSN // // // oldestLSN is initialized to appliedLSN + 1. If there are no log entries in the log, then everything has been // pruned already or there has not been any log entries yet. Setting this +1 avoids trying to clean up log entries // that do not exist. If there are some, we'll set oldestLSN to the head of the log below. mgr.oldestLSN = mgr.appliedLSN + 1 - // appendedLSN is initialized to appliedLSN. If there are no log entries, then there has been no transaction yet, or + // committedLSN is initialized to appliedLSN. If there are no log entries, then there has been no transaction yet, or // all log entries have been applied and have been already pruned. If there are some in the log, we'll update this // below to match. - mgr.appendedLSN = mgr.appliedLSN + mgr.committedLSN = mgr.appliedLSN if logEntries, err := os.ReadDir(walFilesPath(mgr.stateDirectory)); err != nil { return fmt.Errorf("read wal directory: %w", err) @@ -2481,13 +2487,13 @@ func (mgr *TransactionManager) initialize(ctx context.Context) error { if mgr.oldestLSN, err = storage.ParseLSN(logEntries[0].Name()); err != nil { return fmt.Errorf("parse oldest LSN: %w", err) } - if mgr.appendedLSN, err = storage.ParseLSN(logEntries[len(logEntries)-1].Name()); err != nil { - return fmt.Errorf("parse appended LSN: %w", err) + if mgr.committedLSN, err = storage.ParseLSN(logEntries[len(logEntries)-1].Name()); err != nil { + return fmt.Errorf("parse committed LSN: %w", err) } } if mgr.consumer != nil { - mgr.consumer.NotifyNewTransactions(mgr.storageName, mgr.partitionID, mgr.oldestLSN, mgr.appendedLSN) + mgr.consumer.NotifyNewTransactions(mgr.storageName, mgr.partitionID, mgr.oldestLSN, mgr.committedLSN) } // Create a snapshot lock for the applied LSN as it is used for synchronizing @@ -2497,11 +2503,11 @@ func (mgr *TransactionManager) initialize(ctx context.Context) error { // Each unapplied log entry should have a snapshot lock as they are created in normal // operation when committing a log entry. Recover these entries. - for i := mgr.appliedLSN + 1; i <= mgr.appendedLSN; i++ { + for i := mgr.appliedLSN + 1; i <= mgr.committedLSN; i++ { mgr.snapshotLocks[i] = &snapshotLock{applied: make(chan struct{})} } - if err := mgr.removeStaleWALFiles(ctx, mgr.oldestLSN, mgr.appendedLSN); err != nil { + if err := mgr.removeStaleWALFiles(ctx, mgr.oldestLSN, mgr.committedLSN); err != nil { return fmt.Errorf("remove stale packs: %w", err) } @@ -2597,15 +2603,15 @@ func (mgr *TransactionManager) removePackedRefsLocks(ctx context.Context, reposi // but the manager was interrupted before successfully persisting the log entry itself. // If the manager deletes a log entry successfully from the database but is interrupted before it cleans // up the associated files, such a directory can also be left at the head of the log. -func (mgr *TransactionManager) removeStaleWALFiles(ctx context.Context, oldestLSN, appendedLSN storage.LSN) error { +func (mgr *TransactionManager) removeStaleWALFiles(ctx context.Context, oldestLSN, committedLSN storage.LSN) error { needsFsync := false for _, possibleStaleFilesPath := range []string{ // Log entries are pruned one by one. If a write is interrupted, the only possible stale files would be // for the log entry preceding the oldest log entry. walFilesPathForLSN(mgr.stateDirectory, oldestLSN-1), - // Log entries are appended one by one to the log. If a write is interrupted, the only possible stale + // Log entries are committed one by one to the log. If a write is interrupted, the only possible stale // files would be for the next LSN. Remove the files if they exist. - walFilesPathForLSN(mgr.stateDirectory, appendedLSN+1), + walFilesPathForLSN(mgr.stateDirectory, committedLSN+1), } { if _, err := os.Stat(possibleStaleFilesPath); err != nil { @@ -2857,7 +2863,7 @@ func (mgr *TransactionManager) verifyReferences(ctx context.Context, transaction // to transaction operations. // // To ensure that we don't modify existing tables and autocompact, we lock the existing tables -// before applying the updates. This way the reftable backend willl only create new tables +// before applying the updates. This way the reftable backend will only create new tables func (mgr *TransactionManager) verifyReferencesWithGitForReftables( ctx context.Context, referenceTransactions []*gitalypb.LogEntry_ReferenceTransaction, @@ -3139,7 +3145,7 @@ func (mgr *TransactionManager) verifyHousekeeping(ctx context.Context, transacti mgr.mutex.Lock() defer mgr.mutex.Unlock() - // Check for any concurrent housekeeping between this transaction's snapshot LSN and the latest appended LSN. + // Check for any concurrent housekeeping between this transaction's snapshot LSN and the latest committed LSN. if err := mgr.walkCommittedEntries(transaction, func(entry *gitalypb.LogEntry, objectDependencies map[git.ObjectID]struct{}) error { if entry.GetHousekeeping() != nil { return errHousekeepingConflictConcurrent @@ -3506,11 +3512,11 @@ func (mgr *TransactionManager) applyReferenceTransaction(ctx context.Context, ch return nil } -// appendLogEntry appends the transaction to the write-ahead log. It first writes the transaction's manifest file +// commitLogEntry commits the transaction to the write-ahead log. It first writes the transaction's manifest file // into the log entry's directory. Afterwards it moves the log entry's directory from the staging area to its final // place in the write-ahead log. -func (mgr *TransactionManager) appendLogEntry(ctx context.Context, objectDependencies map[git.ObjectID]struct{}, logEntry *gitalypb.LogEntry, logEntryPath string) error { - defer trace.StartRegion(ctx, "appendLogEntry").End() +func (mgr *TransactionManager) commitLogEntry(ctx context.Context, objectDependencies map[git.ObjectID]struct{}, logEntry *gitalypb.LogEntry, logEntryPath string) error { + defer trace.StartRegion(ctx, "commitLogEntry").End() manifestBytes, err := proto.Marshal(logEntry) if err != nil { @@ -3538,7 +3544,7 @@ func (mgr *TransactionManager) appendLogEntry(ctx context.Context, objectDepende return fmt.Errorf("synchronizing WAL directory: %w", err) } - nextLSN := mgr.appendedLSN + 1 + nextLSN := mgr.committedLSN + 1 mgr.testHooks.beforeAppendLogEntry(nextLSN) // Move the log entry from the staging directory into its place in the log. @@ -3566,7 +3572,7 @@ func (mgr *TransactionManager) appendLogEntry(ctx context.Context, objectDepende // After this latch block, the transaction is committed and all subsequent transactions // are guaranteed to read it. mgr.mutex.Lock() - mgr.appendedLSN = nextLSN + mgr.committedLSN = nextLSN mgr.snapshotLocks[nextLSN] = &snapshotLock{applied: make(chan struct{})} mgr.committedEntries.PushBack(&committedEntry{ lsn: nextLSN, diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go index 3fadb588822..c9a1a3ad802 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go @@ -2067,7 +2067,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t }, }, { - desc: "transaction manager cleans up left-over committed entries when appliedLSN == appendedLSN", + desc: "transaction manager cleans up left-over committed entries when appliedLSN == committedLSN", steps: steps{ StartManager{}, Begin{ @@ -2141,7 +2141,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t string(keyAppliedLSN): storage.LSN(3).ToProto(), }) require.Equal(t, tm.appliedLSN, storage.LSN(3)) - require.Equal(t, tm.appendedLSN, storage.LSN(3)) + require.Equal(t, tm.committedLSN, storage.LSN(3)) }), }, expectedState: StateAssertion{ @@ -2213,7 +2213,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t }, }, { - desc: "transaction manager cleans up left-over committed entries when appliedLSN < appendedLSN", + desc: "transaction manager cleans up left-over committed entries when appliedLSN < committedLSN", skip: func(t *testing.T) { testhelper.SkipWithReftable(t, "test requires manual log addition") }, @@ -2264,11 +2264,11 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { // Insert an out-of-band log-entry directly into the database for easier test // setup. It's a bit tricky to simulate committed log entries and un-processed - // appended log entries at the same time. + // committed log entries at the same time. logEntryPath := filepath.Join(t.TempDir(), "log_entry") require.NoError(t, os.Mkdir(logEntryPath, mode.Directory)) require.NoError(t, os.WriteFile(filepath.Join(logEntryPath, "1"), []byte(setup.Commits.First.OID+"\n"), mode.File)) - require.NoError(t, tm.appendLogEntry(ctx, map[git.ObjectID]struct{}{setup.Commits.First.OID: {}}, refChangeLogEntry(setup, "refs/heads/branch-3", setup.Commits.First.OID), logEntryPath)) + require.NoError(t, tm.commitLogEntry(ctx, map[git.ObjectID]struct{}{setup.Commits.First.OID: {}}, refChangeLogEntry(setup, "refs/heads/branch-3", setup.Commits.First.OID), logEntryPath)) RequireDatabase(t, ctx, tm.db, DatabaseState{ string(keyAppliedLSN): storage.LSN(3).ToProto(), @@ -2297,7 +2297,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t string(keyAppliedLSN): storage.LSN(4).ToProto(), }) require.Equal(t, tm.appliedLSN, storage.LSN(4)) - require.Equal(t, tm.appendedLSN, storage.LSN(4)) + require.Equal(t, tm.committedLSN, storage.LSN(4)) }), }, expectedState: StateAssertion{ -- GitLab From a2368e6855443b4cc01c9c907d582d8f470d173c Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Thu, 10 Oct 2024 14:34:57 +0700 Subject: [PATCH 05/19] storagemgr: Re-introduce intermediate appendedLSN index In the previous commit, the TransactionManager renames appendedLSN to committedLSN. This commit re-introduces appendedLSN but with a different meaning. It now tracks the intermediate state waiting to be committed by other nodes. The `commitLogEntry` is split into two smaller functions. The first one encapsulates and updates appendedLSN. The second one is supposed to engage Raft updates committedLSN. As now we haven't integrated the Raft library yet, the second function simply increases the committedLSN index. This commit also introduces a new non-subtle change: committedLSN does not persist in KV DB but is next to appliedLSN. Before, as soon as a log entry is flushed to disk, that log entry will be applied, eventually. After splitting the append stage and commit stage, things might go wrong after a log entry is appended but is not committed. This is particularly true if Raft enters the picture and adds network latency to the log entry proposal. Thus, the transaction manager needs to persist the committedLSN to avoid applying for uncommitted entries after a crash. After restarts, it resumes from the latest committedLSN, applies unapplied entries, if any, and removes obsoleted appended entries, if any. --- .../storagemgr/partition/testhelper_test.go | 9 + .../partition/transaction_manager.go | 118 ++++++- .../transaction_manager_hook_test.go | 1 + .../partition/transaction_manager_test.go | 317 +++++++++++++++++- 4 files changed, 427 insertions(+), 18 deletions(-) diff --git a/internal/gitaly/storage/storagemgr/partition/testhelper_test.go b/internal/gitaly/storage/storagemgr/partition/testhelper_test.go index c7f5b00464d..fac426aa5a5 100644 --- a/internal/gitaly/storage/storagemgr/partition/testhelper_test.go +++ b/internal/gitaly/storage/storagemgr/partition/testhelper_test.go @@ -627,6 +627,13 @@ func RequireDatabase(tb testing.TB, ctx context.Context, database keyvalue.Trans if expectedState == nil { expectedState = DatabaseState{} } + // Most of the time, persisted committedLSN is equal to appliedLSN except for some intentional tests. Thus, if + // appliedLSN is asserted, the test should backfill committedLSN if it's not there. + if appliedLSN, appliedExist := expectedState[string(keyAppliedLSN)]; appliedExist { + if _, committedExist := expectedState[string(keyCommittedLSN)]; !committedExist { + expectedState[string(keyCommittedLSN)] = appliedLSN + } + } actualState := DatabaseState{} unexpectedKeys := []string{} @@ -705,6 +712,8 @@ type testTransactionHooks struct { BeforeApplyLogEntry hookFunc // BeforeAppendLogEntry is called before a log entry is appended to the log. BeforeAppendLogEntry hookFunc + // BeforeAppendLogEntry is called before a log entry is marked as committed. + BeforeCommitLogEntry hookFunc // AfterDeleteLogEntry is called after a log entry is deleted. AfterDeleteLogEntry hookFunc // BeforeReadAppliedLSN is invoked before the applied LSN is read. diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager.go index 534586b807a..54d3ce4acdf 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager.go @@ -101,6 +101,8 @@ var ( // keyAppliedLSN is the database key storing a partition's last applied log entry's LSN. keyAppliedLSN = []byte("applied_lsn") + // keyCommittedLSN is the database key storing a partition's last committed log entry's LSN. + keyCommittedLSN = []byte("committed_lsn") ) const relativePathKeyPrefix = "r/" @@ -1012,6 +1014,10 @@ type TransactionManager struct { // ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ■ ■ ■ ■ ■ ■ ■ ■ ■ ■ □ □ □ □ □ □ □ // └─ appliedLSN └─ appendedLSN // + // appendedLSN holds the LSN of the last log entry emitted by the current node but not yet acknowledged by other + // nodes. After so, the committedLSN is increment respectively and catches up with appendedLSN. If Raft is not + // enabled or functions as a single-node cluster, committedLSN is increment instantly. + appendedLSN storage.LSN // committedLSN holds the LSN of the last log entry committed to the partition's write-ahead log. A log entry is // considered to be committed if it's accepted by the majority of cluster members. Eventually, it will be // applied by all cluster members. @@ -1026,6 +1032,10 @@ type TransactionManager struct { // the partition. It's keyed by the LSN the transaction is waiting to be applied and the // value is the resultChannel that is waiting the result. awaitingTransactions map[storage.LSN]resultChannel + + // appendedEntries keeps track of appended but not-yet committed entries. After an entry is committed, it is + // removed from this map. This provides quick reference to those entries. + appendedEntries map[storage.LSN]*gitalypb.LogEntry // committedEntries keeps some latest committed log entries around. Some types of transactions, such as // housekeeping, operate on snapshot repository. There is a gap between transaction doing its work and the time // when it is committed. They need to verify if concurrent operations can cause conflict. These log entries are @@ -1052,6 +1062,7 @@ type TransactionManager struct { type testHooks struct { beforeInitialization func() beforeAppendLogEntry func(storage.LSN) + beforeCommitLogEntry func(storage.LSN) beforeApplyLogEntry func(storage.LSN) beforeStoreAppliedLSN func(storage.LSN) beforeDeleteLogEntryFiles func(storage.LSN) @@ -1095,6 +1106,7 @@ func NewTransactionManager( stateDirectory: stateDir, stagingDirectory: stagingDir, awaitingTransactions: make(map[storage.LSN]resultChannel), + appendedEntries: map[storage.LSN]*gitalypb.LogEntry{}, committedEntries: list.New(), metrics: metrics, consumer: consumer, @@ -1104,6 +1116,7 @@ func NewTransactionManager( testHooks: testHooks{ beforeInitialization: func() {}, beforeAppendLogEntry: func(storage.LSN) {}, + beforeCommitLogEntry: func(storage.LSN) {}, beforeApplyLogEntry: func(storage.LSN) {}, beforeStoreAppliedLSN: func(storage.LSN) {}, beforeDeleteLogEntryFiles: func(storage.LSN) {}, @@ -2168,6 +2181,7 @@ func (mgr *TransactionManager) run(ctx context.Context) (returnedErr error) { } for { + // We prioritize applying committed log entries to the partition first. if mgr.appliedLSN < mgr.committedLSN { lsn := mgr.appliedLSN + 1 @@ -2313,7 +2327,7 @@ func (mgr *TransactionManager) processTransaction(ctx context.Context) (returned logEntry.Operations = transaction.walEntry.Operations() - return mgr.commitLogEntry(ctx, transaction.objectDependencies, logEntry, transaction.walFilesPath()) + return mgr.proposeLogEntry(ctx, transaction.objectDependencies, logEntry, transaction.walFilesPath()) }(); err != nil { transaction.result <- err return nil @@ -2476,19 +2490,51 @@ func (mgr *TransactionManager) initialize(ctx context.Context) error { // pruned already or there has not been any log entries yet. Setting this +1 avoids trying to clean up log entries // that do not exist. If there are some, we'll set oldestLSN to the head of the log below. mgr.oldestLSN = mgr.appliedLSN + 1 - // committedLSN is initialized to appliedLSN. If there are no log entries, then there has been no transaction yet, or - // all log entries have been applied and have been already pruned. If there are some in the log, we'll update this - // below to match. - mgr.committedLSN = mgr.appliedLSN + + // CommittedLSN is loaded from DB. A log entry is appended first and marked as committed later. There's a chance + // that log entry is never marked as committed. After a restart, especially after a crash, the manager won't be + // able to tell if it's committed or not. Thus, we need to persist this index. + // Because index persistence is introduced later, we need to fallback to appliedLSN if that key does not exist + // in the DB. + var committedLSN gitalypb.LSN + if err := mgr.readKey(keyCommittedLSN, &committedLSN); err != nil { + if !errors.Is(err, badger.ErrKeyNotFound) { + return fmt.Errorf("read committed LSN: %w", err) + } + mgr.committedLSN = mgr.appliedLSN + } else { + mgr.committedLSN = storage.LSN(committedLSN.GetValue()) + } + + // appendedLSN is always set to committedLSN after starting. If a log entry hasn't been committed, it could be + // discarded. Its caller never received the acknowledgement. + mgr.appendedLSN = mgr.committedLSN if logEntries, err := os.ReadDir(walFilesPath(mgr.stateDirectory)); err != nil { return fmt.Errorf("read wal directory: %w", err) } else if len(logEntries) > 0 { - if mgr.oldestLSN, err = storage.ParseLSN(logEntries[0].Name()); err != nil { - return fmt.Errorf("parse oldest LSN: %w", err) - } - if mgr.committedLSN, err = storage.ParseLSN(logEntries[len(logEntries)-1].Name()); err != nil { - return fmt.Errorf("parse committed LSN: %w", err) + // All log entries starting from mgr.committedLSN + 1 are not committed. No reason to keep them around. + // Returned log entries are sorted in ascending order. We iterate backward and break when the iterating + // LSN drops below committedLSN. + for i := len(logEntries) - 1; i >= 0; i-- { + logEntry := logEntries[i] + + lsn, err := storage.ParseLSN(logEntry.Name()) + if err != nil { + return fmt.Errorf("parse LSN: %w", err) + } + if lsn <= mgr.committedLSN { + // Found some on-disk log entries older than or equal to committedLSN. They might be + // referenced by other transactions before restart. Eventually, they'll be removed in + // the main loop. + if mgr.oldestLSN, err = storage.ParseLSN(logEntries[0].Name()); err != nil { + return fmt.Errorf("parse oldest LSN: %w", err) + } + break + } + if err := mgr.deleteLogEntry(mgr.ctx, lsn); err != nil { + return fmt.Errorf("cleaning uncommitted log entry: %w", err) + } } } @@ -3512,11 +3558,23 @@ func (mgr *TransactionManager) applyReferenceTransaction(ctx context.Context, ch return nil } -// commitLogEntry commits the transaction to the write-ahead log. It first writes the transaction's manifest file -// into the log entry's directory. Afterwards it moves the log entry's directory from the staging area to its final -// place in the write-ahead log. -func (mgr *TransactionManager) commitLogEntry(ctx context.Context, objectDependencies map[git.ObjectID]struct{}, logEntry *gitalypb.LogEntry, logEntryPath string) error { - defer trace.StartRegion(ctx, "commitLogEntry").End() +// proposeLogEntry proposes a log etnry of a transaction to the write-ahead log. It first writes the transaction's +// manifest file into the log entry's directory. Second, it sends the log entry to other cluster members if needed. +// Afterwards it moves the log entry's directory from the staging area to its final place in the write-ahead log. +func (mgr *TransactionManager) proposeLogEntry(ctx context.Context, objectDependencies map[git.ObjectID]struct{}, logEntry *gitalypb.LogEntry, logEntryPath string) error { + nextLSN := mgr.appendedLSN + 1 + if err := mgr.appendLogEntry(ctx, nextLSN, logEntry, logEntryPath); err != nil { + return fmt.Errorf("append log entry: %w", err) + } + if err := mgr.commitLogEntry(ctx, nextLSN, objectDependencies); err != nil { + return fmt.Errorf("commit log entry: %w", err) + } + return nil +} + +// appendLogEntry appends the transaction to the write-ahead log. +func (mgr *TransactionManager) appendLogEntry(ctx context.Context, nextLSN storage.LSN, logEntry *gitalypb.LogEntry, logEntryPath string) error { + defer trace.StartRegion(ctx, "appendLogEntry").End() manifestBytes, err := proto.Marshal(logEntry) if err != nil { @@ -3544,7 +3602,6 @@ func (mgr *TransactionManager) commitLogEntry(ctx context.Context, objectDepende return fmt.Errorf("synchronizing WAL directory: %w", err) } - nextLSN := mgr.committedLSN + 1 mgr.testHooks.beforeAppendLogEntry(nextLSN) // Move the log entry from the staging directory into its place in the log. @@ -3571,9 +3628,32 @@ func (mgr *TransactionManager) commitLogEntry(ctx context.Context, objectDepende // After this latch block, the transaction is committed and all subsequent transactions // are guaranteed to read it. + mgr.mutex.Lock() + mgr.appendedLSN = nextLSN + mgr.appendedEntries[mgr.appendedLSN] = logEntry + mgr.mutex.Unlock() + + return nil +} + +// commitLogEntry commits the transaction to the write-ahead log. +func (mgr *TransactionManager) commitLogEntry(ctx context.Context, nextLSN storage.LSN, objectDependencies map[git.ObjectID]struct{}) error { + defer trace.StartRegion(ctx, "commitLogEntry").End() + mgr.testHooks.beforeCommitLogEntry(nextLSN) + + // Persist committed LSN before updating internal states. + if err := mgr.storeCommittedLSN(nextLSN); err != nil { + return fmt.Errorf("persisting committed entry: %w", err) + } + mgr.mutex.Lock() mgr.committedLSN = nextLSN mgr.snapshotLocks[nextLSN] = &snapshotLock{applied: make(chan struct{})} + if _, exist := mgr.appendedEntries[nextLSN]; !exist { + mgr.mutex.Unlock() + return fmt.Errorf("log entry %s not found in the appended list", nextLSN) + } + delete(mgr.appendedEntries, nextLSN) mgr.committedEntries.PushBack(&committedEntry{ lsn: nextLSN, objectDependencies: objectDependencies, @@ -3583,6 +3663,7 @@ func (mgr *TransactionManager) commitLogEntry(ctx context.Context, objectDepende if mgr.consumer != nil { mgr.consumer.NotifyNewTransactions(mgr.storageName, mgr.partitionID, mgr.lowWaterMark(), nextLSN) } + return nil } @@ -4011,6 +4092,11 @@ func (mgr *TransactionManager) storeAppliedLSN(lsn storage.LSN) error { return mgr.setKey(keyAppliedLSN, lsn.ToProto()) } +// storeCommittedLSN stores the partition's committed LSN in the database. +func (mgr *TransactionManager) storeCommittedLSN(lsn storage.LSN) error { + return mgr.setKey(keyCommittedLSN, lsn.ToProto()) +} + // setKey marshals and stores a given protocol buffer message into the database under the given key. func (mgr *TransactionManager) setKey(key []byte, value proto.Message) error { marshaledValue, err := proto.Marshal(value) diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go index 218f09d48e8..d2467b3dab2 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go @@ -47,6 +47,7 @@ func installHooks(mgr *TransactionManager, inflightTransactions *sync.WaitGroup, for destination, source := range map[*func(storage.LSN)]hookFunc{ &mgr.testHooks.beforeApplyLogEntry: hooks.BeforeApplyLogEntry, &mgr.testHooks.beforeAppendLogEntry: hooks.BeforeAppendLogEntry, + &mgr.testHooks.beforeCommitLogEntry: hooks.BeforeCommitLogEntry, &mgr.testHooks.beforeStoreAppliedLSN: hooks.BeforeStoreAppliedLSN, &mgr.testHooks.beforeDeleteLogEntryFiles: hooks.AfterDeleteLogEntry, } { diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go index c9a1a3ad802..725447712ff 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go @@ -346,6 +346,7 @@ func TestTransactionManager(t *testing.T) { subTests := map[string][]transactionTestCase{ "Common": generateCommonTests(t, ctx, setup), "CommittedEntries": generateCommittedEntriesTests(t, setup), + "AppendedEntries": generateAppendedEntriesTests(t, setup), "ModifyReferences": generateModifyReferencesTests(t, setup), "CreateRepository": generateCreateRepositoryTests(t, setup), "DeleteRepository": generateDeleteRepositoryTests(t, setup), @@ -531,6 +532,11 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio }, }, expectedState: StateAssertion{ + Database: DatabaseState{ + // The process crashes before apply but after it's committed. So, only + // committedLSN is persisted. + string(keyCommittedLSN): storage.LSN(1).ToProto(), + }, Directory: gittest.FilesOrReftables(testhelper.DirectoryState{ "/": {Mode: mode.Directory}, "/wal": {Mode: mode.Directory}, @@ -1751,6 +1757,8 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t } i++ } + + require.Empty(t, manager.appendedEntries) } return []transactionTestCase{ @@ -2142,6 +2150,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t }) require.Equal(t, tm.appliedLSN, storage.LSN(3)) require.Equal(t, tm.committedLSN, storage.LSN(3)) + require.Equal(t, tm.appendedLSN, storage.LSN(3)) }), }, expectedState: StateAssertion{ @@ -2268,10 +2277,13 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t logEntryPath := filepath.Join(t.TempDir(), "log_entry") require.NoError(t, os.Mkdir(logEntryPath, mode.Directory)) require.NoError(t, os.WriteFile(filepath.Join(logEntryPath, "1"), []byte(setup.Commits.First.OID+"\n"), mode.File)) - require.NoError(t, tm.commitLogEntry(ctx, map[git.ObjectID]struct{}{setup.Commits.First.OID: {}}, refChangeLogEntry(setup, "refs/heads/branch-3", setup.Commits.First.OID), logEntryPath)) + require.NoError(t, tm.proposeLogEntry(ctx, map[git.ObjectID]struct{}{setup.Commits.First.OID: {}}, refChangeLogEntry(setup, "refs/heads/branch-3", setup.Commits.First.OID), logEntryPath)) RequireDatabase(t, ctx, tm.db, DatabaseState{ string(keyAppliedLSN): storage.LSN(3).ToProto(), + // Because this is an out-of-band insertion, the committedLSN is updated + // but new log entry is not applied until waken up. + string(keyCommittedLSN): storage.LSN(4).ToProto(), }) // Transaction 2 and 3 are left-over. testhelper.RequireDirectoryState(t, tm.stateDirectory, "", testhelper.DirectoryState{ @@ -2298,11 +2310,13 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t }) require.Equal(t, tm.appliedLSN, storage.LSN(4)) require.Equal(t, tm.committedLSN, storage.LSN(4)) + require.Equal(t, tm.appendedLSN, storage.LSN(4)) }), }, expectedState: StateAssertion{ Database: DatabaseState{ - string(keyAppliedLSN): storage.LSN(4).ToProto(), + string(keyAppliedLSN): storage.LSN(4).ToProto(), + string(keyCommittedLSN): storage.LSN(4).ToProto(), }, Repositories: RepositoryStates{ setup.RelativePath: { @@ -2324,6 +2338,305 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t } } +func generateAppendedEntriesTests(t *testing.T, setup testTransactionSetup) []transactionTestCase { + assertAppendedEntries := func(t *testing.T, manager *TransactionManager, expectedList []storage.LSN) { + actual := manager.appendedEntries + assert.Equalf(t, len(expectedList), len(actual), "appended entries not matched") + + for _, lsn := range expectedList { + expectedEntry, err := manager.readLogEntry(lsn) + assert.NoError(t, err) + testhelper.ProtoEqualAssert(t, expectedEntry, actual[lsn]) + } + } + + return []transactionTestCase{ + { + desc: "manager has just initialized", + steps: steps{ + StartManager{Hooks: testTransactionHooks{ + BeforeCommitLogEntry: func(c hookContext) { + assert.Fail(t, "there shouldn't be any committed entry") + }, + }}, + AssertManager{}, + AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + require.Equal(t, storage.LSN(0), tm.appendedLSN) + assertAppendedEntries(t, tm, nil) + }), + CloseManager{}, + }, + }, + { + desc: "appended entries are removed after committed", + steps: steps{ + StartManager{Hooks: testTransactionHooks{ + BeforeCommitLogEntry: func(c hookContext) { + // It is not the best solution. But as we intercept to assert an + // intermediate state, there is no cleaner way than using hooks. + lsn := c.lsn + manager := c.manager + switch lsn { + case storage.LSN(1): + assert.Equal(t, storage.LSN(1), manager.appendedLSN) + assert.Equal(t, storage.LSN(0), manager.committedLSN) + assertAppendedEntries(t, manager, []storage.LSN{1}) + case storage.LSN(2): + assert.Equal(t, storage.LSN(2), manager.appendedLSN) + assert.Equal(t, storage.LSN(1), manager.committedLSN) + assertAppendedEntries(t, manager, []storage.LSN{2}) + default: + assert.Fail(t, "there shouldn't be another committed entry") + } + }, + }}, + Begin{ + TransactionID: 1, + RelativePaths: []string{setup.RelativePath}, + }, + Commit{ + TransactionID: 1, + ReferenceUpdates: git.ReferenceUpdates{ + "refs/heads/branch-1": {OldOID: setup.ObjectHash.ZeroOID, NewOID: setup.Commits.First.OID}, + }, + }, + AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + require.Equal(t, storage.LSN(1), tm.appendedLSN) + require.Equal(t, storage.LSN(1), tm.committedLSN) + assertAppendedEntries(t, tm, nil) + }), + Begin{ + TransactionID: 2, + RelativePaths: []string{setup.RelativePath}, + ExpectedSnapshotLSN: 1, + }, + AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + require.Equal(t, storage.LSN(1), tm.appendedLSN) + require.Equal(t, storage.LSN(1), tm.committedLSN) + assertAppendedEntries(t, tm, nil) + }), + Commit{ + TransactionID: 2, + ReferenceUpdates: git.ReferenceUpdates{ + "refs/heads/main": {OldOID: setup.ObjectHash.ZeroOID, NewOID: setup.Commits.First.OID}, + }, + }, + AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + require.Equal(t, storage.LSN(2), tm.appendedLSN) + require.Equal(t, storage.LSN(2), tm.committedLSN) + assertAppendedEntries(t, tm, nil) + }), + CloseManager{}, + }, + expectedState: StateAssertion{ + Database: DatabaseState{ + string(keyAppliedLSN): storage.LSN(2).ToProto(), + }, + // The appended-but-not-committed log entry from transaction 1 is also removed. + Directory: testhelper.DirectoryState{ + "/": {Mode: mode.Directory}, + "/wal": {Mode: mode.Directory}, + }, + Repositories: RepositoryStates{ + setup.RelativePath: { + DefaultBranch: "refs/heads/main", + References: gittest.FilesOrReftables( + &ReferencesState{ + FilesBackend: &FilesBackendState{ + LooseReferences: map[git.ReferenceName]git.ObjectID{ + "refs/heads/main": setup.Commits.First.OID, + "refs/heads/branch-1": setup.Commits.First.OID, + }, + }, + }, &ReferencesState{ + ReftableBackend: &ReftableBackendState{ + Tables: []ReftableTable{ + { + MinIndex: 1, + MaxIndex: 1, + References: []git.Reference{ + { + Name: "HEAD", + Target: "refs/heads/main", + IsSymbolic: true, + }, + }, + }, + { + MinIndex: 2, + MaxIndex: 2, + References: []git.Reference{ + { + Name: "refs/heads/branch-1", + Target: setup.Commits.First.OID.String(), + }, + }, + }, + { + MinIndex: 3, + MaxIndex: 3, + References: []git.Reference{ + { + Name: "refs/heads/main", + Target: setup.Commits.First.OID.String(), + }, + }, + }, + }, + }, + }, + ), + }, + }, + }, + }, + { + desc: "transaction manager crashes after appending", + steps: steps{ + StartManager{ + Hooks: testTransactionHooks{ + BeforeCommitLogEntry: func(c hookContext) { + assert.Equal(t, storage.LSN(1), c.manager.appendedLSN) + assertAppendedEntries(t, c.manager, []storage.LSN{1}) + simulateCrashHook()(c) + }, + WaitForTransactionsWhenClosing: true, + }, + ExpectedError: errSimulatedCrash, + }, + Begin{ + TransactionID: 1, + RelativePaths: []string{setup.RelativePath}, + }, + AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + require.Equal(t, storage.LSN(0), tm.appendedLSN) + require.Equal(t, storage.LSN(0), tm.committedLSN) + assertAppendedEntries(t, tm, nil) + }), + Commit{ + TransactionID: 1, + ReferenceUpdates: git.ReferenceUpdates{ + "refs/heads/branch-1": {OldOID: setup.ObjectHash.ZeroOID, NewOID: setup.Commits.First.OID}, + }, + ExpectedError: storage.ErrTransactionProcessingStopped, + }, + AssertManager{ + ExpectedError: errSimulatedCrash, + }, + AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + // Appended entries are persisted. + testhelper.RequireDirectoryState(t, tm.stateDirectory, "", gittest.FilesOrReftables(testhelper.DirectoryState{ + "/": {Mode: mode.Directory}, + "/wal": {Mode: mode.Directory}, + "/wal/0000000000001": {Mode: mode.Directory}, + "/wal/0000000000001/MANIFEST": manifestDirectoryEntry(&gitalypb.LogEntry{ + RelativePath: setup.RelativePath, + ReferenceTransactions: []*gitalypb.LogEntry_ReferenceTransaction{ + { + Changes: []*gitalypb.LogEntry_ReferenceTransaction_Change{ + { + ReferenceName: []byte("refs/heads/branch-1"), + NewOid: []byte(setup.Commits.First.OID), + }, + }, + }, + }, + Operations: []*gitalypb.LogEntry_Operation{ + { + Operation: &gitalypb.LogEntry_Operation_CreateHardLink_{ + CreateHardLink: &gitalypb.LogEntry_Operation_CreateHardLink{ + SourcePath: []byte("1"), + DestinationPath: []byte(filepath.Join(setup.RelativePath, "refs/heads/branch-1")), + }, + }, + }, + }, + }), + "/wal/0000000000001/1": {Mode: mode.File, Content: []byte(setup.Commits.First.OID + "\n")}, + }, buildReftableDirectory(map[int][]git.ReferenceUpdates{ + 1: {{"refs/heads/branch-1": git.ReferenceUpdate{NewOID: setup.Commits.First.OID}}}, + }))) + }), + StartManager{}, + AssertManager{}, + // No-op, just to ensure the manager is initialized. + AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + // Both of them are set to 0 now. + require.Equal(t, storage.LSN(0), tm.appendedLSN) + require.Equal(t, storage.LSN(0), tm.committedLSN) + // Appended entries are removed now. + testhelper.RequireDirectoryState(t, tm.stateDirectory, "", testhelper.DirectoryState{ + "/": {Mode: mode.Directory}, + "/wal": {Mode: mode.Directory}, + }) + assertAppendedEntries(t, tm, nil) + }), + Begin{ + TransactionID: 2, + RelativePaths: []string{setup.RelativePath}, + }, + Commit{ + TransactionID: 2, + ReferenceUpdates: git.ReferenceUpdates{ + "refs/heads/branch-1": {OldOID: setup.ObjectHash.ZeroOID, NewOID: setup.Commits.Second.OID}, + }, + }, + AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + // Apply new commit only. The prior commit was rejected. + require.Equal(t, storage.LSN(1), tm.appendedLSN) + require.Equal(t, storage.LSN(1), tm.committedLSN) + assertAppendedEntries(t, tm, nil) + }), + }, + expectedState: StateAssertion{ + Database: DatabaseState{ + string(keyAppliedLSN): storage.LSN(1).ToProto(), + }, + Repositories: RepositoryStates{ + setup.RelativePath: { + DefaultBranch: "refs/heads/main", + References: gittest.FilesOrReftables( + &ReferencesState{ + FilesBackend: &FilesBackendState{ + LooseReferences: map[git.ReferenceName]git.ObjectID{ + "refs/heads/branch-1": setup.Commits.Second.OID, + }, + }, + }, &ReferencesState{ + ReftableBackend: &ReftableBackendState{ + Tables: []ReftableTable{ + { + MinIndex: 1, + MaxIndex: 1, + References: []git.Reference{ + { + Name: "HEAD", + Target: "refs/heads/main", + IsSymbolic: true, + }, + }, + }, + { + MinIndex: 2, + MaxIndex: 2, + References: []git.Reference{ + { + Name: "refs/heads/branch-1", + Target: setup.Commits.Second.OID.String(), + }, + }, + }, + }, + }, + }, + ), + }, + }, + }, + }, + } +} + // BenchmarkTransactionManager benchmarks the transaction throughput of the TransactionManager at various levels // of concurrency and transaction sizes. func BenchmarkTransactionManager(b *testing.B) { -- GitLab From 1d048f805131a9ddfe3ab0a27032459561ea6170 Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Mon, 21 Oct 2024 14:00:29 +0700 Subject: [PATCH 06/19] storagemgr: Extract transaction preparation steps out This commit extracts a portion of transaction preparation out as independent functions. There are essentially no changes in terms of functionality. Those independent functions will be used by Raft in later commits. When Raft is enabled, cluster-wide changes are recorded as internal log entries. Those entries use the same LSN sequence as normal log entries. We don't want to maintain two parallel indices systems. Hence, The raft manager must back-filled entries back to the WAL transaction manager. Internal entries have associated LSN and could not be changed. As a result, the transaction manager needs a "fast-track" to insert certain log entries at desirable locations. --- .../partition/transaction_manager.go | 211 ++++++++++-------- 1 file changed, 114 insertions(+), 97 deletions(-) diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager.go index 54d3ce4acdf..0f83e047636 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager.go @@ -1134,8 +1134,6 @@ func (mgr *TransactionManager) commit(ctx context.Context, transaction *Transact span, ctx := tracing.StartSpanIfHasParent(ctx, "transaction.Commit", nil) defer span.Finish() - transaction.result = make(resultChannel, 1) - if transaction.repositoryTarget() && !transaction.repositoryExists { // Determine if the repository was created in this transaction and stage its state // for committing if so. @@ -1149,6 +1147,43 @@ func (mgr *TransactionManager) commit(ctx context.Context, transaction *Transact } } + if err := mgr.prepareCommit(ctx, transaction); err != nil { + return err + } + + if err := func() error { + defer trace.StartRegion(ctx, "commit queue").End() + transaction.metrics.commitQueueDepth.Inc() + defer transaction.metrics.commitQueueDepth.Dec() + defer prometheus.NewTimer(mgr.metrics.commitQueueWaitSeconds).ObserveDuration() + + select { + case mgr.admissionQueue <- transaction: + transaction.admitted = true + return nil + case <-ctx.Done(): + return ctx.Err() + case <-mgr.closing: + return storage.ErrTransactionProcessingStopped + } + }(); err != nil { + return err + } + + defer trace.StartRegion(ctx, "result wait").End() + select { + case err := <-transaction.result: + return unwrapExpectedError(err) + case <-ctx.Done(): + return ctx.Err() + case <-mgr.closed: + return storage.ErrTransactionProcessingStopped + } +} + +func (mgr *TransactionManager) prepareCommit(ctx context.Context, transaction *Transaction) error { + transaction.result = make(resultChannel, 1) + // Create a directory to store all staging files. if err := os.Mkdir(transaction.walFilesPath(), mode.Directory); err != nil { return fmt.Errorf("create wal files directory: %w", err) @@ -1270,34 +1305,7 @@ func (mgr *TransactionManager) commit(ctx context.Context, transaction *Transact } } - if err := func() error { - defer trace.StartRegion(ctx, "commit queue").End() - transaction.metrics.commitQueueDepth.Inc() - defer transaction.metrics.commitQueueDepth.Dec() - defer prometheus.NewTimer(mgr.metrics.commitQueueWaitSeconds).ObserveDuration() - - select { - case mgr.admissionQueue <- transaction: - transaction.admitted = true - return nil - case <-ctx.Done(): - return ctx.Err() - case <-mgr.closing: - return storage.ErrTransactionProcessingStopped - } - }(); err != nil { - return err - } - - defer trace.StartRegion(ctx, "result wait").End() - select { - case err := <-transaction.result: - return unwrapExpectedError(err) - case <-ctx.Done(): - return ctx.Err() - case <-mgr.closed: - return storage.ErrTransactionProcessingStopped - } + return nil } // replaceObjectDirectory replaces the snapshot repository's object directory @@ -2247,95 +2255,104 @@ func (mgr *TransactionManager) processTransaction(ctx context.Context) (returned span, ctx := tracing.StartSpanIfHasParent(ctx, "transaction.processTransaction", nil) defer span.Finish() - if err := func() (commitErr error) { - repositoryExists, err := mgr.doesRepositoryExist(ctx, transaction.relativePath) + if err := func() error { + logEntry, logEntryPath, err := mgr.packageLogEntry(ctx, transaction) if err != nil { - return fmt.Errorf("does repository exist: %w", err) + return err } - logEntry := &gitalypb.LogEntry{ - RelativePath: transaction.relativePath, - } + return mgr.proposeLogEntry(ctx, transaction.objectDependencies, logEntry, logEntryPath) + }(); err != nil { + transaction.result <- err + return nil + } - if transaction.repositoryCreation != nil && repositoryExists { - return ErrRepositoryAlreadyExists - } else if transaction.repositoryCreation == nil && !repositoryExists { - return storage.ErrRepositoryNotFound - } + mgr.awaitingTransactions[mgr.committedLSN] = transaction.result - alternateRelativePath, err := mgr.verifyAlternateUpdate(ctx, transaction) - if err != nil { - return fmt.Errorf("verify alternate update: %w", err) - } + return nil +} - if err := mgr.setupStagingRepository(ctx, transaction, alternateRelativePath); err != nil { - return fmt.Errorf("setup staging snapshot: %w", err) - } +// packageLogEntry verifies the transaction and packaged its content to respective log entry. +func (mgr *TransactionManager) packageLogEntry(ctx context.Context, transaction *Transaction) (*gitalypb.LogEntry, string, error) { + repositoryExists, err := mgr.doesRepositoryExist(ctx, transaction.relativePath) + if err != nil { + return nil, "", fmt.Errorf("does repository exist: %w", err) + } - // Verify that all objects this transaction depends on are present in the repository. The dependency - // objects are the reference tips set in the transaction and the objects the transaction's packfile - // is based on. If an object dependency is missing, the transaction is aborted as applying it would - // result in repository corruption. - if err := mgr.verifyObjectsExist(ctx, transaction.stagingRepository, transaction.objectDependencies); err != nil { - return fmt.Errorf("verify object dependencies: %w", err) - } + logEntry := &gitalypb.LogEntry{ + RelativePath: transaction.relativePath, + } - if transaction.repositoryCreation == nil && transaction.runHousekeeping == nil && !transaction.deleteRepository { - logEntry.ReferenceTransactions, err = mgr.verifyReferences(ctx, transaction) - if err != nil { - return fmt.Errorf("verify references: %w", err) - } - } + if transaction.repositoryCreation != nil && repositoryExists { + return nil, "", ErrRepositoryAlreadyExists + } else if transaction.repositoryCreation == nil && !repositoryExists { + return nil, "", storage.ErrRepositoryNotFound + } - if transaction.customHooksUpdated { - // Log a deletion of the existing custom hooks so they are removed before the - // new ones are put in place. - if err := transaction.walEntry.RecordDirectoryRemoval( - mgr.storagePath, - filepath.Join(transaction.relativePath, repoutil.CustomHooksDir), - ); err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("record custom hook removal: %w", err) - } - } + alternateRelativePath, err := mgr.verifyAlternateUpdate(ctx, transaction) + if err != nil { + return nil, "", fmt.Errorf("verify alternate update: %w", err) + } - if transaction.deleteRepository { - logEntry.RepositoryDeletion = &gitalypb.LogEntry_RepositoryDeletion{} + if err := mgr.setupStagingRepository(ctx, transaction, alternateRelativePath); err != nil { + return nil, "", fmt.Errorf("setup staging snapshot: %w", err) + } - if err := transaction.walEntry.RecordDirectoryRemoval( - mgr.storagePath, - transaction.relativePath, - ); err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("record repository removal: %w", err) - } + // Verify that all objects this transaction depends on are present in the repository. The dependency + // objects are the reference tips set in the transaction and the objects the transaction's packfile + // is based on. If an object dependency is missing, the transaction is aborted as applying it would + // result in repository corruption. + if err := mgr.verifyObjectsExist(ctx, transaction.stagingRepository, transaction.objectDependencies); err != nil { + return nil, "", fmt.Errorf("verify object dependencies: %w", err) + } - if err := transaction.KV().Delete(relativePathKey(transaction.relativePath)); err != nil { - return fmt.Errorf("delete relative path: %w", err) - } + if transaction.repositoryCreation == nil && transaction.runHousekeeping == nil && !transaction.deleteRepository { + logEntry.ReferenceTransactions, err = mgr.verifyReferences(ctx, transaction) + if err != nil { + return nil, "", fmt.Errorf("verify references: %w", err) } + } - if transaction.runHousekeeping != nil { - housekeepingEntry, err := mgr.verifyHousekeeping(ctx, transaction) - if err != nil { - return fmt.Errorf("verifying pack refs: %w", err) - } - logEntry.Housekeeping = housekeepingEntry + if transaction.customHooksUpdated { + // Log a deletion of the existing custom hooks so they are removed before the + // new ones are put in place. + if err := transaction.walEntry.RecordDirectoryRemoval( + mgr.storagePath, + filepath.Join(transaction.relativePath, repoutil.CustomHooksDir), + ); err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, "", fmt.Errorf("record custom hook removal: %w", err) } + } + + if transaction.deleteRepository { + logEntry.RepositoryDeletion = &gitalypb.LogEntry_RepositoryDeletion{} - if err := mgr.verifyKeyValueOperations(ctx, transaction); err != nil { - return fmt.Errorf("verify key-value operations: %w", err) + if err := transaction.walEntry.RecordDirectoryRemoval( + mgr.storagePath, + transaction.relativePath, + ); err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, "", fmt.Errorf("record repository removal: %w", err) } - logEntry.Operations = transaction.walEntry.Operations() + if err := transaction.KV().Delete(relativePathKey(transaction.relativePath)); err != nil { + return nil, "", fmt.Errorf("delete relative path: %w", err) + } + } - return mgr.proposeLogEntry(ctx, transaction.objectDependencies, logEntry, transaction.walFilesPath()) - }(); err != nil { - transaction.result <- err - return nil + if transaction.runHousekeeping != nil { + housekeepingEntry, err := mgr.verifyHousekeeping(ctx, transaction) + if err != nil { + return nil, "", fmt.Errorf("verifying pack refs: %w", err) + } + logEntry.Housekeeping = housekeepingEntry } - mgr.awaitingTransactions[mgr.committedLSN] = transaction.result + if err := mgr.verifyKeyValueOperations(ctx, transaction); err != nil { + return nil, "", fmt.Errorf("verify key-value operations: %w", err) + } - return nil + logEntry.Operations = transaction.walEntry.Operations() + return logEntry, transaction.walFilesPath(), nil } // verifyKeyValueOperations checks the key-value operations of the transaction for conflicts and includes -- GitLab From 9212833464afea44cc2199c8f937deed609fded8 Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Mon, 4 Nov 2024 10:00:19 +0700 Subject: [PATCH 07/19] storagemgr: Make RequireRepositories print failing relative path The RequireRepositories test utility adds an additional "require.Failf" statement if it detects a failure in any of the asserting repositories. This statement has an undesirable side-effect: it swallows the stacktrace of the data race. The stacktrace is printed out after the test finishes. "require.Failf" prints failed tests itself and then calls "t.FailNow()" to exit the tests instantly. This commit updates the statement to use "tb.Log" instead. --- internal/gitaly/storage/storagemgr/partition/testhelper_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/gitaly/storage/storagemgr/partition/testhelper_test.go b/internal/gitaly/storage/storagemgr/partition/testhelper_test.go index fac426aa5a5..05bc906871c 100644 --- a/internal/gitaly/storage/storagemgr/partition/testhelper_test.go +++ b/internal/gitaly/storage/storagemgr/partition/testhelper_test.go @@ -606,7 +606,7 @@ func RequireRepositories(tb testing.TB, ctx context.Context, cfg config.Cfg, sto // relative path of the repository that failed. If the call failed the test, // print out the relative path to ease troubleshooting. if tb.Failed() { - require.Failf(tb, "unexpected repository state", "relative path: %q", relativePath) + tb.Log("unexpected repository state", "relative path: %q", relativePath) } }() -- GitLab From 0ceef9acd851414828b40304e28cc6bf1746a27d Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Wed, 23 Oct 2024 11:48:03 +0700 Subject: [PATCH 08/19] raft: Remove NodeID config In the previous integration attempt, Raft used a static node ID as a unique identifier. That approach was deemed ineffective. Gitaly uses "storage" as an encapsulated functional unit. A server might serve requests targeting multiple storage at the same time. Adding another ID system increases the complexity and might conflict with the existing storage system. In addition, in scenarios like a node joins/re-joins, re-using a static node ID might need extra handling to avoid conflicts. As a result, Raft will use an automatic node ID system based on node joining events instead. This commit removes obsolete NodeID config. --- internal/gitaly/config/config.go | 9 ++++++--- internal/gitaly/config/config_test.go | 26 -------------------------- 2 files changed, 6 insertions(+), 29 deletions(-) diff --git a/internal/gitaly/config/config.go b/internal/gitaly/config/config.go index 4f92c006f6a..8b775f39b89 100644 --- a/internal/gitaly/config/config.go +++ b/internal/gitaly/config/config.go @@ -1235,8 +1235,6 @@ type Raft struct { Enabled bool `json:"enabled" toml:"enabled"` // ClusterID is the unique ID of the cluster. It ensures the current node joins the right cluster. ClusterID string `json:"cluster_id" toml:"cluster_id"` - // NodeID is the unique ID of the node. - NodeID uint64 `json:"node_id" toml:"node_id"` // RTTMilliseconds is the maximum round trip between two nodes in the cluster. It's used to // calculate multiple types of timeouts of Raft protocol. RTTMilliseconds uint64 `json:"rtt_milliseconds" toml:"rtt_milliseconds"` @@ -1258,6 +1256,12 @@ const ( RaftDefaultHeartbeatTicks = 2 ) +// DefaultRaftConfig returns a Raft configuration filled with default values. +func DefaultRaftConfig(clusterID string) Raft { + r := Raft{Enabled: true, ClusterID: clusterID} + return r.fulfillDefaults() +} + func (r Raft) fulfillDefaults() Raft { if r.RTTMilliseconds == 0 { r.RTTMilliseconds = RaftDefaultRTT @@ -1287,7 +1291,6 @@ func (r Raft) Validate(transactions Transactions) error { cfgErr = cfgErr. Append(cfgerror.NotEmpty(r.ClusterID), "cluster_id"). - Append(cfgerror.Comparable(r.NodeID).GreaterThan(0), "node_id"). Append(cfgerror.Comparable(r.RTTMilliseconds).GreaterThan(0), "rtt_millisecond"). Append(cfgerror.Comparable(r.ElectionTicks).GreaterThan(0), "election_rtt"). Append(cfgerror.Comparable(r.HeartbeatTicks).GreaterThan(0), "heartbeat_rtt") diff --git a/internal/gitaly/config/config_test.go b/internal/gitaly/config/config_test.go index dcc94004ad1..b49490bde7d 100644 --- a/internal/gitaly/config/config_test.go +++ b/internal/gitaly/config/config_test.go @@ -2802,7 +2802,6 @@ func TestRaftConfig_Validate(t *testing.T) { cfgRaft: Raft{ Enabled: true, ClusterID: "4f04a0e2-0db8-4bfa-b846-01b5b4a093fb", - NodeID: 1, RTTMilliseconds: 200, ElectionTicks: 20, HeartbeatTicks: 2, @@ -2814,7 +2813,6 @@ func TestRaftConfig_Validate(t *testing.T) { cfgRaft: Raft{ Enabled: true, ClusterID: "", - NodeID: 1, RTTMilliseconds: 200, ElectionTicks: 20, HeartbeatTicks: 2, @@ -2832,7 +2830,6 @@ func TestRaftConfig_Validate(t *testing.T) { cfgRaft: Raft{ Enabled: true, ClusterID: "1234", - NodeID: 1, RTTMilliseconds: 200, ElectionTicks: 20, HeartbeatTicks: 2, @@ -2845,30 +2842,11 @@ func TestRaftConfig_Validate(t *testing.T) { ), }, }, - { - name: "invalid node ID", - cfgRaft: Raft{ - Enabled: true, - ClusterID: "4f04a0e2-0db8-4bfa-b846-01b5b4a093fb", - NodeID: 0, - RTTMilliseconds: 200, - ElectionTicks: 20, - HeartbeatTicks: 2, - }, - cfgTransactions: Transactions{Enabled: true}, - expectedErr: cfgerror.ValidationErrors{ - cfgerror.NewValidationError( - fmt.Errorf("%w: 0 is not greater than 0", cfgerror.ErrNotInRange), - "node_id", - ), - }, - }, { name: "invalid RTT", cfgRaft: Raft{ Enabled: true, ClusterID: "4f04a0e2-0db8-4bfa-b846-01b5b4a093fb", - NodeID: 1, RTTMilliseconds: 0, ElectionTicks: 20, HeartbeatTicks: 2, @@ -2886,7 +2864,6 @@ func TestRaftConfig_Validate(t *testing.T) { cfgRaft: Raft{ Enabled: true, ClusterID: "4f04a0e2-0db8-4bfa-b846-01b5b4a093fb", - NodeID: 1, RTTMilliseconds: 200, ElectionTicks: 0, HeartbeatTicks: 2, @@ -2904,7 +2881,6 @@ func TestRaftConfig_Validate(t *testing.T) { cfgRaft: Raft{ Enabled: true, ClusterID: "4f04a0e2-0db8-4bfa-b846-01b5b4a093fb", - NodeID: 1, RTTMilliseconds: 200, ElectionTicks: 20, HeartbeatTicks: 0, @@ -2922,7 +2898,6 @@ func TestRaftConfig_Validate(t *testing.T) { cfgRaft: Raft{ Enabled: true, ClusterID: "4f04a0e2-0db8-4bfa-b846-01b5b4a093fb", - NodeID: 1, RTTMilliseconds: 200, ElectionTicks: 20, HeartbeatTicks: 2, @@ -2958,7 +2933,6 @@ initial_members = {1 = "localhost:4001", 2 = "localhost:4002", 3 = "localhost:40 Raft: Raft{ Enabled: true, ClusterID: "7766d7c1-7266-4bc9-9dad-5ee8617c455b", - NodeID: 1, RTTMilliseconds: 200, ElectionTicks: 20, HeartbeatTicks: 0, -- GitLab From 5ce89caee82eb6bc114ccd837ee305cb065187f8 Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Wed, 23 Oct 2024 11:52:45 +0700 Subject: [PATCH 09/19] raft: Add Raft protobuf definition This commit adds essential protobuf definitions that will soon be used by the Raft manager. The protobuf file is named "cluster.proto" to avoid naming conflict with the imported ectd/raft package. In the protobuf file, we re-declared some core etcd/raft package's protobuf due to incompatibility with the upstream. The upstream uses proto2 syntax and a fairly outdated protobuf library. Gitaly uses proto3 syntax and up-to-date protobuf library. As a result, Gitaly's existing protobuf infras can't marshal/unmarshal imported ones. They are perfectly fine for internal representation, though. This commit also adds Raft metadata to WAL LogEntry. --- proto/cluster.proto | 107 ++++++ proto/go/gitalypb/cluster.pb.go | 600 ++++++++++++++++++++++++++++++++ proto/go/gitalypb/log.pb.go | 353 +++++++++++-------- proto/go/gitalypb/protolist.go | 1 + proto/log.proto | 13 + 5 files changed, 936 insertions(+), 138 deletions(-) create mode 100644 proto/cluster.proto create mode 100644 proto/go/gitalypb/cluster.pb.go diff --git a/proto/cluster.proto b/proto/cluster.proto new file mode 100644 index 00000000000..6e39ac1015d --- /dev/null +++ b/proto/cluster.proto @@ -0,0 +1,107 @@ +syntax = "proto3"; + +package gitaly; +import "log.proto"; + +option go_package = "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"; + +// RaftMessageType defines the types of messages that can be used within the Raft protocol. +// These types help in identifying the nature of the message being processed. +enum RaftMessageType { + // UNSPECIFIED is used to indicate unspecified message type enum. + UNSPECIFIED = 0; // protolint:disable:this ENUM_FIELD_NAMES_PREFIX + // NORMAL represents a standard Raft log entry proposed by the application. + NORMAL = 1; // protolint:disable:this ENUM_FIELD_NAMES_PREFIX + // VERIFICATION refers to a special type of entry used for verifying state + // without altering it, ensuring the integrity of operations and logs. + VERIFICATION = 2; // protolint:disable:this ENUM_FIELD_NAMES_PREFIX + // CONFIG_CHANGE signifies a change in the configuration of the Raft cluster, + // typically involving node additions or removals. + CONFIG_CHANGE = 3; // protolint:disable:this ENUM_FIELD_NAMES_PREFIX +} + +// RaftMessageV1 serves as a wrapper for messages exchanged in the Raft protocol, +// encapsulating essential information such as the log entry and related metadata. +message RaftMessageV1 { + // ReferencedLogData holds a reference path to the log data stored externally, + // which can be used to access large log entries without including them directly. + message ReferencedLogData { + // path represents the external storage location of the log data. + bytes path = 1; + } + // PackedLogData contains serialized log data including log entry itself and + // all attached files in the log entry directory. Those data are exchanged at + // the Transport layer before sending after after receiving messages. Hence, + // they are transparent to the core Raft engine. + message PackedLogData { + // data is serialized form of the log entry data. Transport implementations + // can choose to populate this data or read the data directly on disk. The + // latter approach is recommended. + bytes data = 1; + } + + // id is unique identifier for the Raft message. This ID is generated by an + // in-memory revent registry. Raft uses this ID to notify the committment + // status of a log entry. + uint64 id = 1; + + // cluster_id is the identifier of the Raft cluster to which this message belongs. + string cluster_id = 2; + + // authority_name is the storage name of storage that creates a partition. + string authority_name = 3; + + // partition_id is the local incremental ID of the specific partition within a + // storage. (authority_name, partition_id) can be used as a unique identifier + // of a partition across the cluster. + uint64 partition_id = 4; + + // log_entry is the actual log entry being processed or transmitted. + LogEntry log_entry = 5; + + // log_data holds files inside log entry dir in one of two possible forms: + // referenced or packed. + oneof log_data { + // referenced represents reference to on-disk log data. + ReferencedLogData referenced = 6; + + // packed represents packed and serialized log data. + PackedLogData packed = 7; + } +} + +// RaftHardStateV1 is a wrapper for raftpb.HardState. The upstream uses proto2 +// syntax while Gitaly uses proto3. In addition, the protobuf package in +// upstream is outdated. The generated structs are not compatible with Gitaly's +// protobuf utilities. +// Source: +// https://github.com/etcd-io/raft/blob/12f0e5dc1b5bfff9bc6886ef1be4cba19495d6f2/raftpb/raft.proto#L110-114 +message RaftHardStateV1 { + // term represents the current term of the raft group. + uint64 term = 1; + // vote represents the vote of the raft group. + uint64 vote = 2; + // commit represents the latest commit index of the raft group. + uint64 commit = 3; +} + +// RaftConfStateV1 is a wrapper for raftpb.ConfState. For more information, +// please refer to RaftHardStateV1. Source: +// https://github.com/etcd-io/raft/blob/12f0e5dc1b5bfff9bc6886ef1be4cba19495d6f2/raftpb/raft.proto#L136 +message RaftConfStateV1 { + // voters in the incoming config. (If the configuration is not joint, + // then the outgoing config is empty). + repeated uint64 voters = 1; + // learners in the incoming config. + repeated uint64 learners = 2; + // voters_outgoing in the outgoing config. + repeated uint64 voters_outgoing = 3; // protolint:disable:this REPEATED_FIELD_NAMES_PLURALIZED + // learners_next is the nodes that will become learners when the outgoing + // config is removed. These nodes are necessarily currently in nodes_joint (or + // they would have been added to the incoming config right away). + repeated uint64 learners_next = 4; // protolint:disable:this REPEATED_FIELD_NAMES_PLURALIZED + // auto_leave is set when the config is joint and Raft will automatically + // transition into the final config (i.e. remove the outgoing config) when + // this is safe. + bool auto_leave = 5; +} diff --git a/proto/go/gitalypb/cluster.pb.go b/proto/go/gitalypb/cluster.pb.go new file mode 100644 index 00000000000..c77541b260f --- /dev/null +++ b/proto/go/gitalypb/cluster.pb.go @@ -0,0 +1,600 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.35.1 +// protoc v4.23.1 +// source: cluster.proto + +package gitalypb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// RaftMessageType defines the types of messages that can be used within the Raft protocol. +// These types help in identifying the nature of the message being processed. +type RaftMessageType int32 + +const ( + // UNSPECIFIED is used to indicate unspecified message type enum. + RaftMessageType_UNSPECIFIED RaftMessageType = 0 // protolint:disable:this ENUM_FIELD_NAMES_PREFIX + // NORMAL represents a standard Raft log entry proposed by the application. + RaftMessageType_NORMAL RaftMessageType = 1 // protolint:disable:this ENUM_FIELD_NAMES_PREFIX + // VERIFICATION refers to a special type of entry used for verifying state + // without altering it, ensuring the integrity of operations and logs. + RaftMessageType_VERIFICATION RaftMessageType = 2 // protolint:disable:this ENUM_FIELD_NAMES_PREFIX + // CONFIG_CHANGE signifies a change in the configuration of the Raft cluster, + // typically involving node additions or removals. + RaftMessageType_CONFIG_CHANGE RaftMessageType = 3 // protolint:disable:this ENUM_FIELD_NAMES_PREFIX +) + +// Enum value maps for RaftMessageType. +var ( + RaftMessageType_name = map[int32]string{ + 0: "UNSPECIFIED", + 1: "NORMAL", + 2: "VERIFICATION", + 3: "CONFIG_CHANGE", + } + RaftMessageType_value = map[string]int32{ + "UNSPECIFIED": 0, + "NORMAL": 1, + "VERIFICATION": 2, + "CONFIG_CHANGE": 3, + } +) + +func (x RaftMessageType) Enum() *RaftMessageType { + p := new(RaftMessageType) + *p = x + return p +} + +func (x RaftMessageType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (RaftMessageType) Descriptor() protoreflect.EnumDescriptor { + return file_cluster_proto_enumTypes[0].Descriptor() +} + +func (RaftMessageType) Type() protoreflect.EnumType { + return &file_cluster_proto_enumTypes[0] +} + +func (x RaftMessageType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use RaftMessageType.Descriptor instead. +func (RaftMessageType) EnumDescriptor() ([]byte, []int) { + return file_cluster_proto_rawDescGZIP(), []int{0} +} + +// RaftMessageV1 serves as a wrapper for messages exchanged in the Raft protocol, +// encapsulating essential information such as the log entry and related metadata. +type RaftMessageV1 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // id is unique identifier for the Raft message. This ID is generated by an + // in-memory revent registry. Raft uses this ID to notify the committment + // status of a log entry. + Id uint64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + // cluster_id is the identifier of the Raft cluster to which this message belongs. + ClusterId string `protobuf:"bytes,2,opt,name=cluster_id,json=clusterId,proto3" json:"cluster_id,omitempty"` + // authority_name is the storage name of storage that creates a partition. + AuthorityName string `protobuf:"bytes,3,opt,name=authority_name,json=authorityName,proto3" json:"authority_name,omitempty"` + // partition_id is the local incremental ID of the specific partition within a + // storage. (authority_name, partition_id) can be used as a unique identifier + // of a partition across the cluster. + PartitionId uint64 `protobuf:"varint,4,opt,name=partition_id,json=partitionId,proto3" json:"partition_id,omitempty"` + // log_entry is the actual log entry being processed or transmitted. + LogEntry *LogEntry `protobuf:"bytes,5,opt,name=log_entry,json=logEntry,proto3" json:"log_entry,omitempty"` + // log_data holds files inside log entry dir in one of two possible forms: + // referenced or packed. + // + // Types that are assignable to LogData: + // + // *RaftMessageV1_Referenced + // *RaftMessageV1_Packed + LogData isRaftMessageV1_LogData `protobuf_oneof:"log_data"` +} + +func (x *RaftMessageV1) Reset() { + *x = RaftMessageV1{} + mi := &file_cluster_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RaftMessageV1) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaftMessageV1) ProtoMessage() {} + +func (x *RaftMessageV1) ProtoReflect() protoreflect.Message { + mi := &file_cluster_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaftMessageV1.ProtoReflect.Descriptor instead. +func (*RaftMessageV1) Descriptor() ([]byte, []int) { + return file_cluster_proto_rawDescGZIP(), []int{0} +} + +func (x *RaftMessageV1) GetId() uint64 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *RaftMessageV1) GetClusterId() string { + if x != nil { + return x.ClusterId + } + return "" +} + +func (x *RaftMessageV1) GetAuthorityName() string { + if x != nil { + return x.AuthorityName + } + return "" +} + +func (x *RaftMessageV1) GetPartitionId() uint64 { + if x != nil { + return x.PartitionId + } + return 0 +} + +func (x *RaftMessageV1) GetLogEntry() *LogEntry { + if x != nil { + return x.LogEntry + } + return nil +} + +func (m *RaftMessageV1) GetLogData() isRaftMessageV1_LogData { + if m != nil { + return m.LogData + } + return nil +} + +func (x *RaftMessageV1) GetReferenced() *RaftMessageV1_ReferencedLogData { + if x, ok := x.GetLogData().(*RaftMessageV1_Referenced); ok { + return x.Referenced + } + return nil +} + +func (x *RaftMessageV1) GetPacked() *RaftMessageV1_PackedLogData { + if x, ok := x.GetLogData().(*RaftMessageV1_Packed); ok { + return x.Packed + } + return nil +} + +type isRaftMessageV1_LogData interface { + isRaftMessageV1_LogData() +} + +type RaftMessageV1_Referenced struct { + // referenced represents reference to on-disk log data. + Referenced *RaftMessageV1_ReferencedLogData `protobuf:"bytes,6,opt,name=referenced,proto3,oneof"` +} + +type RaftMessageV1_Packed struct { + // packed represents packed and serialized log data. + Packed *RaftMessageV1_PackedLogData `protobuf:"bytes,7,opt,name=packed,proto3,oneof"` +} + +func (*RaftMessageV1_Referenced) isRaftMessageV1_LogData() {} + +func (*RaftMessageV1_Packed) isRaftMessageV1_LogData() {} + +// RaftHardStateV1 is a wrapper for raftpb.HardState. The upstream uses proto2 +// syntax while Gitaly uses proto3. In addition, the protobuf package in +// upstream is outdated. The generated structs are not compatible with Gitaly's +// protobuf utilities. +// Source: +// https://github.com/etcd-io/raft/blob/12f0e5dc1b5bfff9bc6886ef1be4cba19495d6f2/raftpb/raft.proto#L110-114 +type RaftHardStateV1 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // term represents the current term of the raft group. + Term uint64 `protobuf:"varint,1,opt,name=term,proto3" json:"term,omitempty"` + // vote represents the vote of the raft group. + Vote uint64 `protobuf:"varint,2,opt,name=vote,proto3" json:"vote,omitempty"` + // commit represents the latest commit index of the raft group. + Commit uint64 `protobuf:"varint,3,opt,name=commit,proto3" json:"commit,omitempty"` +} + +func (x *RaftHardStateV1) Reset() { + *x = RaftHardStateV1{} + mi := &file_cluster_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RaftHardStateV1) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaftHardStateV1) ProtoMessage() {} + +func (x *RaftHardStateV1) ProtoReflect() protoreflect.Message { + mi := &file_cluster_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaftHardStateV1.ProtoReflect.Descriptor instead. +func (*RaftHardStateV1) Descriptor() ([]byte, []int) { + return file_cluster_proto_rawDescGZIP(), []int{1} +} + +func (x *RaftHardStateV1) GetTerm() uint64 { + if x != nil { + return x.Term + } + return 0 +} + +func (x *RaftHardStateV1) GetVote() uint64 { + if x != nil { + return x.Vote + } + return 0 +} + +func (x *RaftHardStateV1) GetCommit() uint64 { + if x != nil { + return x.Commit + } + return 0 +} + +// RaftConfStateV1 is a wrapper for raftpb.ConfState. For more information, +// please refer to RaftHardStateV1. Source: +// https://github.com/etcd-io/raft/blob/12f0e5dc1b5bfff9bc6886ef1be4cba19495d6f2/raftpb/raft.proto#L136 +type RaftConfStateV1 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // voters in the incoming config. (If the configuration is not joint, + // then the outgoing config is empty). + Voters []uint64 `protobuf:"varint,1,rep,packed,name=voters,proto3" json:"voters,omitempty"` + // learners in the incoming config. + Learners []uint64 `protobuf:"varint,2,rep,packed,name=learners,proto3" json:"learners,omitempty"` + // voters_outgoing in the outgoing config. + VotersOutgoing []uint64 `protobuf:"varint,3,rep,packed,name=voters_outgoing,json=votersOutgoing,proto3" json:"voters_outgoing,omitempty"` // protolint:disable:this REPEATED_FIELD_NAMES_PLURALIZED + // learners_next is the nodes that will become learners when the outgoing + // config is removed. These nodes are necessarily currently in nodes_joint (or + // they would have been added to the incoming config right away). + LearnersNext []uint64 `protobuf:"varint,4,rep,packed,name=learners_next,json=learnersNext,proto3" json:"learners_next,omitempty"` // protolint:disable:this REPEATED_FIELD_NAMES_PLURALIZED + // auto_leave is set when the config is joint and Raft will automatically + // transition into the final config (i.e. remove the outgoing config) when + // this is safe. + AutoLeave bool `protobuf:"varint,5,opt,name=auto_leave,json=autoLeave,proto3" json:"auto_leave,omitempty"` +} + +func (x *RaftConfStateV1) Reset() { + *x = RaftConfStateV1{} + mi := &file_cluster_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RaftConfStateV1) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaftConfStateV1) ProtoMessage() {} + +func (x *RaftConfStateV1) ProtoReflect() protoreflect.Message { + mi := &file_cluster_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaftConfStateV1.ProtoReflect.Descriptor instead. +func (*RaftConfStateV1) Descriptor() ([]byte, []int) { + return file_cluster_proto_rawDescGZIP(), []int{2} +} + +func (x *RaftConfStateV1) GetVoters() []uint64 { + if x != nil { + return x.Voters + } + return nil +} + +func (x *RaftConfStateV1) GetLearners() []uint64 { + if x != nil { + return x.Learners + } + return nil +} + +func (x *RaftConfStateV1) GetVotersOutgoing() []uint64 { + if x != nil { + return x.VotersOutgoing + } + return nil +} + +func (x *RaftConfStateV1) GetLearnersNext() []uint64 { + if x != nil { + return x.LearnersNext + } + return nil +} + +func (x *RaftConfStateV1) GetAutoLeave() bool { + if x != nil { + return x.AutoLeave + } + return false +} + +// ReferencedLogData holds a reference path to the log data stored externally, +// which can be used to access large log entries without including them directly. +type RaftMessageV1_ReferencedLogData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // path represents the external storage location of the log data. + Path []byte `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` +} + +func (x *RaftMessageV1_ReferencedLogData) Reset() { + *x = RaftMessageV1_ReferencedLogData{} + mi := &file_cluster_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RaftMessageV1_ReferencedLogData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaftMessageV1_ReferencedLogData) ProtoMessage() {} + +func (x *RaftMessageV1_ReferencedLogData) ProtoReflect() protoreflect.Message { + mi := &file_cluster_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaftMessageV1_ReferencedLogData.ProtoReflect.Descriptor instead. +func (*RaftMessageV1_ReferencedLogData) Descriptor() ([]byte, []int) { + return file_cluster_proto_rawDescGZIP(), []int{0, 0} +} + +func (x *RaftMessageV1_ReferencedLogData) GetPath() []byte { + if x != nil { + return x.Path + } + return nil +} + +// PackedLogData contains serialized log data including log entry itself and +// all attached files in the log entry directory. Those data are exchanged at +// the Transport layer before sending after after receiving messages. Hence, +// they are transparent to the core Raft engine. +type RaftMessageV1_PackedLogData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // data is serialized form of the log entry data. Transport implementations + // can choose to populate this data or read the data directly on disk. The + // latter approach is recommended. + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` +} + +func (x *RaftMessageV1_PackedLogData) Reset() { + *x = RaftMessageV1_PackedLogData{} + mi := &file_cluster_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RaftMessageV1_PackedLogData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaftMessageV1_PackedLogData) ProtoMessage() {} + +func (x *RaftMessageV1_PackedLogData) ProtoReflect() protoreflect.Message { + mi := &file_cluster_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaftMessageV1_PackedLogData.ProtoReflect.Descriptor instead. +func (*RaftMessageV1_PackedLogData) Descriptor() ([]byte, []int) { + return file_cluster_proto_rawDescGZIP(), []int{0, 1} +} + +func (x *RaftMessageV1_PackedLogData) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +var File_cluster_proto protoreflect.FileDescriptor + +var file_cluster_proto_rawDesc = []byte{ + 0x0a, 0x0d, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x06, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x1a, 0x09, 0x6c, 0x6f, 0x67, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x22, 0x9b, 0x03, 0x0a, 0x0d, 0x52, 0x61, 0x66, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x56, 0x31, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x61, 0x75, 0x74, + 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61, + 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x2d, 0x0a, + 0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x10, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x49, 0x0a, 0x0a, + 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x27, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x56, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x64, 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, 0x61, 0x48, 0x00, 0x52, 0x0a, 0x72, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x64, 0x12, 0x3d, 0x0a, 0x06, 0x70, 0x61, 0x63, 0x6b, 0x65, + 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, + 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x56, 0x31, 0x2e, 0x50, + 0x61, 0x63, 0x6b, 0x65, 0x64, 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, 0x61, 0x48, 0x00, 0x52, 0x06, + 0x70, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x1a, 0x27, 0x0a, 0x11, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, + 0x6e, 0x63, 0x65, 0x64, 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x70, + 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x1a, + 0x23, 0x0a, 0x0d, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, 0x61, + 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, + 0x64, 0x61, 0x74, 0x61, 0x42, 0x0a, 0x0a, 0x08, 0x6c, 0x6f, 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, + 0x22, 0x51, 0x0a, 0x0f, 0x52, 0x61, 0x66, 0x74, 0x48, 0x61, 0x72, 0x64, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x56, 0x31, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x65, 0x72, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x04, 0x74, 0x65, 0x72, 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x76, 0x6f, 0x74, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x76, 0x6f, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, + 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x63, 0x6f, 0x6d, + 0x6d, 0x69, 0x74, 0x22, 0xb2, 0x01, 0x0a, 0x0f, 0x52, 0x61, 0x66, 0x74, 0x43, 0x6f, 0x6e, 0x66, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x56, 0x31, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x6f, 0x74, 0x65, 0x72, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x04, 0x52, 0x06, 0x76, 0x6f, 0x74, 0x65, 0x72, 0x73, 0x12, + 0x1a, 0x0a, 0x08, 0x6c, 0x65, 0x61, 0x72, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x04, 0x52, 0x08, 0x6c, 0x65, 0x61, 0x72, 0x6e, 0x65, 0x72, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x76, + 0x6f, 0x74, 0x65, 0x72, 0x73, 0x5f, 0x6f, 0x75, 0x74, 0x67, 0x6f, 0x69, 0x6e, 0x67, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x04, 0x52, 0x0e, 0x76, 0x6f, 0x74, 0x65, 0x72, 0x73, 0x4f, 0x75, 0x74, 0x67, + 0x6f, 0x69, 0x6e, 0x67, 0x12, 0x23, 0x0a, 0x0d, 0x6c, 0x65, 0x61, 0x72, 0x6e, 0x65, 0x72, 0x73, + 0x5f, 0x6e, 0x65, 0x78, 0x74, 0x18, 0x04, 0x20, 0x03, 0x28, 0x04, 0x52, 0x0c, 0x6c, 0x65, 0x61, + 0x72, 0x6e, 0x65, 0x72, 0x73, 0x4e, 0x65, 0x78, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x75, 0x74, + 0x6f, 0x5f, 0x6c, 0x65, 0x61, 0x76, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, + 0x75, 0x74, 0x6f, 0x4c, 0x65, 0x61, 0x76, 0x65, 0x2a, 0x53, 0x0a, 0x0f, 0x52, 0x61, 0x66, 0x74, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x55, + 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, + 0x4e, 0x4f, 0x52, 0x4d, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x56, 0x45, 0x52, 0x49, + 0x46, 0x49, 0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x43, 0x4f, + 0x4e, 0x46, 0x49, 0x47, 0x5f, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x10, 0x03, 0x42, 0x34, 0x5a, + 0x32, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, + 0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, + 0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, + 0x79, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_cluster_proto_rawDescOnce sync.Once + file_cluster_proto_rawDescData = file_cluster_proto_rawDesc +) + +func file_cluster_proto_rawDescGZIP() []byte { + file_cluster_proto_rawDescOnce.Do(func() { + file_cluster_proto_rawDescData = protoimpl.X.CompressGZIP(file_cluster_proto_rawDescData) + }) + return file_cluster_proto_rawDescData +} + +var file_cluster_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_cluster_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_cluster_proto_goTypes = []any{ + (RaftMessageType)(0), // 0: gitaly.RaftMessageType + (*RaftMessageV1)(nil), // 1: gitaly.RaftMessageV1 + (*RaftHardStateV1)(nil), // 2: gitaly.RaftHardStateV1 + (*RaftConfStateV1)(nil), // 3: gitaly.RaftConfStateV1 + (*RaftMessageV1_ReferencedLogData)(nil), // 4: gitaly.RaftMessageV1.ReferencedLogData + (*RaftMessageV1_PackedLogData)(nil), // 5: gitaly.RaftMessageV1.PackedLogData + (*LogEntry)(nil), // 6: gitaly.LogEntry +} +var file_cluster_proto_depIdxs = []int32{ + 6, // 0: gitaly.RaftMessageV1.log_entry:type_name -> gitaly.LogEntry + 4, // 1: gitaly.RaftMessageV1.referenced:type_name -> gitaly.RaftMessageV1.ReferencedLogData + 5, // 2: gitaly.RaftMessageV1.packed:type_name -> gitaly.RaftMessageV1.PackedLogData + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_cluster_proto_init() } +func file_cluster_proto_init() { + if File_cluster_proto != nil { + return + } + file_log_proto_init() + file_cluster_proto_msgTypes[0].OneofWrappers = []any{ + (*RaftMessageV1_Referenced)(nil), + (*RaftMessageV1_Packed)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_cluster_proto_rawDesc, + NumEnums: 1, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_cluster_proto_goTypes, + DependencyIndexes: file_cluster_proto_depIdxs, + EnumInfos: file_cluster_proto_enumTypes, + MessageInfos: file_cluster_proto_msgTypes, + }.Build() + File_cluster_proto = out.File + file_cluster_proto_rawDesc = nil + file_cluster_proto_goTypes = nil + file_cluster_proto_depIdxs = nil +} diff --git a/proto/go/gitalypb/log.pb.go b/proto/go/gitalypb/log.pb.go index 01f99c4e6ab..5acd783dafb 100644 --- a/proto/go/gitalypb/log.pb.go +++ b/proto/go/gitalypb/log.pb.go @@ -44,6 +44,8 @@ type LogEntry struct { // operations is an ordered list of operations to run in order to apply // this log entry. Operations []*LogEntry_Operation `protobuf:"bytes,10,rep,name=operations,proto3" json:"operations,omitempty"` + // metadata contains some extra information, mostly Raft-related metadata. + Metadata *LogEntry_Metadata `protobuf:"bytes,11,opt,name=metadata,proto3" json:"metadata,omitempty"` } func (x *LogEntry) Reset() { @@ -111,6 +113,13 @@ func (x *LogEntry) GetOperations() []*LogEntry_Operation { return nil } +func (x *LogEntry) GetMetadata() *LogEntry_Metadata { + if x != nil { + return x.Metadata + } + return nil +} + // LSN serializes a log sequence number. It's used for storing a partition's // applied LSN in the database. // @@ -440,6 +449,64 @@ func (*LogEntry_Operation_SetKey_) isLogEntry_Operation_Operation() {} func (*LogEntry_Operation_DeleteKey_) isLogEntry_Operation_Operation() {} +// Metadata contains some metadata data of the log entry. +type LogEntry_Metadata struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // raft_term is the term sequence of the Raft group finalized this + // log entry. + RaftTerm uint64 `protobuf:"varint,1,opt,name=raft_term,json=raftTerm,proto3" json:"raft_term,omitempty"` + // raft_type is the type of sequence. It's essentially equal to + // raftpb.EntryType. + RaftType uint64 `protobuf:"varint,2,opt,name=raft_type,json=raftType,proto3" json:"raft_type,omitempty"` +} + +func (x *LogEntry_Metadata) Reset() { + *x = LogEntry_Metadata{} + mi := &file_log_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LogEntry_Metadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LogEntry_Metadata) ProtoMessage() {} + +func (x *LogEntry_Metadata) ProtoReflect() protoreflect.Message { + mi := &file_log_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LogEntry_Metadata.ProtoReflect.Descriptor instead. +func (*LogEntry_Metadata) Descriptor() ([]byte, []int) { + return file_log_proto_rawDescGZIP(), []int{0, 4} +} + +func (x *LogEntry_Metadata) GetRaftTerm() uint64 { + if x != nil { + return x.RaftTerm + } + return 0 +} + +func (x *LogEntry_Metadata) GetRaftType() uint64 { + if x != nil { + return x.RaftType + } + return 0 +} + // Change models a single reference change. type LogEntry_ReferenceTransaction_Change struct { state protoimpl.MessageState @@ -461,7 +528,7 @@ type LogEntry_ReferenceTransaction_Change struct { func (x *LogEntry_ReferenceTransaction_Change) Reset() { *x = LogEntry_ReferenceTransaction_Change{} - mi := &file_log_proto_msgTypes[6] + mi := &file_log_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -473,7 +540,7 @@ func (x *LogEntry_ReferenceTransaction_Change) String() string { func (*LogEntry_ReferenceTransaction_Change) ProtoMessage() {} func (x *LogEntry_ReferenceTransaction_Change) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[6] + mi := &file_log_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -524,7 +591,7 @@ type LogEntry_Housekeeping_PackRefs struct { func (x *LogEntry_Housekeeping_PackRefs) Reset() { *x = LogEntry_Housekeeping_PackRefs{} - mi := &file_log_proto_msgTypes[7] + mi := &file_log_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -536,7 +603,7 @@ func (x *LogEntry_Housekeeping_PackRefs) String() string { func (*LogEntry_Housekeeping_PackRefs) ProtoMessage() {} func (x *LogEntry_Housekeeping_PackRefs) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[7] + mi := &file_log_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -578,7 +645,7 @@ type LogEntry_Housekeeping_Repack struct { func (x *LogEntry_Housekeeping_Repack) Reset() { *x = LogEntry_Housekeeping_Repack{} - mi := &file_log_proto_msgTypes[8] + mi := &file_log_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -590,7 +657,7 @@ func (x *LogEntry_Housekeeping_Repack) String() string { func (*LogEntry_Housekeeping_Repack) ProtoMessage() {} func (x *LogEntry_Housekeeping_Repack) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[8] + mi := &file_log_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -637,7 +704,7 @@ type LogEntry_Housekeeping_WriteCommitGraphs struct { func (x *LogEntry_Housekeeping_WriteCommitGraphs) Reset() { *x = LogEntry_Housekeeping_WriteCommitGraphs{} - mi := &file_log_proto_msgTypes[9] + mi := &file_log_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -649,7 +716,7 @@ func (x *LogEntry_Housekeeping_WriteCommitGraphs) String() string { func (*LogEntry_Housekeeping_WriteCommitGraphs) ProtoMessage() {} func (x *LogEntry_Housekeeping_WriteCommitGraphs) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[9] + mi := &file_log_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -684,7 +751,7 @@ type LogEntry_Operation_CreateHardLink struct { func (x *LogEntry_Operation_CreateHardLink) Reset() { *x = LogEntry_Operation_CreateHardLink{} - mi := &file_log_proto_msgTypes[10] + mi := &file_log_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -696,7 +763,7 @@ func (x *LogEntry_Operation_CreateHardLink) String() string { func (*LogEntry_Operation_CreateHardLink) ProtoMessage() {} func (x *LogEntry_Operation_CreateHardLink) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[10] + mi := &file_log_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -747,7 +814,7 @@ type LogEntry_Operation_RemoveDirectoryEntry struct { func (x *LogEntry_Operation_RemoveDirectoryEntry) Reset() { *x = LogEntry_Operation_RemoveDirectoryEntry{} - mi := &file_log_proto_msgTypes[11] + mi := &file_log_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -759,7 +826,7 @@ func (x *LogEntry_Operation_RemoveDirectoryEntry) String() string { func (*LogEntry_Operation_RemoveDirectoryEntry) ProtoMessage() {} func (x *LogEntry_Operation_RemoveDirectoryEntry) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[11] + mi := &file_log_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -796,7 +863,7 @@ type LogEntry_Operation_CreateDirectory struct { func (x *LogEntry_Operation_CreateDirectory) Reset() { *x = LogEntry_Operation_CreateDirectory{} - mi := &file_log_proto_msgTypes[12] + mi := &file_log_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -808,7 +875,7 @@ func (x *LogEntry_Operation_CreateDirectory) String() string { func (*LogEntry_Operation_CreateDirectory) ProtoMessage() {} func (x *LogEntry_Operation_CreateDirectory) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[12] + mi := &file_log_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -852,7 +919,7 @@ type LogEntry_Operation_SetKey struct { func (x *LogEntry_Operation_SetKey) Reset() { *x = LogEntry_Operation_SetKey{} - mi := &file_log_proto_msgTypes[13] + mi := &file_log_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -864,7 +931,7 @@ func (x *LogEntry_Operation_SetKey) String() string { func (*LogEntry_Operation_SetKey) ProtoMessage() {} func (x *LogEntry_Operation_SetKey) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[13] + mi := &file_log_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -906,7 +973,7 @@ type LogEntry_Operation_DeleteKey struct { func (x *LogEntry_Operation_DeleteKey) Reset() { *x = LogEntry_Operation_DeleteKey{} - mi := &file_log_proto_msgTypes[14] + mi := &file_log_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -918,7 +985,7 @@ func (x *LogEntry_Operation_DeleteKey) String() string { func (*LogEntry_Operation_DeleteKey) ProtoMessage() {} func (x *LogEntry_Operation_DeleteKey) ProtoReflect() protoreflect.Message { - mi := &file_log_proto_msgTypes[14] + mi := &file_log_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -945,7 +1012,7 @@ var File_log_proto protoreflect.FileDescriptor var file_log_proto_rawDesc = []byte{ 0x0a, 0x09, 0x6c, 0x6f, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x67, 0x69, 0x74, - 0x61, 0x6c, 0x79, 0x22, 0xe7, 0x0d, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x61, 0x6c, 0x79, 0x22, 0xe4, 0x0e, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x5c, 0x0a, 0x16, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, @@ -967,101 +1034,109 @@ var file_log_proto_rawDesc = []byte{ 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x6f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0xc7, 0x01, 0x0a, 0x14, 0x52, 0x65, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x46, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x54, 0x72, - 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, - 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x1a, 0x67, 0x0a, 0x06, 0x43, 0x68, 0x61, - 0x6e, 0x67, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x72, 0x65, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x65, - 0x77, 0x5f, 0x6f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x6e, 0x65, 0x77, - 0x4f, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x65, 0x77, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6e, 0x65, 0x77, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x1a, 0x14, 0x0a, 0x12, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, - 0x44, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xa6, 0x03, 0x0a, 0x0c, 0x48, 0x6f, 0x75, - 0x73, 0x65, 0x6b, 0x65, 0x65, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x43, 0x0a, 0x09, 0x70, 0x61, 0x63, - 0x6b, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x67, - 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, - 0x6f, 0x75, 0x73, 0x65, 0x6b, 0x65, 0x65, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x63, 0x6b, - 0x52, 0x65, 0x66, 0x73, 0x52, 0x08, 0x70, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x66, 0x73, 0x12, 0x3c, - 0x0a, 0x06, 0x72, 0x65, 0x70, 0x61, 0x63, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, - 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x2e, 0x48, 0x6f, 0x75, 0x73, 0x65, 0x6b, 0x65, 0x65, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, - 0x70, 0x61, 0x63, 0x6b, 0x52, 0x06, 0x72, 0x65, 0x70, 0x61, 0x63, 0x6b, 0x12, 0x5f, 0x0a, 0x13, - 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x5f, 0x67, 0x72, 0x61, - 0x70, 0x68, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x67, 0x69, 0x74, 0x61, - 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x6f, 0x75, 0x73, - 0x65, 0x6b, 0x65, 0x65, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x6f, - 0x6d, 0x6d, 0x69, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, 0x73, 0x52, 0x11, 0x77, 0x72, 0x69, 0x74, - 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, 0x73, 0x1a, 0x2b, 0x0a, - 0x08, 0x50, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x66, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x72, 0x75, - 0x6e, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, - 0x70, 0x72, 0x75, 0x6e, 0x65, 0x64, 0x52, 0x65, 0x66, 0x73, 0x1a, 0x70, 0x0a, 0x06, 0x52, 0x65, - 0x70, 0x61, 0x63, 0x6b, 0x12, 0x1b, 0x0a, 0x09, 0x6e, 0x65, 0x77, 0x5f, 0x66, 0x69, 0x6c, 0x65, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6e, 0x65, 0x77, 0x46, 0x69, 0x6c, 0x65, - 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, - 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x69, 0x73, 0x5f, 0x66, 0x75, 0x6c, - 0x6c, 0x5f, 0x72, 0x65, 0x70, 0x61, 0x63, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, - 0x69, 0x73, 0x46, 0x75, 0x6c, 0x6c, 0x52, 0x65, 0x70, 0x61, 0x63, 0x6b, 0x1a, 0x13, 0x0a, 0x11, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x35, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x69, 0x74, + 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x1a, + 0xc7, 0x01, 0x0a, 0x14, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x46, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, + 0x67, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, 0x69, 0x74, 0x61, + 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, + 0x1a, 0x67, 0x0a, 0x06, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, + 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x0d, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x65, 0x77, 0x5f, 0x6f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x06, 0x6e, 0x65, 0x77, 0x4f, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x65, + 0x77, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, + 0x6e, 0x65, 0x77, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x1a, 0x14, 0x0a, 0x12, 0x52, 0x65, 0x70, + 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x1a, + 0xa6, 0x03, 0x0a, 0x0c, 0x48, 0x6f, 0x75, 0x73, 0x65, 0x6b, 0x65, 0x65, 0x70, 0x69, 0x6e, 0x67, + 0x12, 0x43, 0x0a, 0x09, 0x70, 0x61, 0x63, 0x6b, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x6f, 0x75, 0x73, 0x65, 0x6b, 0x65, 0x65, 0x70, 0x69, + 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x66, 0x73, 0x52, 0x08, 0x70, 0x61, 0x63, + 0x6b, 0x52, 0x65, 0x66, 0x73, 0x12, 0x3c, 0x0a, 0x06, 0x72, 0x65, 0x70, 0x61, 0x63, 0x6b, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, + 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x48, 0x6f, 0x75, 0x73, 0x65, 0x6b, 0x65, 0x65, + 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x70, 0x61, 0x63, 0x6b, 0x52, 0x06, 0x72, 0x65, 0x70, + 0x61, 0x63, 0x6b, 0x12, 0x5f, 0x0a, 0x13, 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x6d, + 0x6d, 0x69, 0x74, 0x5f, 0x67, 0x72, 0x61, 0x70, 0x68, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x2f, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x48, 0x6f, 0x75, 0x73, 0x65, 0x6b, 0x65, 0x65, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, - 0x73, 0x1a, 0xf9, 0x05, 0x0a, 0x09, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x55, 0x0a, 0x10, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x6c, - 0x69, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x67, 0x69, 0x74, 0x61, - 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x48, 0x61, 0x72, 0x64, - 0x4c, 0x69, 0x6e, 0x6b, 0x48, 0x00, 0x52, 0x0e, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x48, 0x61, - 0x72, 0x64, 0x4c, 0x69, 0x6e, 0x6b, 0x12, 0x67, 0x0a, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, - 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, - 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x48, 0x00, 0x52, 0x14, 0x72, 0x65, 0x6d, 0x6f, 0x76, - 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x57, 0x0a, 0x10, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x67, 0x69, 0x74, 0x61, - 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x69, 0x72, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x79, 0x48, 0x00, 0x52, 0x0f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, - 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x3c, 0x0a, 0x07, 0x73, 0x65, 0x74, 0x5f, - 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x67, 0x69, 0x74, 0x61, - 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x48, 0x00, 0x52, 0x06, - 0x73, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x45, 0x0a, 0x0a, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x69, 0x74, - 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4b, 0x65, 0x79, - 0x48, 0x00, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x1a, 0x88, 0x01, - 0x0a, 0x0e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x48, 0x61, 0x72, 0x64, 0x4c, 0x69, 0x6e, 0x6b, - 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, 0x61, 0x74, - 0x68, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x69, 0x6e, 0x5f, 0x73, - 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x12, 0x29, 0x0a, - 0x10, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x61, 0x74, - 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0f, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x74, 0x68, 0x1a, 0x2a, 0x0a, 0x14, 0x52, 0x65, 0x6d, 0x6f, - 0x76, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, - 0x70, 0x61, 0x74, 0x68, 0x1a, 0x39, 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x69, - 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6d, - 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x1a, - 0x30, 0x0a, 0x06, 0x53, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x1a, 0x1d, 0x0a, 0x09, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x42, 0x0b, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x1b, 0x0a, - 0x03, 0x4c, 0x53, 0x4e, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x69, - 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2d, - 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x36, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x70, 0x62, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x52, 0x11, 0x77, 0x72, 0x69, 0x74, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x47, 0x72, + 0x61, 0x70, 0x68, 0x73, 0x1a, 0x2b, 0x0a, 0x08, 0x50, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x66, 0x73, + 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x72, 0x75, 0x6e, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, 0x70, 0x72, 0x75, 0x6e, 0x65, 0x64, 0x52, 0x65, 0x66, + 0x73, 0x1a, 0x70, 0x0a, 0x06, 0x52, 0x65, 0x70, 0x61, 0x63, 0x6b, 0x12, 0x1b, 0x0a, 0x09, 0x6e, + 0x65, 0x77, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, + 0x6e, 0x65, 0x77, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x65, 0x6c, 0x65, + 0x74, 0x65, 0x64, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x0c, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x24, 0x0a, + 0x0e, 0x69, 0x73, 0x5f, 0x66, 0x75, 0x6c, 0x6c, 0x5f, 0x72, 0x65, 0x70, 0x61, 0x63, 0x6b, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x73, 0x46, 0x75, 0x6c, 0x6c, 0x52, 0x65, 0x70, + 0x61, 0x63, 0x6b, 0x1a, 0x13, 0x0a, 0x11, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x6f, 0x6d, 0x6d, + 0x69, 0x74, 0x47, 0x72, 0x61, 0x70, 0x68, 0x73, 0x1a, 0xf9, 0x05, 0x0a, 0x09, 0x4f, 0x70, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x55, 0x0a, 0x10, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x5f, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x29, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x48, 0x61, 0x72, 0x64, 0x4c, 0x69, 0x6e, 0x6b, 0x48, 0x00, 0x52, 0x0e, 0x63, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x48, 0x61, 0x72, 0x64, 0x4c, 0x69, 0x6e, 0x6b, 0x12, 0x67, 0x0a, + 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x79, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, + 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, + 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, + 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x48, 0x00, + 0x52, 0x14, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x57, 0x0a, 0x10, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x2a, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x48, 0x00, 0x52, 0x0f, + 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, + 0x3c, 0x0a, 0x07, 0x73, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x21, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, + 0x4b, 0x65, 0x79, 0x48, 0x00, 0x52, 0x06, 0x73, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x45, 0x0a, + 0x0a, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x4c, 0x6f, 0x67, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x44, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x48, 0x00, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x4b, 0x65, 0x79, 0x1a, 0x88, 0x01, 0x0a, 0x0e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x48, + 0x61, 0x72, 0x64, 0x4c, 0x69, 0x6e, 0x6b, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x5f, 0x69, 0x6e, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x53, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0f, + 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x74, 0x68, 0x1a, + 0x2a, 0x0a, 0x14, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x1a, 0x39, 0x0a, 0x0f, 0x43, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x12, + 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x70, 0x61, + 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x1a, 0x30, 0x0a, 0x06, 0x53, 0x65, 0x74, 0x4b, 0x65, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x1d, 0x0a, 0x09, 0x44, 0x65, 0x6c, 0x65, + 0x74, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x42, 0x0b, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x44, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x74, 0x65, 0x72, 0x6d, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x08, 0x72, 0x61, 0x66, 0x74, 0x54, 0x65, 0x72, 0x6d, 0x12, 0x1b, 0x0a, + 0x09, 0x72, 0x61, 0x66, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x08, 0x72, 0x61, 0x66, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0x1b, 0x0a, 0x03, 0x4c, 0x53, + 0x4e, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x6c, 0x61, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, + 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x70, 0x62, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1076,7 +1151,7 @@ func file_log_proto_rawDescGZIP() []byte { return file_log_proto_rawDescData } -var file_log_proto_msgTypes = make([]protoimpl.MessageInfo, 15) +var file_log_proto_msgTypes = make([]protoimpl.MessageInfo, 16) var file_log_proto_goTypes = []any{ (*LogEntry)(nil), // 0: gitaly.LogEntry (*LSN)(nil), // 1: gitaly.LSN @@ -1084,35 +1159,37 @@ var file_log_proto_goTypes = []any{ (*LogEntry_RepositoryDeletion)(nil), // 3: gitaly.LogEntry.RepositoryDeletion (*LogEntry_Housekeeping)(nil), // 4: gitaly.LogEntry.Housekeeping (*LogEntry_Operation)(nil), // 5: gitaly.LogEntry.Operation - (*LogEntry_ReferenceTransaction_Change)(nil), // 6: gitaly.LogEntry.ReferenceTransaction.Change - (*LogEntry_Housekeeping_PackRefs)(nil), // 7: gitaly.LogEntry.Housekeeping.PackRefs - (*LogEntry_Housekeeping_Repack)(nil), // 8: gitaly.LogEntry.Housekeeping.Repack - (*LogEntry_Housekeeping_WriteCommitGraphs)(nil), // 9: gitaly.LogEntry.Housekeeping.WriteCommitGraphs - (*LogEntry_Operation_CreateHardLink)(nil), // 10: gitaly.LogEntry.Operation.CreateHardLink - (*LogEntry_Operation_RemoveDirectoryEntry)(nil), // 11: gitaly.LogEntry.Operation.RemoveDirectoryEntry - (*LogEntry_Operation_CreateDirectory)(nil), // 12: gitaly.LogEntry.Operation.CreateDirectory - (*LogEntry_Operation_SetKey)(nil), // 13: gitaly.LogEntry.Operation.SetKey - (*LogEntry_Operation_DeleteKey)(nil), // 14: gitaly.LogEntry.Operation.DeleteKey + (*LogEntry_Metadata)(nil), // 6: gitaly.LogEntry.Metadata + (*LogEntry_ReferenceTransaction_Change)(nil), // 7: gitaly.LogEntry.ReferenceTransaction.Change + (*LogEntry_Housekeeping_PackRefs)(nil), // 8: gitaly.LogEntry.Housekeeping.PackRefs + (*LogEntry_Housekeeping_Repack)(nil), // 9: gitaly.LogEntry.Housekeeping.Repack + (*LogEntry_Housekeeping_WriteCommitGraphs)(nil), // 10: gitaly.LogEntry.Housekeeping.WriteCommitGraphs + (*LogEntry_Operation_CreateHardLink)(nil), // 11: gitaly.LogEntry.Operation.CreateHardLink + (*LogEntry_Operation_RemoveDirectoryEntry)(nil), // 12: gitaly.LogEntry.Operation.RemoveDirectoryEntry + (*LogEntry_Operation_CreateDirectory)(nil), // 13: gitaly.LogEntry.Operation.CreateDirectory + (*LogEntry_Operation_SetKey)(nil), // 14: gitaly.LogEntry.Operation.SetKey + (*LogEntry_Operation_DeleteKey)(nil), // 15: gitaly.LogEntry.Operation.DeleteKey } var file_log_proto_depIdxs = []int32{ 2, // 0: gitaly.LogEntry.reference_transactions:type_name -> gitaly.LogEntry.ReferenceTransaction 3, // 1: gitaly.LogEntry.repository_deletion:type_name -> gitaly.LogEntry.RepositoryDeletion 4, // 2: gitaly.LogEntry.housekeeping:type_name -> gitaly.LogEntry.Housekeeping 5, // 3: gitaly.LogEntry.operations:type_name -> gitaly.LogEntry.Operation - 6, // 4: gitaly.LogEntry.ReferenceTransaction.changes:type_name -> gitaly.LogEntry.ReferenceTransaction.Change - 7, // 5: gitaly.LogEntry.Housekeeping.pack_refs:type_name -> gitaly.LogEntry.Housekeeping.PackRefs - 8, // 6: gitaly.LogEntry.Housekeeping.repack:type_name -> gitaly.LogEntry.Housekeeping.Repack - 9, // 7: gitaly.LogEntry.Housekeeping.write_commit_graphs:type_name -> gitaly.LogEntry.Housekeeping.WriteCommitGraphs - 10, // 8: gitaly.LogEntry.Operation.create_hard_link:type_name -> gitaly.LogEntry.Operation.CreateHardLink - 11, // 9: gitaly.LogEntry.Operation.remove_directory_entry:type_name -> gitaly.LogEntry.Operation.RemoveDirectoryEntry - 12, // 10: gitaly.LogEntry.Operation.create_directory:type_name -> gitaly.LogEntry.Operation.CreateDirectory - 13, // 11: gitaly.LogEntry.Operation.set_key:type_name -> gitaly.LogEntry.Operation.SetKey - 14, // 12: gitaly.LogEntry.Operation.delete_key:type_name -> gitaly.LogEntry.Operation.DeleteKey - 13, // [13:13] is the sub-list for method output_type - 13, // [13:13] is the sub-list for method input_type - 13, // [13:13] is the sub-list for extension type_name - 13, // [13:13] is the sub-list for extension extendee - 0, // [0:13] is the sub-list for field type_name + 6, // 4: gitaly.LogEntry.metadata:type_name -> gitaly.LogEntry.Metadata + 7, // 5: gitaly.LogEntry.ReferenceTransaction.changes:type_name -> gitaly.LogEntry.ReferenceTransaction.Change + 8, // 6: gitaly.LogEntry.Housekeeping.pack_refs:type_name -> gitaly.LogEntry.Housekeeping.PackRefs + 9, // 7: gitaly.LogEntry.Housekeeping.repack:type_name -> gitaly.LogEntry.Housekeeping.Repack + 10, // 8: gitaly.LogEntry.Housekeeping.write_commit_graphs:type_name -> gitaly.LogEntry.Housekeeping.WriteCommitGraphs + 11, // 9: gitaly.LogEntry.Operation.create_hard_link:type_name -> gitaly.LogEntry.Operation.CreateHardLink + 12, // 10: gitaly.LogEntry.Operation.remove_directory_entry:type_name -> gitaly.LogEntry.Operation.RemoveDirectoryEntry + 13, // 11: gitaly.LogEntry.Operation.create_directory:type_name -> gitaly.LogEntry.Operation.CreateDirectory + 14, // 12: gitaly.LogEntry.Operation.set_key:type_name -> gitaly.LogEntry.Operation.SetKey + 15, // 13: gitaly.LogEntry.Operation.delete_key:type_name -> gitaly.LogEntry.Operation.DeleteKey + 14, // [14:14] is the sub-list for method output_type + 14, // [14:14] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name } func init() { file_log_proto_init() } @@ -1133,7 +1210,7 @@ func file_log_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_log_proto_rawDesc, NumEnums: 0, - NumMessages: 15, + NumMessages: 16, NumExtensions: 0, NumServices: 0, }, diff --git a/proto/go/gitalypb/protolist.go b/proto/go/gitalypb/protolist.go index 3f7817e7827..2819219565f 100644 --- a/proto/go/gitalypb/protolist.go +++ b/proto/go/gitalypb/protolist.go @@ -7,6 +7,7 @@ var GitalyProtos = []string{ "analysis.proto", "blob.proto", "cleanup.proto", + "cluster.proto", "commit.proto", "conflicts.proto", "diff.proto", diff --git a/proto/log.proto b/proto/log.proto index fbf7879b79c..595a36064e8 100644 --- a/proto/log.proto +++ b/proto/log.proto @@ -146,9 +146,22 @@ message LogEntry { }; } + // Metadata contains some metadata data of the log entry. + message Metadata { + // raft_term is the term sequence of the Raft group finalized this + // log entry. + uint64 raft_term = 1; + // raft_type is the type of sequence. It's essentially equal to + // raftpb.EntryType. + uint64 raft_type = 2; + } + // operations is an ordered list of operations to run in order to apply // this log entry. repeated Operation operations = 10; + + // metadata contains some extra information, mostly Raft-related metadata. + Metadata metadata = 11; } // LSN serializes a log sequence number. It's used for storing a partition's -- GitLab From 6ddac756bc16afdfbede4b25a7d6c582a94b4bd2 Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Fri, 18 Oct 2024 09:06:27 +0700 Subject: [PATCH 10/19] raft: Add etcd/raft package This commit introduces `etcd/raft` package to power Raft consensus algorithm for Gitaly. This is a replacement for `dragonboat`. We removed it due to compatibility issue with Gitaly's WAL. For more information: https://gitlab.com/gitlab-org/gitaly/-/issues/6303 --- go.mod | 1 + go.sum | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/go.mod b/go.mod index 0fca8abe8f2..e1a13751d68 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/urfave/cli/v2 v2.27.5 gitlab.com/gitlab-org/labkit v1.21.2 + go.etcd.io/etcd/raft/v3 v3.5.16 go.uber.org/automaxprocs v1.6.0 go.uber.org/goleak v1.3.0 gocloud.dev v0.40.0 diff --git a/go.sum b/go.sum index 6b8da5724b6..fb668be6d88 100644 --- a/go.sum +++ b/go.sum @@ -189,6 +189,8 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI= github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= @@ -645,6 +647,10 @@ gitlab.com/gitlab-org/go/reopen v1.0.0 h1:6BujZ0lkkjGIejTUJdNO1w56mN1SI10qcVQyQl gitlab.com/gitlab-org/go/reopen v1.0.0/go.mod h1:D6OID8YJDzEVZNYW02R/Pkj0v8gYFSIhXFTArAsBQw8= gitlab.com/gitlab-org/labkit v1.21.2 h1:GlFHh8OdkrIMH3Qi0ByOzva0fGYXMICsuahGpJe4KNQ= gitlab.com/gitlab-org/labkit v1.21.2/go.mod h1:Q++SWyCH/abH2pytnX2SU/3mrCX6aK/xKz/WpM1hLbA= +go.etcd.io/etcd/client/pkg/v3 v3.5.16 h1:ZgY48uH6UvB+/7R9Yf4x574uCO3jIx0TRDyetSfId3Q= +go.etcd.io/etcd/client/pkg/v3 v3.5.16/go.mod h1:V8acl8pcEK0Y2g19YlOV9m9ssUe6MgiDSobSoaBAM0E= +go.etcd.io/etcd/raft/v3 v3.5.16 h1:zBXA3ZUpYs1AwiLGPafYAKKl/CORn/uaxYDwlNwndAk= +go.etcd.io/etcd/raft/v3 v3.5.16/go.mod h1:P4UP14AxofMJ/54boWilabqqWoW9eLodl6I5GdGzazI= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -- GitLab From cd117b7e354080698f5b7fbec598d0b128b8cd24 Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Mon, 14 Oct 2024 13:56:13 +0700 Subject: [PATCH 11/19] raft: Implement raft logger wrapper The raft logger wrapper is implemented to bridge compatibility between our internal logging and the etcd/raft Raft library's logging requirements. This wrapper ensures that the log messages generated during Raft operations integrate seamlessly with our existing logging infrastructure. One big difference is that the etcd/raft interfaces (Debug, Info, Warn, Error, Fatal, and Panic) support multiple input args. Thus, the logger needs to generate format string corresponding to the number of input. Weird enough, internally, the library uses *f version in most cases. --- internal/gitaly/storage/raftmgr/logger.go | 93 ++++++++++++ .../gitaly/storage/raftmgr/logger_test.go | 135 ++++++++++++++++++ .../gitaly/storage/raftmgr/testhelper_test.go | 11 ++ internal/testhelper/logger.go | 20 ++- 4 files changed, 255 insertions(+), 4 deletions(-) create mode 100644 internal/gitaly/storage/raftmgr/logger.go create mode 100644 internal/gitaly/storage/raftmgr/logger_test.go create mode 100644 internal/gitaly/storage/raftmgr/testhelper_test.go diff --git a/internal/gitaly/storage/raftmgr/logger.go b/internal/gitaly/storage/raftmgr/logger.go new file mode 100644 index 00000000000..710721981e2 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/logger.go @@ -0,0 +1,93 @@ +package raftmgr + +import ( + "fmt" + "strings" + + "gitlab.com/gitlab-org/gitaly/v16/internal/log" +) + +// raftLogger is a wrapper around the log.Logger interface to +// implement etcd/raft's logger interface methods. +type raftLogger struct { + logger log.Logger +} + +// Debug logs a debug level message using variable arguments. +func (l *raftLogger) Debug(args ...any) { + l.Debugf(l.generateFormatString(len(args)), args...) +} + +// Debugf logs a formatted debug level message. +func (l *raftLogger) Debugf(format string, args ...any) { + l.logger.Debug(fmt.Sprintf(format, args...)) +} + +// Info logs an info level message using variable arguments. +func (l *raftLogger) Info(args ...any) { + l.Infof(l.generateFormatString(len(args)), args...) +} + +// Infof logs a formatted info level message. +func (l *raftLogger) Infof(format string, args ...any) { + l.logger.Info(fmt.Sprintf(format, args...)) +} + +// Warning logs a warning level message using variable arguments. +func (l *raftLogger) Warning(args ...any) { + l.Warningf(l.generateFormatString(len(args)), args...) +} + +// Warningf logs a formatted warning level message. +func (l *raftLogger) Warningf(format string, args ...any) { + l.logger.Warn(fmt.Sprintf(format, args...)) +} + +// Error logs an error level message using variable arguments. +func (l *raftLogger) Error(args ...any) { + l.Errorf(l.generateFormatString(len(args)), args...) +} + +// Errorf logs a formatted error level message. +func (l *raftLogger) Errorf(format string, args ...any) { + l.logger.Error(fmt.Sprintf(format, args...)) +} + +// Fatal logs a fatal level message using variable arguments. +func (l *raftLogger) Fatal(args ...any) { + l.Fatalf(l.generateFormatString(len(args)), args...) +} + +// Fatalf logs a formatted fatal level message using error level log. +func (l *raftLogger) Fatalf(format string, args ...any) { + l.logger.Error(fmt.Sprintf(format, args...)) +} + +// Panic logs a panic level message using variable arguments. +func (l *raftLogger) Panic(args ...any) { + l.Panicf(l.generateFormatString(len(args)), args...) +} + +// Panicf logs a formatted panic level message. +func (l *raftLogger) Panicf(format string, args ...any) { + msg := fmt.Sprintf(format, args...) + l.logger.Error(msg) + panic(msg) +} + +// generateFormatString generates a format string for the provided number of arguments. +func (l *raftLogger) generateFormatString(argCount int) string { + switch argCount { + case 0: + return "" + case 1: + // Fast path, most common use cases. + return "%s" + default: + formatSpecifiers := make([]string, argCount) + for i := 0; i < argCount; i++ { + formatSpecifiers[i] = "%s" + } + return strings.Join(formatSpecifiers, " ") + } +} diff --git a/internal/gitaly/storage/raftmgr/logger_test.go b/internal/gitaly/storage/raftmgr/logger_test.go new file mode 100644 index 00000000000..0bac85f3707 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/logger_test.go @@ -0,0 +1,135 @@ +package raftmgr + +import ( + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" +) + +func TestRaftLogger(t *testing.T) { + t.Parallel() + + newLogger := func(t *testing.T) (*raftLogger, testhelper.LoggerHook) { + logger := testhelper.NewLogger(t, testhelper.WithLevel(logrus.DebugLevel)) + hook := testhelper.AddLoggerHook(logger) + hook.Reset() + return &raftLogger{logger: logger}, hook + } + + tests := []struct { + name string + logFunc func(logger *raftLogger, message string, args ...any) + level string + message string + args []any + expected string + }{ + { + name: "Debugf", + logFunc: func(l *raftLogger, msg string, args ...any) { l.Debugf(msg, args...) }, + level: "debug", + message: "debug message %s", + args: []any{"test"}, + expected: "debug message test", + }, + { + name: "Debug", + logFunc: func(l *raftLogger, msg string, args ...any) { l.Debug(msg) }, + level: "debug", + message: "debug message test", + expected: "debug message test", + }, + { + name: "Infof", + logFunc: func(l *raftLogger, msg string, args ...any) { l.Infof(msg, args...) }, + level: "info", + message: "info message %+s", + args: []any{"test"}, + expected: "info message test", + }, + { + name: "Info", + logFunc: func(l *raftLogger, msg string, args ...any) { l.Info(msg) }, + level: "info", + message: "info message test", + expected: "info message test", + }, + { + name: "Warningf", + logFunc: func(l *raftLogger, msg string, args ...any) { l.Warningf(msg, args...) }, + level: "warning", + message: "warning message %s", + args: []any{"test"}, + expected: "warning message test", + }, + { + name: "Warning", + logFunc: func(l *raftLogger, msg string, args ...any) { l.Warning(msg) }, + level: "warning", + message: "warning message test", + expected: "warning message test", + }, + { + name: "Errorf", + logFunc: func(l *raftLogger, msg string, args ...any) { l.Errorf(msg, args...) }, + level: "error", + message: "error message %s", + args: []any{"test"}, + expected: "error message test", + }, + { + name: "Error", + logFunc: func(l *raftLogger, msg string, args ...any) { l.Error(msg) }, + level: "error", + message: "error message test", + expected: "error message test", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + logger, hook := newLogger(t) + + tc.logFunc(logger, tc.message, tc.args...) + + entries := hook.AllEntries() + require.Equal(t, 1, len(entries), "expected exactly one log entry") + require.Equal(t, tc.expected, entries[0].Message) + require.Equal(t, tc.level, entries[0].Level.String()) + }) + } + + t.Run("Fatal", func(t *testing.T) { + t.Parallel() + + logger, hook := newLogger(t) + logger.Fatal("fatal message", "test") + + entries := hook.AllEntries() + require.Equal(t, 1, len(entries), "expected exactly one log entry after fatal") + require.Equal(t, "fatal message test", entries[0].Message) + require.Equal(t, "error", entries[0].Level.String()) // Assuming Fatal logs as error level before exiting + }) + + t.Run("Panic", func(t *testing.T) { + t.Parallel() + + logger, hook := newLogger(t) + func() { + defer func() { + r := recover() + require.NotNilf(t, r, "calling logger.Panic should raise a panic") + + entries := hook.AllEntries() + require.Equal(t, 1, len(entries), "expected exactly one log entry after panic") + require.Equal(t, "panic message test", entries[0].Message) + require.Equal(t, "error", entries[0].Level.String()) // Assuming Panic logs as error level before panicking + }() + logger.Panic("panic message", "test") + }() + }) +} diff --git a/internal/gitaly/storage/raftmgr/testhelper_test.go b/internal/gitaly/storage/raftmgr/testhelper_test.go new file mode 100644 index 00000000000..c80ed0e3813 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/testhelper_test.go @@ -0,0 +1,11 @@ +package raftmgr + +import ( + "testing" + + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" +) + +func TestMain(m *testing.M) { + testhelper.Run(m) +} diff --git a/internal/testhelper/logger.go b/internal/testhelper/logger.go index eb6dc86685a..6fdced6c758 100644 --- a/internal/testhelper/logger.go +++ b/internal/testhelper/logger.go @@ -85,7 +85,8 @@ func (b *syncBuffer) String() string { } type loggerOptions struct { - name string + name string + level logrus.Level } // LoggerOption configures a logger. @@ -99,16 +100,24 @@ func WithLoggerName(name string) LoggerOption { } } +// WithLevel sets the level of the logger. It's useful when we would like +// to capture debug logs. +func WithLevel(level logrus.Level) LoggerOption { + return func(opts *loggerOptions) { + opts.level = level + } +} + // NewLogger returns a logger that records the log output and // prints it out only if the test fails. func NewLogger(tb testing.TB, options ...LoggerOption) log.LogrusLogger { - logger, logOutput := NewCapturedLogger() - var opts loggerOptions for _, apply := range options { apply(&opts) } + logger, logOutput := NewCapturedLogger(opts) + tb.Cleanup(func() { if !tb.Failed() || logOutput.Len() == 0 { return @@ -126,10 +135,13 @@ func NewLogger(tb testing.TB, options ...LoggerOption) log.LogrusLogger { // NewCapturedLogger returns a logger that records the log outputs in a buffer. The caller // decides how and when the logs are dumped out. -func NewCapturedLogger() (log.LogrusLogger, *syncBuffer) { +func NewCapturedLogger(opts loggerOptions) (log.LogrusLogger, *syncBuffer) { logOutput := &syncBuffer{} logger := logrus.New() //nolint:forbidigo logger.Out = logOutput + if opts.level != 0 { + logger.SetLevel(opts.level) + } return log.FromLogrusEntry(logrus.NewEntry(logger)), logOutput } -- GitLab From b7eecd69b4bcfa253f6a1c0b826c348f124b6d1a Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Wed, 23 Oct 2024 13:03:47 +0700 Subject: [PATCH 12/19] raft: Implement entry recorder Internally, Raft emits some internal log entries automatically, such as config changes, empty verification entries, etc. Those entries use the same ID system with normal log entries. The inserting order is not deterministic. As a result, when Raft manager is integrated into some later commits, we need a way to stabilize the test suite. This commit implements a recorder that records all log entries the Raft manager processes. It provides some utility methods to offset the LSNs in tests when Raft is enabled. --- .../gitaly/storage/raftmgr/entry_recorder.go | 138 ++++++++++++++++++ .../storage/raftmgr/entry_recorder_test.go | 137 +++++++++++++++++ 2 files changed, 275 insertions(+) create mode 100644 internal/gitaly/storage/raftmgr/entry_recorder.go create mode 100644 internal/gitaly/storage/raftmgr/entry_recorder_test.go diff --git a/internal/gitaly/storage/raftmgr/entry_recorder.go b/internal/gitaly/storage/raftmgr/entry_recorder.go new file mode 100644 index 00000000000..2af9103d45d --- /dev/null +++ b/internal/gitaly/storage/raftmgr/entry_recorder.go @@ -0,0 +1,138 @@ +package raftmgr + +import ( + "sync" + + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" + "google.golang.org/protobuf/proto" +) + +// EntryRecorder is a utility for recording and classifying log entries processed by the Raft manager. In addition to +// standard log entries, Raft may generate internal entries such as configuration changes or empty logs for verification +// purposes. These internal entries are backfilled into the Write-Ahead Log (WAL), occupying Log Sequence Number (LSN) +// slots. Consequently, the LSN sequence may diverge from expectations when Raft is enabled. +// +// This recorder is equipped with the capability to offset the LSN, which is particularly useful in testing environments +// to mitigate differences in LSN sequences. It is strongly advised that this feature be restricted to testing purposes +// and not utilized in production or other non-testing scenarios. +type EntryRecorder struct { + mu sync.Mutex // Mutex for safe concurrent access + Items []Item // Slice to store recorded log entries +} + +// Item represents an entry recorded by the LogEntryRecorder, with a flag indicating if it's from Raft. +type Item struct { + FromRaft bool + LSN storage.LSN + Entry *gitalypb.LogEntry +} + +// NewEntryRecorder returns a new instance of NewEntryRecorder. +func NewEntryRecorder() *EntryRecorder { + return &EntryRecorder{} +} + +// Record logs an entry, marking it as originating from Raft if specified. +// If the LSN is from the past, it removes all entries after that LSN. +func (r *EntryRecorder) Record(fromRaft bool, lsn storage.LSN, entry *gitalypb.LogEntry) { + r.mu.Lock() + defer r.mu.Unlock() + + // Trim items that have an LSN greater than the current LSN + if len(r.Items) > 0 { + idx := -1 + for i, itm := range r.Items { + if itm.LSN >= lsn { + idx = i + break + } + } + if idx != -1 { + r.Items = r.Items[:idx] + } + } + + // Append the new entry + r.Items = append(r.Items, Item{ + FromRaft: fromRaft, + LSN: lsn, + Entry: proto.Clone(entry).(*gitalypb.LogEntry), + }) +} + +// Offset adjusts the log sequence number (LSN) by accounting for internal Raft entries that may occupy slots. +func (r *EntryRecorder) Offset(lsn storage.LSN) storage.LSN { + r.mu.Lock() + defer r.mu.Unlock() + + offset := lsn + for _, itm := range r.Items { + if itm.FromRaft { + offset++ + } + if itm.LSN >= offset { + break + } + } + return offset +} + +// Metadata returns metadata of an entry if it exists. +func (r *EntryRecorder) Metadata(lsn storage.LSN) *gitalypb.LogEntry_Metadata { + r.mu.Lock() + defer r.mu.Unlock() + + for _, itm := range r.Items { + if itm.LSN == lsn { + return itm.Entry.GetMetadata() + } + } + return nil +} + +// Latest returns the latest recorded LSN. +func (r *EntryRecorder) Latest() storage.LSN { + r.mu.Lock() + defer r.mu.Unlock() + + if len(r.Items) == 0 { + return 0 + } + return r.Items[len(r.Items)-1].LSN +} + +// Len returns the length of recorded entries. +func (r *EntryRecorder) Len() int { + r.mu.Lock() + defer r.mu.Unlock() + + return len(r.Items) +} + +// IsFromRaft returns true if the asserting LSN is an entry emitted by Raft. +func (r *EntryRecorder) IsFromRaft(lsn storage.LSN) bool { + r.mu.Lock() + defer r.mu.Unlock() + + for _, itm := range r.Items { + if itm.LSN == lsn { + return itm.FromRaft + } + } + return false +} + +// FromRaft retrieves all log entries that originated from the Raft system. +func (r *EntryRecorder) FromRaft() map[storage.LSN]*gitalypb.LogEntry { + r.mu.Lock() + defer r.mu.Unlock() + + raftEntries := map[storage.LSN]*gitalypb.LogEntry{} + for _, itm := range r.Items { + if itm.FromRaft { + raftEntries[itm.LSN] = itm.Entry + } + } + return raftEntries +} diff --git a/internal/gitaly/storage/raftmgr/entry_recorder_test.go b/internal/gitaly/storage/raftmgr/entry_recorder_test.go new file mode 100644 index 00000000000..6e1d35c60a7 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/entry_recorder_test.go @@ -0,0 +1,137 @@ +package raftmgr + +import ( + "testing" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" +) + +func TestLogEntryRecorder(t *testing.T) { + testCases := []struct { + desc string + records []Item + initialLSN storage.LSN + expectedOffset storage.LSN + }{ + { + desc: "no entries", + records: []Item{}, + initialLSN: 1, + expectedOffset: 1, + }, + { + desc: "no internal raft entries", + records: []Item{ + {FromRaft: false, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 2, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 1, + expectedOffset: 1, + }, + { + desc: "no internal raft entries", + records: []Item{ + {FromRaft: false, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 2, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 2, + expectedOffset: 2, + }, + { + desc: "all raft entries", + records: []Item{ + {FromRaft: true, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: true, LSN: 2, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 1, + expectedOffset: 3, // Two raft entries increment the offset by 2 + }, + { + desc: "mix of raft and non-raft, with raft at beginning", + records: []Item{ + {FromRaft: true, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 2, Entry: &gitalypb.LogEntry{}}, + {FromRaft: true, LSN: 3, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 1, + expectedOffset: 2, + }, + { + desc: "mix of raft and non-raft, with raft at beginning", + records: []Item{ + {FromRaft: true, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 2, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 3, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 2, + expectedOffset: 3, + }, + { + desc: "mix of raft and non-raft, with raft at beginning", + records: []Item{ + {FromRaft: true, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 2, Entry: &gitalypb.LogEntry{}}, + {FromRaft: true, LSN: 3, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 3, + expectedOffset: 5, + }, + { + desc: "mix of raft and non-raft, no raft at beginning", + records: []Item{ + {FromRaft: false, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: true, LSN: 2, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 3, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 1, + expectedOffset: 1, + }, + { + desc: "mix of raft and non-raft, no raft at beginning", + records: []Item{ + {FromRaft: false, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: true, LSN: 2, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 3, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 3, + expectedOffset: 4, + }, + { + desc: "initial LSN beyond recorded entries", + records: []Item{ + {FromRaft: false, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: true, LSN: 2, Entry: &gitalypb.LogEntry{}}, + }, + initialLSN: 3, + expectedOffset: 4, + }, + { + desc: "replace entries with a past LSN", + records: []Item{ + {FromRaft: true, LSN: 1, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 2, Entry: &gitalypb.LogEntry{}}, + {FromRaft: false, LSN: 3, Entry: &gitalypb.LogEntry{}}, + {FromRaft: true, LSN: 2, Entry: &gitalypb.LogEntry{}}, // Insert with past LSN, should remove LSN 2 and 3 entry + }, + initialLSN: 1, + expectedOffset: 3, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + recorder := EntryRecorder{} + + // Record entries as specified in the test case + for _, record := range tc.records { + recorder.Record(record.FromRaft, record.LSN, record.Entry) + } + + // Validate the offset + offset := recorder.Offset(tc.initialLSN) + require.Equal(t, tc.expectedOffset, offset) + }) + } +} -- GitLab From 950f09c229f195ef53bb5ee42ab4c610bc7a303b Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Wed, 23 Oct 2024 13:12:49 +0700 Subject: [PATCH 13/19] raft: Implement event registry etcd/raft provides Propose interface to submit a log entry to the cluster. This interface exits when the Raft engine accepts the entry. There is no guarantee when the proposal is committed by a quorum or even if the proposal ever leaves the node at all. In some later commits, the transaction manager must propose WAL log entries in the process loop. Without concrete evidence of log entry being committed, it's risky to continue. Thus, this commit implements an event registry. Before the caller proposes, it announces this proposal to the event registry and embeds the returned event ID in the transporting Raft message. After the log entry is committed, the caller is unlocked. --- internal/gitaly/storage/raftmgr/registry.go | 103 ++++++++++ .../gitaly/storage/raftmgr/registry_test.go | 181 ++++++++++++++++++ 2 files changed, 284 insertions(+) create mode 100644 internal/gitaly/storage/raftmgr/registry.go create mode 100644 internal/gitaly/storage/raftmgr/registry_test.go diff --git a/internal/gitaly/storage/raftmgr/registry.go b/internal/gitaly/storage/raftmgr/registry.go new file mode 100644 index 00000000000..468a95dc709 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/registry.go @@ -0,0 +1,103 @@ +package raftmgr + +import ( + "fmt" + "sync" + + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" +) + +// ErrObsoleted is returned when an event associated with a LSN is shadowed by another one with higher term. That event +// must be unlocked and removed from the registry. +var ErrObsoleted = fmt.Errorf("obsoleted event, shadowed by a log entry with higher term") + +// EventID uniquely identifies an event in the registry. +type EventID uint64 + +// Waiter holds the information required to wait for an event to be committed. +type Waiter struct { + ID EventID + LSN storage.LSN + C chan struct{} + Err error +} + +// Registry manages events and their associated waiters, enabling the registration +// and removal of waiters upon event commitment. +type Registry struct { + mu sync.Mutex + nextEventID EventID + waiters map[EventID]*Waiter +} + +// NewRegistry initializes and returns a new instance of Registry. +func NewRegistry() *Registry { + return &Registry{ + waiters: make(map[EventID]*Waiter), + } +} + +// Register creates a new Waiter for an upcoming event and returns it. +// It must be called whenever an event is proposed, with the event ID embedded +// in the corresponding Raft message. +func (r *Registry) Register() *Waiter { + r.mu.Lock() + defer r.mu.Unlock() + + r.nextEventID++ + waiter := &Waiter{ + ID: r.nextEventID, + C: make(chan struct{}), + } + r.waiters[r.nextEventID] = waiter + + return waiter +} + +// AssignLSN assigns LSN to an event. LSN of an event is used to unlock obsolete proposals if Raft detects duplicated +// LSNs but with higher term. +func (r *Registry) AssignLSN(id EventID, lsn storage.LSN) { + r.mu.Lock() + defer r.mu.Unlock() + + waiter, exists := r.waiters[id] + if !exists { + return + } + waiter.LSN = lsn +} + +// UntrackSince untracks all events having LSNs greater than or equal to the input LSN. +func (r *Registry) UntrackSince(lsn storage.LSN) { + r.mu.Lock() + defer r.mu.Unlock() + + var toRemove []EventID + for id, w := range r.waiters { + if w.LSN >= lsn { + toRemove = append(toRemove, id) + } + } + for _, id := range toRemove { + close(r.waiters[id].C) + r.waiters[id].Err = ErrObsoleted + delete(r.waiters, id) + } +} + +// Untrack closes the channel associated with a given EventID and removes the waiter from the registry once the event is +// committed. This function returns if the registry is still tracking the event. +func (r *Registry) Untrack(id EventID) bool { + r.mu.Lock() + defer r.mu.Unlock() + + waiter, exists := r.waiters[id] + if !exists { + return false + } + + // Close the channel to notify any goroutines waiting on this event. + close(waiter.C) + delete(r.waiters, id) + return true +} diff --git a/internal/gitaly/storage/raftmgr/registry_test.go b/internal/gitaly/storage/raftmgr/registry_test.go new file mode 100644 index 00000000000..beca124804a --- /dev/null +++ b/internal/gitaly/storage/raftmgr/registry_test.go @@ -0,0 +1,181 @@ +package raftmgr + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" +) + +func TestRegistry(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + action func(*Registry) []*Waiter + expectedEvents []EventID + }{ + { + name: "Register and Remove single event", + action: func(r *Registry) []*Waiter { + waiter := r.Register() + require.True(t, r.Untrack(waiter.ID)) + return []*Waiter{waiter} + }, + expectedEvents: []EventID{1}, + }, + { + name: "Register multiple events and remove in order", + action: func(r *Registry) []*Waiter { + w1 := r.Register() + w2 := r.Register() + require.True(t, r.Untrack(w1.ID)) + require.True(t, r.Untrack(w2.ID)) + return []*Waiter{w1, w2} + }, + expectedEvents: []EventID{1, 2}, + }, + { + name: "Register multiple events and remove out of order", + action: func(r *Registry) []*Waiter { + w1 := r.Register() + w2 := r.Register() + require.True(t, r.Untrack(w2.ID)) // Removing the second one first + require.True(t, r.Untrack(w1.ID)) // Then the first one + return []*Waiter{w1, w2} + }, + expectedEvents: []EventID{1, 2}, + }, + { + name: "Remove non-existent event", + action: func(r *Registry) []*Waiter { + require.False(t, r.Untrack(1234)) + + c := make(chan struct{}) + close(c) + return []*Waiter{{ID: 99999, C: c}} // Non-existent event + }, + expectedEvents: []EventID{99999}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + registry := NewRegistry() + waiters := tc.action(registry) + + for _, waiter := range waiters { + select { + case <-waiter.C: + // Success, channel was closed + case <-time.After(10 * time.Second): + t.Fatalf("Expected channel for event %d to be closed", waiter.ID) + } + require.Contains(t, tc.expectedEvents, waiter.ID) + } + }) + } +} + +func TestRegistry_AssignLSN(t *testing.T) { + t.Parallel() + registry := NewRegistry() + + waiter1 := registry.Register() + waiter2 := registry.Register() + + // Assign LSN to the registered waiters + registry.AssignLSN(waiter1.ID, 10) + registry.AssignLSN(waiter2.ID, 20) + registry.AssignLSN(999, 99) + + // Verify that LSNs are assigned correctly + require.Equal(t, storage.LSN(10), waiter1.LSN) + require.Equal(t, storage.LSN(20), waiter2.LSN) +} + +func TestRegistry_UntrackSince(t *testing.T) { + t.Parallel() + registry := NewRegistry() + + waiter1 := registry.Register() + waiter2 := registry.Register() + waiter3 := registry.Register() + + // Assign LSNs + registry.AssignLSN(waiter1.ID, 10) + registry.AssignLSN(waiter2.ID, 11) + registry.AssignLSN(waiter3.ID, 12) + + // Call UntrackSince with threshold LSN + registry.UntrackSince(11) + + // Waiters with LSN > 10 should be obsoleted + select { + case <-waiter1.C: + t.Fatalf("Expected channel for event %d to remain open", waiter1.ID) + default: + // Waiter1 should not be closed + } + select { + case <-waiter2.C: + // Expected behavior, channel closed + require.Equal(t, ErrObsoleted, waiter2.Err) + default: + t.Fatalf("Expected channel for event %d to be closed", waiter2.ID) + } + + select { + case <-waiter3.C: + // Expected behavior, channel closed + require.Equal(t, ErrObsoleted, waiter3.Err) + default: + t.Fatalf("Expected channel for event %d to be closed", waiter3.ID) + } + + // Waiter1 should still be tracked + require.True(t, registry.Untrack(waiter1.ID)) + // Waiter2 and Waiter3 should not be tracked anymore + require.False(t, registry.Untrack(waiter2.ID)) + require.False(t, registry.Untrack(waiter3.ID)) +} + +func TestRegistry_ConcurrentAccess(t *testing.T) { + t.Parallel() + const numEvents = 100 + + registry := NewRegistry() + waiters := make(chan EventID, numEvents) + var producerWg, consumerWg sync.WaitGroup + + // Register events concurrently + for i := 0; i < 10; i++ { + producerWg.Add(1) + go func() { + defer producerWg.Done() + for i := 0; i < numEvents; i++ { + waiters <- registry.Register().ID + } + }() + } + + // Untrack events concurrently + for i := 0; i < 10; i++ { + consumerWg.Add(1) + go func() { + defer consumerWg.Done() + for i := range waiters { + assert.True(t, registry.Untrack(i)) + } + }() + } + + producerWg.Wait() + close(waiters) + consumerWg.Wait() + + require.Emptyf(t, registry.waiters, "waiter list must be empty") +} -- GitLab From 099d9c3616efe7d22830648ffe45fe2f1db7217a Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Wed, 23 Oct 2024 18:10:13 +0700 Subject: [PATCH 14/19] raft: Implement Leadership management struct This commit introduces the Leadership struct, a supporting component in managing leadership state within the distributed system using the Raft consensus algorithm. The Leadership struct handles the current leader's ID, whether the node is the leader, and records the duration since the last leadership change. Additionally, it offers a notification channel to signal leadership changes, improving the responsiveness of operations dependent on leadership events. --- internal/gitaly/storage/raftmgr/leadership.go | 63 ++++++++++ .../gitaly/storage/raftmgr/leadership_test.go | 116 ++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 internal/gitaly/storage/raftmgr/leadership.go create mode 100644 internal/gitaly/storage/raftmgr/leadership_test.go diff --git a/internal/gitaly/storage/raftmgr/leadership.go b/internal/gitaly/storage/raftmgr/leadership.go new file mode 100644 index 00000000000..fa130a5513b --- /dev/null +++ b/internal/gitaly/storage/raftmgr/leadership.go @@ -0,0 +1,63 @@ +package raftmgr + +import ( + "sync" + "time" +) + +// Leadership manages the state of leadership in a distributed system using the Raft consensus algorithm. It tracks the +// current leader's ID, whether this node is the leader, and the time since the last leadership change. It also provides +// a notification mechanism for leadership changes through a channel. +type Leadership struct { + sync.Mutex + leaderID uint64 + isLeader bool + lastChange time.Time + newLeaderC chan struct{} +} + +// NewLeadership initializes a new Leadership instance with the current time and a buffered channel. +func NewLeadership() *Leadership { + return &Leadership{ + lastChange: time.Now(), + newLeaderC: make(chan struct{}, 1), + } +} + +// SetLeader updates the leadership information if there is a change in the leaderID. +// It returns a boolean indicating whether a change occurred and the duration of the last leadership. +func (l *Leadership) SetLeader(leaderID uint64, isLeader bool) (changed bool, lastDuration time.Duration) { + l.Lock() + defer l.Unlock() + + if l.leaderID == leaderID && l.isLeader == isLeader { + return false, 0 + } + + l.leaderID = leaderID + l.isLeader = isLeader + now := time.Now() + lastDuration = now.Sub(l.lastChange) + l.lastChange = now + + select { + case l.newLeaderC <- struct{}{}: + default: + } + + return true, lastDuration +} + +// IsLeader returns true if the current instance is the leader. +func (l *Leadership) IsLeader() bool { + l.Lock() + defer l.Unlock() + return l.isLeader +} + +// GetLeaderID retrieves the current leader's ID. +func (l *Leadership) GetLeaderID() uint64 { + l.Lock() + defer l.Unlock() + return l.leaderID +} diff --git a/internal/gitaly/storage/raftmgr/leadership_test.go b/internal/gitaly/storage/raftmgr/leadership_test.go new file mode 100644 index 00000000000..a4f9bf9ff06 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/leadership_test.go @@ -0,0 +1,116 @@ +package raftmgr + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestLeadership(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + initialLeader uint64 + leaderID uint64 + isLeader bool + wantChanged bool + wantDuration time.Duration // Maximum acceptable duration difference for test purposes + wantIsLeader bool + }{ + { + name: "Initial leader set", + initialLeader: 0, + leaderID: 1, + isLeader: true, + wantChanged: true, + wantDuration: 0, + wantIsLeader: true, + }, + { + name: "No leader change", + initialLeader: 1, + leaderID: 1, + isLeader: true, + wantChanged: false, + wantDuration: 0, + wantIsLeader: true, + }, + { + name: "Change leader and role", + initialLeader: 1, + leaderID: 2, + isLeader: false, + wantChanged: true, + wantDuration: 0, + wantIsLeader: false, + }, + { + name: "Become leader again", + initialLeader: 2, + leaderID: 2, + isLeader: true, + wantChanged: false, + wantDuration: 0, + wantIsLeader: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + leadership := NewLeadership() + // Initial leader setup if needed + if tc.initialLeader != 0 { + leadership.SetLeader(tc.initialLeader, true) + } + + changed, duration := leadership.SetLeader(tc.leaderID, tc.isLeader) + require.Equal(t, tc.wantChanged, changed, "Leadership change status mismatch") + if tc.wantChanged { + require.InDelta(t, tc.wantDuration, duration.Milliseconds(), 100, "Unexpected leadership duration difference") + } + require.Equal(t, tc.wantIsLeader, leadership.IsLeader(), "Leadership role mismatch") + require.Equal(t, tc.leaderID, leadership.GetLeaderID(), "Leader ID mismatch") + + // Verify the channel behavior if changed + if tc.wantChanged { + select { + case <-leadership.newLeaderC: + // Success, channel was correctly notified + case <-time.After(1 * time.Second): + t.Fatal("Expected newLeaderC to be notified but it wasn't") + } + } + }) + } +} + +func TestLeadership_MultipleChanges(t *testing.T) { + t.Parallel() + leadership := NewLeadership() + + changes := []struct { + leaderID uint64 + isLeader bool + }{ + {1, true}, + {2, false}, + {3, true}, + } + + for _, change := range changes { + // newLeaderC should not block. + leadership.SetLeader(change.leaderID, change.isLeader) + } + + require.Equal(t, true, leadership.IsLeader(), "Leadership role mismatch") + require.Equal(t, uint64(3), leadership.GetLeaderID(), "Leader ID mismatch") + + select { + case <-leadership.newLeaderC: + // Channel was correctly notified + default: + t.Fatal("Expected newLeaderC to be notified") + } +} -- GitLab From c5a0276b55d73f08e9f429bbcc66bc8abd3bd28d Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Mon, 28 Oct 2024 12:14:18 +0700 Subject: [PATCH 15/19] raft: Implement noop transportation Introduce a NoopTransport in the Raft module for testing purposes. This transport logs messages and optionally records them without performing any actual data transmission. This Transport is used at this stage so that Raft Manager can swallow all outgoing messages as the current primary member is the only member of the Raft group. --- internal/gitaly/storage/raftmgr/transport.go | 122 +++++++++ .../gitaly/storage/raftmgr/transport_test.go | 238 ++++++++++++++++++ 2 files changed, 360 insertions(+) create mode 100644 internal/gitaly/storage/raftmgr/transport.go create mode 100644 internal/gitaly/storage/raftmgr/transport_test.go diff --git a/internal/gitaly/storage/raftmgr/transport.go b/internal/gitaly/storage/raftmgr/transport.go new file mode 100644 index 00000000000..6f4f1f7b6b1 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/transport.go @@ -0,0 +1,122 @@ +package raftmgr + +import ( + "bufio" + "bytes" + "context" + "fmt" + + "gitlab.com/gitlab-org/gitaly/v16/internal/archive" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" + "go.etcd.io/etcd/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +// Transport defines the interface for sending Raft protocol messages. +type Transport interface { + // Send dispatches a batch of Raft messages. It returns an error if the sending fails. + Send(context.Context, func(storage.LSN) string, []raftpb.Message) error + + // GetRecordedMessages retrieves all recorded messages if recording is enabled. + // This is typically used in a testing environment to verify message transmission. + GetRecordedMessages() []raftpb.Message +} + +// NoopTransport is a transport implementation that logs messages and optionally records them. +// It is useful in testing environments where message delivery is non-functional but needs to be observed. +type NoopTransport struct { + logger log.Logger // Logger for outputting message information + recordTransport bool // Flag indicating whether message recording is enabled + recordedMessages []*raftpb.Message // Slice to store recorded messages +} + +// NewNoopTransport constructs a new NoopTransport instance. +// The logger is used for logging message information, and the recordTransport flag +// determines whether messages should be recorded. +func NewNoopTransport(logger log.Logger, recordTransport bool) Transport { + return &NoopTransport{ + logger: logger, + recordTransport: recordTransport, + } +} + +// Send logs each message being sent and records it if recording is enabled. +func (t *NoopTransport) Send(ctx context.Context, pathForLSN func(storage.LSN) string, messages []raftpb.Message) error { + for i := range messages { + for j := range messages[i].Entries { + if messages[i].Entries[j].Type == raftpb.EntryNormal { + var msg gitalypb.RaftMessageV1 + + if err := proto.Unmarshal(messages[i].Entries[j].Data, &msg); err != nil { + return fmt.Errorf("unmarshalling entry type: %w", err) + } + + // This is a very native implementation. Noop Transport is only used for testing + // purposes. All external messages are swallowed and stored in a recorder. It packages + // the whole log entry directory as a tar ball using an existing backup utility. The + // resulting binary data is stored inside a subfield of the message for examining + // purpose. A real implementation of Transaction will likely use an optimized method + // (such as sidechannel) to deliver the data. It does not necessarily store the data in + // the memory. + switch msg.GetLogData().(type) { + case *gitalypb.RaftMessageV1_Packed: + continue + case *gitalypb.RaftMessageV1_Referenced: + lsn := storage.LSN(messages[i].Entries[j].Index) + path := pathForLSN(lsn) + if err := t.packLogData(ctx, lsn, &msg, path); err != nil { + return fmt.Errorf("packing log data: %w", err) + } + } + data, err := proto.Marshal(&msg) + if err != nil { + return fmt.Errorf("marshaling Raft entry: %w", err) + } + messages[i].Entries[j].Data = data + } + } + + t.logger.WithFields(log.Fields{ + "raft.type": messages[i].Type, + "raft.to": messages[i].To, + "raft.from": messages[i].From, + "raft.term": messages[i].Term, + "raft.num_entries": len(messages[i].Entries), + }).Info("sending message") + + // Record messages if recording is enabled. + if t.recordTransport { + t.recordedMessages = append(t.recordedMessages, &messages[i]) + } + } + return nil +} + +func (t *NoopTransport) packLogData(ctx context.Context, lsn storage.LSN, message *gitalypb.RaftMessageV1, logEntryPath string) error { + var logData bytes.Buffer + writer := bufio.NewWriter(&logData) + if err := archive.WriteTarball(ctx, t.logger.WithFields(log.Fields{ + "raft.component": "WAL archiver", + "raft.log_entry_lsn": lsn, + "raft.log_entry_path": logEntryPath, + }), writer, logEntryPath, "."); err != nil { + return fmt.Errorf("archiving WAL log entry") + } + message.LogData = &gitalypb.RaftMessageV1_Packed{ + Packed: &gitalypb.RaftMessageV1_PackedLogData{ + Data: logData.Bytes(), + }, + } + return nil +} + +// GetRecordedMessages returns the list of recorded messages. +func (t *NoopTransport) GetRecordedMessages() []raftpb.Message { + messages := make([]raftpb.Message, 0, len(t.recordedMessages)) + for _, m := range t.recordedMessages { + messages = append(messages, *m) + } + return messages +} diff --git a/internal/gitaly/storage/raftmgr/transport_test.go b/internal/gitaly/storage/raftmgr/transport_test.go new file mode 100644 index 00000000000..c15bbdd8e80 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/transport_test.go @@ -0,0 +1,238 @@ +package raftmgr + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" + "go.etcd.io/etcd/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +func TestNoopTransport_Send(t *testing.T) { + t.Parallel() + + mustMarshalProto := func(msg proto.Message) []byte { + data, err := proto.Marshal(msg) + if err != nil { + panic(fmt.Sprintf("failed to marshal proto: %v", err)) + } + return data + } + + tests := []struct { + name string + setupFunc func(tempDir string) ([]*gitalypb.LogEntry, []raftpb.Message, map[string]string) + expectedOutput map[string]string + }{ + { + name: "No messages", + setupFunc: func(tempDir string) ([]*gitalypb.LogEntry, []raftpb.Message, map[string]string) { + return nil, []raftpb.Message{}, nil + }, + }, + { + name: "Empty Entries", + setupFunc: func(tempDir string) ([]*gitalypb.LogEntry, []raftpb.Message, map[string]string) { + return nil, []raftpb.Message{ + { + Type: raftpb.MsgApp, + From: 2, + To: 1, + Term: 1, + Entries: []raftpb.Entry{}, // Empty Entries + }, + }, nil + }, + }, + { + name: "Messages with already packed data", + setupFunc: func(tempDir string) ([]*gitalypb.LogEntry, []raftpb.Message, map[string]string) { + logEntry := &gitalypb.LogEntry{ + RelativePath: "relative-path", + Operations: []*gitalypb.LogEntry_Operation{ + { + Operation: &gitalypb.LogEntry_Operation_CreateHardLink_{ + CreateHardLink: &gitalypb.LogEntry_Operation_CreateHardLink{ + SourcePath: []byte("source"), + DestinationPath: []byte("destination"), + }, + }, + }, + }, + } + initialMessage := gitalypb.RaftMessageV1{ + Id: 1, + ClusterId: "44c58f50-0a8b-4849-bf8b-d5a56198ea7c", + AuthorityName: "sample-storage", + PartitionId: 1, + LogEntry: logEntry, + LogData: &gitalypb.RaftMessageV1_Packed{Packed: &gitalypb.RaftMessageV1_PackedLogData{Data: []byte("already packed data")}}, + } + messages := []raftpb.Message{ + { + Type: raftpb.MsgApp, + From: 2, + To: 1, + Term: 1, + Index: 1, + Entries: []raftpb.Entry{{Index: uint64(1), Type: raftpb.EntryNormal, Data: mustMarshalProto(&initialMessage)}}, + }, + } + return []*gitalypb.LogEntry{logEntry}, messages, nil + }, + }, + { + name: "Messages with referenced data", + setupFunc: func(tempDir string) ([]*gitalypb.LogEntry, []raftpb.Message, map[string]string) { + // Simulate a log entry dir with files + fileContents := map[string]string{ + "1": "file1 content", + "2": "file2 content", + "3": "file3 content", + } + for name, content := range fileContents { + require.NoError(t, os.WriteFile(filepath.Join(tempDir, name), []byte(content), 0o644)) + } + + // Construct message with ReferencedLogData + logEntry := &gitalypb.LogEntry{ + RelativePath: "relative-path", + Operations: []*gitalypb.LogEntry_Operation{ + { + Operation: &gitalypb.LogEntry_Operation_CreateHardLink_{ + CreateHardLink: &gitalypb.LogEntry_Operation_CreateHardLink{ + SourcePath: []byte("source"), + DestinationPath: []byte("destination"), + }, + }, + }, + }, + } + initialMessage := gitalypb.RaftMessageV1{ + Id: 1, + ClusterId: "44c58f50-0a8b-4849-bf8b-d5a56198ea7c", + AuthorityName: "sample-storage", + PartitionId: 1, + LogEntry: logEntry, + LogData: &gitalypb.RaftMessageV1_Referenced{Referenced: &gitalypb.RaftMessageV1_ReferencedLogData{}}, + } + + messages := []raftpb.Message{ + { + Type: raftpb.MsgApp, + From: 2, + To: 1, + Term: 1, + Index: 1, + Entries: []raftpb.Entry{ + {Index: uint64(1), Type: raftpb.EntryNormal, Data: mustMarshalProto(&initialMessage)}, + }, + }, + } + return []*gitalypb.LogEntry{logEntry}, messages, fileContents + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Create a temporary directory + tempDir := testhelper.TempDir(t) + defer func() { require.NoError(t, os.RemoveAll(tempDir)) }() + + // Execute setup function to prepare messages and any necessary file contents + entries, messages, expectedContents := tc.setupFunc(tempDir) + + // Setup logger and transport + logger := testhelper.SharedLogger(t) + transport := NewNoopTransport(logger, true) + + // Execute the Send operation + require.NoError(t, transport.Send(context.Background(), func(storage.LSN) string { return tempDir }, messages)) + + // Fetch recorded messages for verification + recordedMessages := transport.GetRecordedMessages() + + require.Len(t, recordedMessages, len(messages)) + + // Messages must be sent in order. + for i := range messages { + require.Equal(t, messages[i].Type, recordedMessages[i].Type) + require.Equal(t, messages[i].From, recordedMessages[i].From) + require.Equal(t, messages[i].To, recordedMessages[i].To) + require.Equal(t, messages[i].Term, recordedMessages[i].Term) + require.Equal(t, messages[i].Index, recordedMessages[i].Index) + + if len(messages[i].Entries) == 0 { + require.Empty(t, recordedMessages[i].Entries) + } else { + var resultMessage gitalypb.RaftMessageV1 + require.NoError(t, proto.Unmarshal(recordedMessages[i].Entries[0].Data, &resultMessage)) + + testhelper.ProtoEqual(t, entries[i], resultMessage.GetLogEntry()) + + packedData, ok := resultMessage.GetLogData().(*gitalypb.RaftMessageV1_Packed) + require.True(t, ok) + + tarballData := packedData.Packed.GetData() + require.NotEmpty(t, tarballData) + + // Optionally verify packed data if expected + if expectedContents != nil { + var resultMessage gitalypb.RaftMessageV1 + require.NoError(t, proto.Unmarshal(recordedMessages[0].Entries[0].Data, &resultMessage)) + packedData, ok := resultMessage.GetLogData().(*gitalypb.RaftMessageV1_Packed) + require.True(t, ok) + // Verify tarball content matches expectations + verifyPackedData(t, packedData.Packed.GetData(), expectedContents) + } + } + } + }) + } +} + +// verifyPackedData checks if the tarballData contains files with content as expected +func verifyPackedData(t *testing.T, tarballData []byte, expectedContents map[string]string) { + // Create a tar reader + buf := bytes.NewReader(tarballData) + tarReader := tar.NewReader(buf) + + extractedContents := make(map[string]string) + + for { + // Read the next header + header, err := tarReader.Next() + if errors.Is(err, io.EOF) { + // End of tar archive + break + } + require.NoError(t, err) + + // We skip directories since they don't contain data + if header.Typeflag == tar.TypeDir { + continue + } + + // Read file content + var extractedData bytes.Buffer + _, err = io.Copy(&extractedData, tarReader) + require.NoError(t, err) + + // Store extracted content in the map + extractedContents[header.Name] = extractedData.String() + } + + require.Equal(t, expectedContents, extractedContents) +} -- GitLab From 74fb9dc404eab94b29c33d35623a08be73a80fa6 Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Mon, 28 Oct 2024 12:14:04 +0700 Subject: [PATCH 16/19] raft: Implement core raft manager This commit introduces an initial iteration of Raft management for Raft consensus algorithm. It lays out the core components necessary for managing Raft log entries, state persistence, and write-ahead logging. The Raft algorithm will help Gitaly manage distributed transactions more reliably by ensuring consensus across nodes, which is crucial for maintaining consistency and facilitating leader election and membership changes in clusters. This integration leverages the etcd/raft library for its robust implementation and community validation. The manager follows the flow described in library's implementation note: https://pkg.go.dev/go.etcd.io/etcd/raft/v3#section-readme In summary, the Raft manager starts a goroutine that polls new Raft state of a partition Rafr group from Raft library. When so, it performs the following action in order: - Handle states (soft & hard) - Persist volatile log entries - Send external messages out - Publish committed entries to state machine. All essential states are saved in file-based KV database so that Raft can resume its activity after a restart. Log entries operations are powered by WAL (TransactionManager at et al). Things that are not covered in this stage: * Networking is out of the picture. Although the log data is packed before sending, the messages are swallowed by a no-op transportation. It will be implemented here: https://gitlab.com/gitlab-org/gitaly/-/issues/6401 * The only peer is the current node. The node ID is set to 1 statically. The node ID will be automatically assigned to a replica based on the ConfChange sequence. The transportation layer will capture the initial message and start replicas proactively: https://gitlab.com/gitlab-org/gitaly/-/issues/6304 * The proposal doesn't wait until a log entry is applied. Gitaly unlocks the caller as soon as a log entry is committed. This approach aligns with the current flow of Transaction Manager. Autocompacting is ignored. https://gitlab.com/gitlab-org/gitaly/-/issues/6463 --- internal/gitaly/storage/raftmgr/manager.go | 686 ++++++++++++++++++ .../storage/raftmgr/persistent_state.go | 239 ++++++ internal/gitaly/storage/wal.go | 42 ++ 3 files changed, 967 insertions(+) create mode 100644 internal/gitaly/storage/raftmgr/manager.go create mode 100644 internal/gitaly/storage/raftmgr/persistent_state.go create mode 100644 internal/gitaly/storage/wal.go diff --git a/internal/gitaly/storage/raftmgr/manager.go b/internal/gitaly/storage/raftmgr/manager.go new file mode 100644 index 00000000000..d1e0a89e0e9 --- /dev/null +++ b/internal/gitaly/storage/raftmgr/manager.go @@ -0,0 +1,686 @@ +package raftmgr + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" + "gitlab.com/gitlab-org/gitaly/v16/internal/safe" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" + "go.etcd.io/etcd/raft/v3" + "go.etcd.io/etcd/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +const ( + defaultMaxSizePerMsg = 10 * 1024 * 1024 + defaultMaxInflightMsgs = 256 +) + +// ready manages the readiness signaling. The caller polls from the channel. +type ready struct { + c chan struct{} // Channel to signal readiness + once sync.Once // Ensures readiness is signaled only once +} + +// set signals readiness by closing the channel, ensuring it +// happens only once. +func (r *ready) set() { + r.once.Do(func() { close(r.c) }) +} + +// ManagerOptions encapsulates optional parameters for configuring +// the Manager. +type ManagerOptions struct { + // readyTimeout sets an timeout when waiting for Raft to be ready. + readyTimeout time.Duration + // opTimeout sets a timeout for propose, append, and commit operations. It's meaningful in tests so that we + // could detect deadlocks and intolerant performance degrades. + opTimeout time.Duration + // entryRecorder indicates whether to store all log entries processed by Raft. It is primarily + // used for testing purposes. + entryRecorder *EntryRecorder + // recordTransport indicates whether to store all messages supposed to sent over the network. It is + // primarily used for testing purposes. + recordTransport bool +} + +// OptionFunc defines a function type for applying optional +// configuration to ManagerOptions. +type OptionFunc func(opt ManagerOptions) ManagerOptions + +// WithReadyTimeout sets a timeout when waiting for Raft service to be ready. +// The default timeout is 5 * election timeout. +func WithReadyTimeout(t time.Duration) OptionFunc { + return func(opt ManagerOptions) ManagerOptions { + opt.readyTimeout = t + return opt + } +} + +// WithOpTimeout sets a timeout for individual operation. It's supposed to be used in testing environment only. +func WithOpTimeout(t time.Duration) OptionFunc { + return func(opt ManagerOptions) ManagerOptions { + opt.opTimeout = t + return opt + } +} + +// WithEntryRecorder configures ManagerOptions to store Raft log entries, useful for testing scenarios. It should +// not be enabled outside of testing environments. +func WithEntryRecorder(recorder *EntryRecorder) OptionFunc { + return func(opt ManagerOptions) ManagerOptions { + opt.entryRecorder = recorder + return opt + } +} + +// WithRecordTransport configures ManagerOptions to store Raft transport messages, useful for testing scenarios. It +// should not be enabled outside of testing environments. +func WithRecordTransport() OptionFunc { + return func(opt ManagerOptions) ManagerOptions { + opt.recordTransport = true + return opt + } +} + +// Manager orchestrates the Raft consensus protocol, managing configuration, +// state synchronization, and communication between distributed nodes. +type Manager struct { + sync.Mutex + ctx context.Context + cancel context.CancelCauseFunc + + // Configurations. + ptnID storage.PartitionID // Unique identifier for the managed partition + storage string // Name of the storage this partition belong to. + node raft.Node // etcd/raft node representation. + raftCfg config.Raft // etcd/raft configurations. + options ManagerOptions // Additional manager options + logger log.Logger // Logger for internal activities. + db keyvalue.Transactioner // Key-value DB to resume Raft activities after restart. + wal storage.WriteAheadLog // Write-ahead logging manager. + state *persistentState // Persistent storage for Raft log entries and states. + registry *Registry // Event registry, used to track events emitted by the manager. + transport Transport // Manage data transportation between nodes. + leadership *Leadership // Store up-to-date leadership information. + syncer safe.Syncer + wg sync.WaitGroup + ready *ready + started bool + + // EntryRecorder stores the list of internal Raft log entries such as config changes, empty verification + // entrires, etc. They are used in testing environments. + EntryRecorder *EntryRecorder +} + +// applyOptions creates a set of default manager options and then apply option setting functions. It also validates the +// resulting options. +func applyOptions(raftCfg config.Raft, opts []OptionFunc) (ManagerOptions, error) { + baseRTT := time.Duration(raftCfg.RTTMilliseconds) * time.Millisecond + options := ManagerOptions{ + // readyTimeout is set to 5 election timeout by default. The Raft manager waits for some couple of + // initial self elections, just in case. + readyTimeout: 5 * time.Duration(raftCfg.ElectionTicks) * baseRTT, + } + for _, opt := range opts { + options = opt(options) + } + if options.readyTimeout == 0 { + return options, fmt.Errorf("readyTimeout must not be zero") + } else if options.readyTimeout < time.Duration(raftCfg.ElectionTicks)*baseRTT { + return options, fmt.Errorf("readyTimeout must not be less than election timeout") + } + return options, nil +} + +// NewManager creates an instance of Manager. +func NewManager( + partitionID storage.PartitionID, + storageName string, + raftCfg config.Raft, + db keyvalue.Transactioner, + logger log.Logger, + opts ...OptionFunc, +) (*Manager, error) { + options, err := applyOptions(raftCfg, opts) + if err != nil { + return nil, fmt.Errorf("invalid raft manager option: %w", err) + } + + logger = logger.WithFields(log.Fields{ + "component": "raft", + "raft.partition": partitionID, + "raft.authority": storageName, + }) + + // We haven't taken care of network transportation at this point. Let's use a no-op transportation to swallow + // external messages. + transport := NewNoopTransport(logger.WithField("raft.component", "transportation"), options.recordTransport) + return &Manager{ + storage: storageName, + ptnID: partitionID, + raftCfg: raftCfg, + options: options, + db: db, + logger: logger, + registry: NewRegistry(), + transport: transport, + syncer: safe.NewSyncer(), + leadership: NewLeadership(), + ready: &ready{c: make(chan struct{})}, + EntryRecorder: options.entryRecorder, + }, nil +} + +// Run starts the Raft manager, including etcd/raft Node instance and internal processing goroutine. +func (mgr *Manager) Run(ctx context.Context, wal storage.WriteAheadLog) error { + if !mgr.raftCfg.Enabled { + return fmt.Errorf("raft is not enabled") + } + + mgr.Lock() + defer mgr.Unlock() + + if mgr.started { + return fmt.Errorf("raft manager for partition %q is re-used", mgr.ptnID) + } + mgr.started = true + + mgr.ctx, mgr.cancel = context.WithCancelCause(ctx) + mgr.wal = wal + mgr.state = newPersistentState(mgr.db, wal) + + bootstrapped, err := mgr.state.Bootstrapped() + if err != nil { + return fmt.Errorf("failed to load raft initial state: %w", err) + } + + // A partition is identified globally by (authorityName, partitionID) tuple. A node (including primary) is + // identified by (authorityName, partitionID, nodeID). Instead of setting the node ID statically in the Raft + // config, Raft manager uses the LSN of config change log entry that contains node joining event. This approach + // yields some benefits: + // - No need for setting node ID statically, thus avoiding dual storage name/node ID identity system. + // - No need for a global node registration system. + // - Work better in the scenario where a node leaves and then re-join the cluster. Each joining event leads to a + // new unique node ID. + // At this stage, Gitaly supports single-node clusters only. Thus, the node ID is always equal to 1. When + // networking replication is implemented, newly joint replicas are aware of their own node ID by examining + // messages at the transportation layer. When so, a node could self-bootstrap replicas. + // Issue: https://gitlab.com/gitlab-org/gitaly/-/issues/6304 + var nodeID uint64 = 1 + + config := &raft.Config{ + ID: nodeID, + ElectionTick: int(mgr.raftCfg.ElectionTicks), + HeartbeatTick: int(mgr.raftCfg.HeartbeatTicks), + Storage: mgr.state, + MaxSizePerMsg: defaultMaxSizePerMsg, + MaxInflightMsgs: defaultMaxInflightMsgs, + Logger: &raftLogger{logger: mgr.logger.WithFields(log.Fields{"raft.component": "manager"})}, + // etcd/raft supports Proposal Forwarding. When a node receives a proposal, and it does not hold the + // primary role, it forwards the entry to the current leader. Unfortunately, it doesn't work for Gitaly. + // + // In Gitaly WAL, a transaction is verified, committed, and then applied in order. Transactions are serialized. + // The next transaction is verified based on the latest state after applying the previous one. Raft is + // involved in the process at the committing phase. The log entry is broadcasted after an entry is + // verified and admitted. In addition, a transaction depends on a snapshot. This snapshot is essentially + // the point-of-time state of a partition that a transaction reads the data from. + // + // If Proposal Forwarding is used, two nodes are allowed to accept mutator requests simultaneously and + // then commit transactions independently. Although the resulting log entries are handled by the + // primary, the latter entry is not verified against the prior one. + // + // Replica: A -> B -> C -> Start D -> Forward to Primary + // Primary: A -> B -> C -> Start E -> Commit E -> Receive D -> Commit D + // => D is not verified against E even though E commits before D. + // + // As a result, we disable proposal forwarding in favor of request forwarding/proxying instead. + // For more information: https://gitlab.com/gitlab-org/gitaly/-/issues/6465 + DisableProposalForwarding: true, + } + + if !bootstrapped { + // When the raft group for this partition bootstraps for the first time, it is typically the next step + // after that partition is created by the storage manager. At this stage, the only peer is itself. + peers := []raft.Peer{{ID: nodeID}} + mgr.node = raft.StartNode(config, peers) + } else { + // Applied index is set to the latest committed log entry index processed by the application. WAL + // doesn't wait until the application phase. As soon as a log entry is committed, it unlocks clients + // return results. Thus, the Applied index (from etcd/raft point of view) is essentially WAL's + // committedLSN. + config.Applied = uint64(mgr.wal.LastLSN()) + mgr.node = raft.RestartNode(config) + } + + go mgr.run(bootstrapped) + return nil +} + +func (mgr *Manager) run(bootstrapped bool) { + defer func() { + if p := recover(); p != nil { + err, ok := p.(error) + if !ok { + err = fmt.Errorf("raft loop panic: %v", p) + } + mgr.logger.WithError(err).Error("raft event loop panic") + mgr.cancel(err) + + } + }() + + mgr.wg.Add(1) + defer mgr.wg.Done() + + ticker := time.NewTicker(time.Duration(mgr.raftCfg.RTTMilliseconds) * time.Millisecond) + defer ticker.Stop() + + // If the Raft group for this partition has not been bootstrapped, Raft needs to wait until complete the first + // conf change event before able to handle the first log entry. After each restart, the current Raft state is + // resumed from the persistent storage. A single cluster doesn't need to wait until the next election. Thus, it + // could be marked as ready instantly. + if bootstrapped { + mgr.ready.set() + } + + // Main event processing loop. + for { + select { + case <-ticker.C: + // etcd/raft library implements an internal tick clock. The application must increase the tick + // manually. Election and timeout depends on the number of ticks. + mgr.node.Tick() + case rd := <-mgr.node.Ready(): + if err := mgr.handleReady(&rd); err != nil { + mgr.ready.set() + if errors.Is(err, context.Canceled) { + mgr.logger.WithError(err).Info("raft event loop stopped") + mgr.cancel(nil) + } else { + mgr.logger.WithError(err).Error("raft event loop failed") + mgr.cancel(err) + } + return + } + mgr.node.Advance() + case <-mgr.ctx.Done(): + return + } + } +} + +// WaitReady waits until the Raft manager is ready to service or exceeds readyTimeout. +func (mgr *Manager) WaitReady() error { + tick := time.NewTicker(mgr.options.readyTimeout) + defer tick.Stop() + + select { + case <-tick.C: + return fmt.Errorf("ready timeout exceeded") + case <-mgr.ready.c: + return nil + } +} + +// Done returns a channel for polling running status of the Raft manager. The caller is expected to call Err() to fetch +// underlying error (if any). +func (mgr *Manager) Done() <-chan struct{} { + return mgr.ctx.Done() +} + +// Err returns the running error of the Raft manager. If the returned value is nil, it means either Raft is still +// running or it exits susccessully. Hence, it should only be called after polling from Done(). +func (mgr *Manager) Err() error { + err := mgr.ctx.Err() + if err != nil && context.Cause(mgr.ctx) != nil { + err = context.Cause(mgr.ctx) + } + return err +} + +// Stop terminates all activities of Raft. +func (mgr *Manager) Stop() { + // The node has not started yet, properly it fails during initialization. + if mgr.node != nil { + mgr.node.Stop() + } + // If the context has been cancelled beforehand, the event processing loop failed with an error. Skip + // cancellation in that case. + if mgr.ctx != nil && mgr.ctx.Err() == nil { + mgr.cancel(nil) + } +} + +// Propose proposes a log entry change to the cluster. The caller is blocked until either the log entry is committed, +// timeout exceeded, or the request is rejected by the cluster. +func (mgr *Manager) Propose(ctx context.Context, logEntry *gitalypb.LogEntry, logEntryPath string) error { + mgr.wg.Add(1) + defer mgr.wg.Done() + + w := mgr.registry.Register() + defer mgr.registry.Untrack(w.ID) + + message := &gitalypb.RaftMessageV1{ + Id: uint64(w.ID), + ClusterId: mgr.raftCfg.ClusterID, + AuthorityName: mgr.storage, + PartitionId: uint64(mgr.ptnID), + LogEntry: logEntry, + LogData: &gitalypb.RaftMessageV1_Referenced{ + Referenced: &gitalypb.RaftMessageV1_ReferencedLogData{ + Path: []byte(logEntryPath), + }, + }, + } + data, err := proto.Marshal(message) + if err != nil { + return fmt.Errorf("marshaling Raft message: %w", err) + } + + // Set an optional timeout to prevent proposal processing takes forever. This option is + // more useful in testing environments. + if mgr.options.opTimeout != 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, mgr.options.opTimeout) + defer cancel() + } + + if err := mgr.node.Propose(ctx, data); err != nil { + return fmt.Errorf("proposing Raft message: %w", err) + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-mgr.ctx.Done(): + return fmt.Errorf("raft manager is closed: %w", mgr.ctx.Err()) + case <-w.C: + return w.Err + } +} + +// handleReady processes the next state signaled by etcd/raft. It includes 3 main stepsS: +// - Persist states, including SoftState, HardState, and Entries (uncommitted). +// - Send messages to other nodes via Transport. +// - Publish commit entries. Committed entries are entries acknowledged by the majority of group members. +// Those steps are noted in https://pkg.go.dev/go.etcd.io/etcd/raft/v3#section-readme. +func (mgr *Manager) handleReady(rd *raft.Ready) error { + // Save current volatile state of a Node. It is used for observability and quick determining the current + // leadership of the group. + if err := mgr.handleSoftState(rd); err != nil { + return fmt.Errorf("handling soft state: %w", err) + } + + // Save essential state to resume its activities after a restart. + if err := mgr.handleHardState(rd); err != nil { + return fmt.Errorf("handling hard state: %w", err) + } + + // Save entries log entries to disk. Those entries haven't been committed yet. They could be overridden by + // entries with the same LSN but with higher term. WAL must cleans up subsequent appended entries if the + // inserting position overlaps. + if err := mgr.saveEntries(rd); err != nil { + return fmt.Errorf("saving entries: %w", err) + } + + // Send outbound messages. From the thesis (10.2) and etcd/raft doc, this process could be pipelined so that + // entries could be sent in parallel as soon as they are persisted to the disk first. However, as WAL serializes + // transactions, only one log entry is processed at a time. Thus, that optimization is not needed. This + // assumption is subject to change in the future if the WAL can process multiple transactions simultaneously. + // Source: https://github.com/ongardie/dissertation/blob/master/stanford.pdf + // + // At this stage, auto-compaction feature is completely ignored. In theory, the application creates "snapshots" + // occasionally. A snapshot is a point-of-time persistent state of the state machine. After a snapshot is + // created, the application can remove all log entries til the point where the snapshot is created. A new + // joining node loads the latest snapshot and replays WAL entries from there. We skipped snapshotting in the + // first iteration because the current node acts as the only primary for a single-node Gitaly cluster. It hasn't + // replicated log entries anywhere else + // + // For more information: https://gitlab.com/gitlab-org/gitaly/-/issues/6463 + if err := mgr.sendMessages(rd); err != nil { + return fmt.Errorf("sending messages: %w", err) + } + + // Mark committed entries in WAL. If the Raft group has only one member, there's a good chance the rd.Entries + // and rd.CommittedEntries overlap (partially or completely). It means those entries are sent to the committed + // state without going the need for sending external messages. + if err := mgr.commitEntries(rd); err != nil { + return fmt.Errorf("processing committed entries: %w", err) + } + return nil +} + +func (mgr *Manager) saveEntries(rd *raft.Ready) error { + if len(rd.Entries) == 0 { + return nil + } + + // Discard all in-flight events having duplicated LSN but with lower term. The corresponding on-disk log entries + // will be wiped by WAL. Events without associated LSN haven't advanced to this stage yet, hence stay instact. + firstLSN := storage.LSN(rd.Entries[0].Index) + mgr.registry.UntrackSince(firstLSN) + + for i := range rd.Entries { + switch rd.Entries[i].Type { + case raftpb.EntryNormal: + lsn := storage.LSN(rd.Entries[i].Index) + + if len(rd.Entries[i].Data) == 0 { + if err := mgr.insertEmptyLogEntry(lsn); err != nil { + return fmt.Errorf("inserting empty log entry: %w", err) + } + } else { + var message gitalypb.RaftMessageV1 + if err := proto.Unmarshal(rd.Entries[i].Data, &message); err != nil { + return fmt.Errorf("unmarshalling entry type: %w", err) + } + + var logEntryPath string + switch m := message.GetLogData().(type) { + case *gitalypb.RaftMessageV1_Packed: + // Log entry data is packed and unpacked in the Transport layer. Before sending + // messages, it packages and compresses log entry directory. Likely, after + // receiving, the data is converted to identical on-disk files. The whole + // process is transparent to Raft manager. + return fmt.Errorf("packed log entry data must be unpacked in Transport layer") + case *gitalypb.RaftMessageV1_Referenced: + logEntryPath = string(m.Referenced.GetPath()) + } + + logEntry := message.GetLogEntry() + // The Raft term is carved into the log entry. LSN and term are only available at this + // stage. Thus, we could not set this value beforehand. + logEntry.Metadata = &gitalypb.LogEntry_Metadata{ + RaftTerm: rd.Entries[i].Index, + RaftType: uint64(gitalypb.RaftMessageType_NORMAL), + } + if err := mgr.wal.AppendLogEntry(mgr.ctx, lsn, logEntry, logEntryPath); err != nil { + return fmt.Errorf("appending log entry: %w", err) + } + mgr.recordEntryIfNeeded(false, lsn, logEntry) + + // Associate the LSN of the log entry to the event. + mgr.registry.AssignLSN(EventID(message.GetId()), lsn) + } + case raftpb.EntryConfChange, raftpb.EntryConfChangeV2: + if err := mgr.saveConfigChange(rd.Entries[i]); err != nil { + return fmt.Errorf("saving config change: %w", err) + } + default: + return fmt.Errorf("raft entry type not supported: %s", rd.Entries[i].Type) + } + } + return nil +} + +func (mgr *Manager) saveConfigChange(entry raftpb.Entry) error { + if entry.Type == raftpb.EntryConfChange { + var cc raftpb.ConfChange + if err := cc.Unmarshal(entry.Data); err != nil { + return fmt.Errorf("unmarshalling EntryConfChange: %w", err) + } + } else { + var cc raftpb.ConfChangeV2 + if err := cc.Unmarshal(entry.Data); err != nil { + return fmt.Errorf("unmarshalling EntryConfChangeV2: %w", err) + } + } + + ctx := mgr.ctx + if mgr.options.opTimeout != 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, mgr.options.opTimeout) + defer cancel() + } + + // Raft manager haven't supported membership management, yet. The only member of the group is the one who + // initiates the Raft group. Thus, there is no need to maintain the list of nodes inside DB. We now place an + // entry log entry, similar to empty verification message. This placement is temporary. + lsn := storage.LSN(entry.Index) + logEntry, err := mgr.wal.InsertLogEntry(ctx, lsn, nil, &gitalypb.LogEntry_Metadata{ + RaftTerm: uint64(lsn), + RaftType: uint64(gitalypb.RaftMessageType_CONFIG_CHANGE), + }) + if err != nil { + return fmt.Errorf("inserting config change log entry: %w", err) + } + mgr.recordEntryIfNeeded(true, lsn, logEntry) + return nil +} + +// insertEmptyLogEntry inserts an empty log entry at the desirable position in WAL. +func (mgr *Manager) insertEmptyLogEntry(lsn storage.LSN) error { + ctx := mgr.ctx + if mgr.options.opTimeout != 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, mgr.options.opTimeout) + defer cancel() + } + logEntry, err := mgr.wal.InsertLogEntry(ctx, lsn, nil, &gitalypb.LogEntry_Metadata{ + RaftTerm: uint64(lsn), + RaftType: uint64(gitalypb.RaftMessageType_VERIFICATION), + }) + if err != nil { + return fmt.Errorf("inserting empty verification log entry: %w", err) + } + mgr.recordEntryIfNeeded(true, lsn, logEntry) + return nil +} + +func (mgr *Manager) commitEntries(rd *raft.Ready) error { + var shouldNotify bool + for i := range rd.Entries { + switch rd.Entries[i].Type { + case raftpb.EntryNormal: + lsn := storage.LSN(rd.Entries[i].Index) + + var message gitalypb.RaftMessageV1 + if err := proto.Unmarshal(rd.Entries[i].Data, &message); err != nil { + return fmt.Errorf("unmarshalling entry type: %w", err) + } + if err := mgr.wal.CommitLogEntry(mgr.ctx, lsn); err != nil { + return fmt.Errorf("processing committed entry: %w", err) + } + + // When Raft injects internal log entries, there are two main scenarios: + // - Those entries are piggy-backed when WAL processes a transaction. + // - They are injected independently while WAL loop is idle. + // In the first scenario, after complete a transaction, WAL circles back to process entries + // since the last committed LSN. Internal log entries are then processed as normal. + // In the second scenario, Raft needs to notify WAL about new committed entries. Otherwise, they + // are not applied until the next incoming transaction. + selfSubmit := mgr.registry.Untrack(EventID(message.GetId())) + metadata := message.GetLogEntry().GetMetadata() + if metadata.GetRaftType() != uint64(gitalypb.RaftMessageType_NORMAL) || !selfSubmit { + shouldNotify = true + } + case raftpb.EntryConfChange, raftpb.EntryConfChangeV2: + if err := mgr.commitConfChange(rd.Entries[i]); err != nil { + return fmt.Errorf("processing config change: %w", err) + } + shouldNotify = true + default: + return fmt.Errorf("raft entry type not supported: %s", rd.Entries[i].Type) + } + } + if shouldNotify { + mgr.wal.NotifyNewCommittedEntry() + } + return nil +} + +func (mgr *Manager) commitConfChange(entry raftpb.Entry) error { + var cc raftpb.ConfChangeI + if entry.Type == raftpb.EntryConfChange { + var cc1 raftpb.ConfChange + if err := cc1.Unmarshal(entry.Data); err != nil { + return fmt.Errorf("unmarshalling EntryConfChange: %w", err) + } + cc = cc1 + } else { + var cc2 raftpb.ConfChangeV2 + if err := cc2.Unmarshal(entry.Data); err != nil { + return fmt.Errorf("unmarshalling EntryConfChangeV2: %w", err) + } + cc = cc2 + } + if err := mgr.wal.CommitLogEntry(mgr.ctx, storage.LSN(entry.Index)); err != nil { + return fmt.Errorf("committing log entry: %w", err) + } + confState := mgr.node.ApplyConfChange(cc) + if err := mgr.state.SaveConfState(confState); err != nil { + return fmt.Errorf("saving config state: %w", err) + } + + // Mark Raft ready for service after the first commit change entry is applied. After restart, + // we don't need this anymore. + mgr.ready.set() + return nil +} + +func (mgr *Manager) sendMessages(rd *raft.Ready) error { + return mgr.transport.Send(mgr.ctx, mgr.wal.LSNDirPath, rd.Messages) +} + +func (mgr *Manager) handleSoftState(rd *raft.Ready) error { + state := rd.SoftState + if state == nil { + return nil + } + prevLeader := mgr.leadership.GetLeaderID() + changed, duration := mgr.leadership.SetLeader(state.Lead, state.RaftState == raft.StateLeader) + + if changed { + mgr.logger.WithFields(log.Fields{ + "raft.leader_id": mgr.leadership.GetLeaderID(), + "raft.is_leader": mgr.leadership.IsLeader(), + "raft.previous_leader_id": prevLeader, + "raft.leadership_duration": duration, + }).Info("leadership updated") + } + return nil +} + +func (mgr *Manager) handleHardState(rd *raft.Ready) error { + if raft.IsEmptyHardState(rd.HardState) { + return nil + } + if err := mgr.state.SaveHardState(rd.HardState); err != nil { + return fmt.Errorf("saving hard state: %w", err) + } + return nil +} + +func (mgr *Manager) recordEntryIfNeeded(fromRaft bool, lsn storage.LSN, logEntry *gitalypb.LogEntry) { + if mgr.EntryRecorder != nil { + mgr.EntryRecorder.Record(fromRaft, lsn, logEntry) + } +} diff --git a/internal/gitaly/storage/raftmgr/persistent_state.go b/internal/gitaly/storage/raftmgr/persistent_state.go new file mode 100644 index 00000000000..52612224ffe --- /dev/null +++ b/internal/gitaly/storage/raftmgr/persistent_state.go @@ -0,0 +1,239 @@ +package raftmgr + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v4" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" + "go.etcd.io/etcd/raft/v3" + "go.etcd.io/etcd/raft/v3/raftpb" + "google.golang.org/protobuf/proto" +) + +var ( + // KeyHardState stores the current state of a Node to be saved to DB before Messages are sent. + KeyHardState = []byte("hard_state") + // KeyConfState stores the current config state used when generating snapshot. + KeyConfState = []byte("conf_state") +) + +// persistentState is responsible for managing the persistence of Raft's state, log entries, and configuration state, +// using a write-ahead log (WAL) and a KV database for durability. It ensures that state is saved and retrievable across +// restarts and node failures, thus aiding in recovery and consistency within the Raft protocol. Raft states are updated +// after each batch of entry processing. Thus, we cannot use WAL as the persistent storage for that kind of data. +// This struct implements `raft.Storage` interface from etcd/raft library. +type persistentState struct { + db keyvalue.Transactioner + wal storage.WriteAheadLog +} + +func newPersistentState(db keyvalue.Transactioner, wal storage.WriteAheadLog) *persistentState { + return &persistentState{ + db: db, + wal: wal, + } +} + +// Entries implements raft.Storage. +func (ps *persistentState) Entries(lo uint64, hi uint64, maxSize uint64) ([]raftpb.Entry, error) { + firstLSN := uint64(ps.wal.FirstLSN()) + lastLSN := uint64(ps.wal.LastLSN()) + if lo < firstLSN { + return nil, raft.ErrCompacted + } + if firstLSN > lastLSN { + return nil, raft.ErrUnavailable + } + if hi > lastLSN+1 { + return nil, fmt.Errorf("reading out-of-bound entries %d > %d", hi, lastLSN) + } + + var entries []raftpb.Entry + + for lsn := lo; lsn < hi; lsn++ { + entry, err := ps.wal.ReadLogEntry(storage.LSN(lsn)) + if err != nil { + return nil, raft.ErrCompacted + } + msg, err := proto.Marshal(entry) + if err != nil { + return nil, fmt.Errorf("marshaling log entry: %w", err) + } + entries = append(entries, raftpb.Entry{ + Term: entry.GetMetadata().GetRaftTerm(), + Index: lsn, + Type: raftpb.EntryType(entry.GetMetadata().GetRaftType()), + Data: msg, + }) + } + return entries, nil +} + +// InitialState retrieves the initial Raft HardState and ConfState from persistent storage. It is used to initialize the +// Raft node with the previously saved state. +func (ps *persistentState) InitialState() (raftpb.HardState, raftpb.ConfState, error) { + hardState, err := ps.readHardState() + if err != nil { + return raftpb.HardState{}, raftpb.ConfState{}, fmt.Errorf("reading hard state: %w", err) + } + + confState, err := ps.readConfState() + if err != nil { + return raftpb.HardState{}, raftpb.ConfState{}, fmt.Errorf("reading conf state: %w", err) + } + + return hardState, confState, nil +} + +// LastIndex returns the last index of all entries currently available in the log. +// This corresponds to the last LSN in the write-ahead log. +func (ps *persistentState) LastIndex() (uint64, error) { + return uint64(ps.wal.LastLSN()), nil +} + +// FirstIndex returns the first index of all entries currently available in the log. +// This corresponds to the first LSN in the write-ahead log. +func (ps *persistentState) FirstIndex() (uint64, error) { + return uint64(ps.wal.FirstLSN()), nil +} + +// Snapshot returns the latest snapshot of the state machine. As we haven't supported autocompaction feature, this +// method always returns Unavailable error. +// For more information: https://gitlab.com/gitlab-org/gitaly/-/issues/6463 +func (ps *persistentState) Snapshot() (raftpb.Snapshot, error) { + return raftpb.Snapshot{}, raft.ErrSnapshotTemporarilyUnavailable +} + +// Term returns the term of the entry at a given index. It retrieves the term from log entry's metadata. If the +// corresponding log entry is compacted, it retrieves the term from the hard state conditionallly. +func (ps *persistentState) Term(i uint64) (uint64, error) { + firstLSN := uint64(ps.wal.FirstLSN()) + lastLSN := uint64(ps.wal.LastLSN()) + if i < firstLSN { + // If firstLSN > lastLSN, it means all log entries are committed and cleaned up from WAL. If the queried + // log entry is equal to lastLSN, we could imply its term from the hard state. + if i == lastLSN { + if hardState, err := ps.readHardState(); err == nil && hardState.Commit == i { + return hardState.Term, nil + } + } + return 0, raft.ErrCompacted + } + if i > lastLSN { + return 0, raft.ErrUnavailable + } + + entry, err := ps.wal.ReadLogEntry(storage.LSN(i)) + if err != nil { + // If the entry does not exist here, there's a good chance WAL cleaned it up after we query the first + // LSN. Similar to the above case, we could fall back to use hard state. + if i == lastLSN { + if hardState, err := ps.readHardState(); err == nil && hardState.Commit == i { + return hardState.Term, nil + } + } + return 0, fmt.Errorf("reading log entry: %w", err) + } + return entry.GetMetadata().GetRaftTerm(), nil +} + +// Bootstrapped checks whether the node has completed its initial bootstrap process +// by verifying the existence of a saved hard state. +func (ps *persistentState) Bootstrapped() (bool, error) { + var hardState gitalypb.RaftHardStateV1 + if err := ps.readKey(KeyHardState, &hardState); err != nil { + if errors.Is(err, badger.ErrKeyNotFound) { + return false, nil + } + return false, err + } + return true, nil +} + +// SaveConfState persists the current Raft configuration state to disk, ensuring that configuration changes are durable. +func (ps *persistentState) SaveHardState(hardState raftpb.HardState) error { + return ps.setKey(KeyHardState, &gitalypb.RaftHardStateV1{ + Term: hardState.Term, + Vote: hardState.Vote, + Commit: hardState.Commit, + }) +} + +func (ps *persistentState) readHardState() (raftpb.HardState, error) { + var hardState gitalypb.RaftHardStateV1 + if err := ps.readKey(KeyHardState, &hardState); err != nil { + if errors.Is(err, badger.ErrKeyNotFound) { + return raftpb.HardState{}, nil + } + return raftpb.HardState{}, err + } + return raftpb.HardState{ + Term: hardState.GetTerm(), + Vote: hardState.GetVote(), + Commit: hardState.GetCommit(), + }, nil +} + +// SaveConfState persists node's current conf state. It is used when generating snapshot. +func (ps *persistentState) SaveConfState(confState *raftpb.ConfState) error { + return ps.setKey(KeyConfState, &gitalypb.RaftConfStateV1{ + Voters: confState.Voters, + Learners: confState.Learners, + VotersOutgoing: confState.VotersOutgoing, + LearnersNext: confState.LearnersNext, + AutoLeave: confState.AutoLeave, + }) +} + +func (ps *persistentState) readConfState() (raftpb.ConfState, error) { + var confState gitalypb.RaftConfStateV1 + if err := ps.readKey(KeyConfState, &confState); err != nil { + if errors.Is(err, badger.ErrKeyNotFound) { + return raftpb.ConfState{}, nil + } + return raftpb.ConfState{}, err + } + return raftpb.ConfState{ + Voters: confState.GetVoters(), + Learners: confState.GetLearners(), + VotersOutgoing: confState.GetVotersOutgoing(), + LearnersNext: confState.GetLearnersNext(), + AutoLeave: confState.GetAutoLeave(), + }, nil +} + +// setKey marshals and stores a given protocol buffer message into the database under the given key. +func (ps *persistentState) setKey(key []byte, value proto.Message) error { + marshaledValue, err := proto.Marshal(value) + if err != nil { + return fmt.Errorf("marshal value: %w", err) + } + + writeBatch := ps.db.NewWriteBatch() + defer writeBatch.Cancel() + + if err := writeBatch.Set(key, marshaledValue); err != nil { + return fmt.Errorf("set: %w", err) + } + + return writeBatch.Flush() +} + +// readKey reads a key from the database and unmarshals its value in to the destination protocol +// buffer message. +func (ps *persistentState) readKey(key []byte, destination proto.Message) error { + return ps.db.View(func(txn keyvalue.ReadWriter) error { + item, err := txn.Get(key) + if err != nil { + return fmt.Errorf("get: %w", err) + } + + return item.Value(func(value []byte) error { return proto.Unmarshal(value, destination) }) + }) +} + +// Compile-time type check. +var _ = (raft.Storage)(&persistentState{}) diff --git a/internal/gitaly/storage/wal.go b/internal/gitaly/storage/wal.go new file mode 100644 index 00000000000..b74b494af9b --- /dev/null +++ b/internal/gitaly/storage/wal.go @@ -0,0 +1,42 @@ +package storage + +import ( + "context" + + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" +) + +// WriteAheadLog is an abstract interface for Gitaly's WAL. +// It defines methods to manage log entries in a write-ahead log system for use with Raft consensus. +type WriteAheadLog interface { + // AppendLogEntry appends a new log entry at the specified LSN (Log Sequence Number), + // using the provided log entry data and file path, ensuring the entry is durably written. + AppendLogEntry(ctx context.Context, lsn LSN, entry *gitalypb.LogEntry, filePath string) error + + // CommitLogEntry marks the log entry at the given LSN as committed, which means it has been accepted + // by a majority of Raft members and can be safely applied to the state machine. + CommitLogEntry(ctx context.Context, lsn LSN) error + + // NotifyNewCommittedEntry notifies WAL that a log entry has been committed, allowing them to proceed with + // processing. The caller should only notify new commits if WAL is idle or the new committed entries are + // issued by Raft itself. Otherwise, WAL is waken up too frequently. + NotifyNewCommittedEntry() + + // InsertLogEntry inserts a KV log entry into the WAL at the specified LSN. The caller can manipulate the data + // by passing a modification function. It's responsibility of the caller NOT TO append/insert any log entries + // while this function is running. All adjacent log entries at and after inserting position are wiped. + InsertLogEntry(ctx context.Context, lsn LSN, txnFunc func(keyvalue.ReadWriter) error, metadata *gitalypb.LogEntry_Metadata) (*gitalypb.LogEntry, error) + + // ReadLogEntry reads and returns the log entry stored at the specified LSN within the WAL. + ReadLogEntry(lsn LSN) (*gitalypb.LogEntry, error) + + // LastLSN returns the last LSN that has been appended to the WAL, indicating the latest position. + LastLSN() LSN + + // FirstLSN retrieves the first LSN available in the WAL, useful for identifying the oldest log entry retained. + FirstLSN() LSN + + // LSNDirPath provides the directory path where files corresponding to a specific LSN are stored. + LSNDirPath(lsn LSN) string +} -- GitLab From 4f90ca8ea95e3f1da3ed06429ad89c583815ec2f Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Thu, 31 Oct 2024 11:44:40 +0700 Subject: [PATCH 17/19] raft: Integrate Raft manager into Transaction Manager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit integrates the Raft manager into the Transaction Manager to establish a clear boundary between the two components while avoiding circular dependencies. The Raft manager interacts with the Transaction Manager through an abstract `WriteAheadLog` interface, while the Transaction Manager maintains direct access to the Raft manager. The Transaction Manager now implements the `WriteAheadLog` interface, detailing essential operations such as appending a log entry, committing a log entry, and inserting a log entry – all critical for Raft's functioning. With the introduction of Raft, its process is engaged after a transaction is admitted but before appending to the WAL. If Raft is not enabled, the Transaction Manager follows the traditional flow. Otherwise, it delegates the append and commit responsibilities to the Raft manager. The interaction flow is depicted in the following chart: Without Raft TransactionManager New Transaction ┌────┬───────┐ │ ▼ │ ▼ ▼ └─►Initialize───►txn.Commit()─►Verify─┤ Append Commit─►Apply │ │ ▲ ▲ ▼ ▼ │ │ ┌── Run()──────────...───────────Propose──┴───────┘ │ ▲ │ Raft Manager Conf Change │ │ ┌──┴─▼─────┐ │Raft Group├◄───► Network └──────────┘ Occasionally, the Raft Manager inserts internal operational log entries such as configuration changes and empty verification entries. These entries utilize the same LSN sequence and occupy some slots within it. Therefore, the Raft manager must ensure these entries are accurately backfilled to the Transaction Manager in the correct positions. This change has been a significant source of test failures once Raft is enabled. To address this, a recorder is used to track all log entries processed by Raft, aiding in adjusting asserted LSNs to correct positions. A new `GITALY_TEST_RAFT` environment variable has been added to toggle. --- internal/gitaly/storage/storage.go | 2 + .../storage/storagemgr/partition/factory.go | 1 + .../storagemgr/partition/testhelper_test.go | 158 ++++++++- .../partition/transaction_manager.go | 314 +++++++++++++++--- .../transaction_manager_hook_test.go | 11 +- .../partition/transaction_manager_test.go | 288 ++++++++++------ internal/testhelper/testhelper.go | 24 ++ 7 files changed, 646 insertions(+), 152 deletions(-) diff --git a/internal/gitaly/storage/storage.go b/internal/gitaly/storage/storage.go index 29cf42cd1fc..92404d5df85 100644 --- a/internal/gitaly/storage/storage.go +++ b/internal/gitaly/storage/storage.go @@ -137,6 +137,8 @@ type BeginOptions struct { // This is a temporary workaround for some RPCs that do not work well with shared // read-only snapshots yet. ForceExclusiveSnapshot bool + // WithoutRaftReady lets the transaction skip Raft readiness waiting. + WithoutRaftReady bool } // Partition is responsible for a single partition of data. diff --git a/internal/gitaly/storage/storagemgr/partition/factory.go b/internal/gitaly/storage/storagemgr/partition/factory.go index e2fabf936fd..3baad57b7c8 100644 --- a/internal/gitaly/storage/storagemgr/partition/factory.go +++ b/internal/gitaly/storage/storagemgr/partition/factory.go @@ -67,6 +67,7 @@ func (f Factory) New( repoFactory, f.metrics.Scope(storageName), f.logConsumer, + nil, ) } diff --git a/internal/gitaly/storage/storagemgr/partition/testhelper_test.go b/internal/gitaly/storage/storagemgr/partition/testhelper_test.go index 05bc906871c..f38c6ab1668 100644 --- a/internal/gitaly/storage/storagemgr/partition/testhelper_test.go +++ b/internal/gitaly/storage/storagemgr/partition/testhelper_test.go @@ -9,11 +9,15 @@ import ( "os" "path/filepath" "reflect" + "regexp" "sort" + "strconv" "strings" "sync" "testing" + "time" + "github.com/google/uuid" "github.com/prometheus/client_golang/prometheus" io_prometheus_client "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" @@ -34,10 +38,12 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/mode" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/snapshot" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" ) func TestMain(m *testing.M) { @@ -627,13 +633,6 @@ func RequireDatabase(tb testing.TB, ctx context.Context, database keyvalue.Trans if expectedState == nil { expectedState = DatabaseState{} } - // Most of the time, persisted committedLSN is equal to appliedLSN except for some intentional tests. Thus, if - // appliedLSN is asserted, the test should backfill committedLSN if it's not there. - if appliedLSN, appliedExist := expectedState[string(keyAppliedLSN)]; appliedExist { - if _, committedExist := expectedState[string(keyCommittedLSN)]; !committedExist { - expectedState[string(keyCommittedLSN)] = appliedLSN - } - } actualState := DatabaseState{} unexpectedKeys := []string{} @@ -663,13 +662,20 @@ func RequireDatabase(tb testing.TB, ctx context.Context, database keyvalue.Trans // Unmarshal the actual value to the same type as the expected value. actualValue := reflect.New(reflect.TypeOf(expectedValue).Elem()).Interface().(proto.Message) require.NoError(tb, proto.Unmarshal(value, actualValue)) + + if _, isAny := expectedValue.(*anypb.Any); isAny { + // Verify if the database contains the key, but the content does not matter. + actualState[string(key)] = expectedValue + continue + } + actualState[string(key)] = actualValue } return nil })) - require.Empty(tb, unexpectedKeys, "database contains unexpected keys") + require.Emptyf(tb, unexpectedKeys, "database contains unexpected keys") testhelper.ProtoEqual(tb, expectedState, actualState) } @@ -954,6 +960,8 @@ type RepositoryAssertion struct { type StateAssertion struct { // Database is the expected state of the database. Database DatabaseState + // NotOffsetDatabaseInRaft indicates if the LSN in the database should not be shifted. + NotOffsetDatabaseInRaft bool // Directory is the expected state of the manager's state directory in the repository. Directory testhelper.DirectoryState // Repositories is the expected state of the repositories in the storage. The key is @@ -1073,11 +1081,35 @@ func runTransactionTest(t *testing.T, ctx context.Context, tc transactionTestCas ).Scope(storageName) } + // Cluster ID and recorder is kept stable in each test run. + clusterID := uuid.New().String() + var raftEntryRecorder *raftmgr.EntryRecorder + var raftManager *raftmgr.Manager + + createRaftManager := func() *raftmgr.Manager { + raftManager, err := raftmgr.NewManager( + setup.PartitionID, + setup.Config.Storages[0].Name, + config.DefaultRaftConfig(clusterID), + database, + logger, + raftmgr.WithEntryRecorder(raftEntryRecorder), + raftmgr.WithRecordTransport(), + raftmgr.WithOpTimeout(1*time.Minute), + ) + require.NoError(t, err) + return raftManager + } + if testhelper.IsRaftEnabled() { + raftEntryRecorder = raftmgr.NewEntryRecorder() + raftManager = createRaftManager() + } + var ( // managerRunning tracks whether the manager is running or closed. managerRunning bool // transactionManager is the current TransactionManager instance. - transactionManager = NewTransactionManager(setup.PartitionID, logger, database, storageName, storagePath, stateDir, stagingDir, setup.CommandFactory, storageScopedFactory, newMetrics(), setup.Consumer) + transactionManager = NewTransactionManager(setup.PartitionID, logger, database, storageName, storagePath, stateDir, stagingDir, setup.CommandFactory, storageScopedFactory, newMetrics(), setup.Consumer, raftManager) // managerErr is used for synchronizing manager closing and returning // the error from Run. managerErr chan error @@ -1124,7 +1156,12 @@ func runTransactionTest(t *testing.T, ctx context.Context, tc transactionTestCas require.NoError(t, os.RemoveAll(stagingDir)) require.NoError(t, os.Mkdir(stagingDir, mode.Directory)) - transactionManager = NewTransactionManager(setup.PartitionID, logger, database, setup.Config.Storages[0].Name, storagePath, stateDir, stagingDir, setup.CommandFactory, storageScopedFactory, newMetrics(), setup.Consumer) + if testhelper.IsRaftEnabled() { + raftManager = createRaftManager() + } else { + raftManager = nil + } + transactionManager = NewTransactionManager(setup.PartitionID, logger, database, setup.Config.Storages[0].Name, storagePath, stateDir, stagingDir, setup.CommandFactory, storageScopedFactory, newMetrics(), setup.Consumer, raftManager) installHooks(transactionManager, &inflightTransactions, step.Hooks) go func() { @@ -1167,7 +1204,11 @@ func runTransactionTest(t *testing.T, ctx context.Context, tc transactionTestCas require.NoError(t, err) tx := transaction.(*Transaction) - require.Equalf(t, step.ExpectedSnapshotLSN, tx.SnapshotLSN(), "mismatched ExpectedSnapshotLSN") + expectedSnapshotLSN := step.ExpectedSnapshotLSN + if testhelper.IsRaftEnabled() { + expectedSnapshotLSN = raftEntryRecorder.Offset(expectedSnapshotLSN) + } + require.Equal(t, expectedSnapshotLSN, tx.SnapshotLSN()) require.NotEmpty(t, tx.Root(), "empty Root") require.Contains(t, tx.Root(), transactionManager.snapshotsDir()) @@ -1451,7 +1492,11 @@ func runTransactionTest(t *testing.T, ctx context.Context, tc transactionTestCas transaction := openTransactions[step.TransactionID] transaction.WriteCommitGraphs(step.Config) case ConsumerAcknowledge: - transactionManager.AcknowledgeTransaction(step.LSN) + lsn := step.LSN + if testhelper.IsRaftEnabled() { + lsn = raftEntryRecorder.Offset(lsn) + } + transactionManager.AcknowledgeTransaction(lsn) case RepositoryAssertion: require.Contains(t, openTransactions, step.TransactionID, "test error: transaction's snapshot asserted before beginning it") transaction := openTransactions[step.TransactionID] @@ -1518,6 +1563,39 @@ func runTransactionTest(t *testing.T, ctx context.Context, tc transactionTestCas require.NoError(t, err) } + // Most of the time, persisted committedLSN is equal to appliedLSN except for some intentional tests. Thus, if + // appliedLSN is asserted, the test should backfill committedLSN if it's not there. + if appliedLSN, appliedExist := tc.expectedState.Database[string(keyAppliedLSN)]; appliedExist { + if _, committedExist := tc.expectedState.Database[string(keyCommittedLSN)]; !committedExist { + tc.expectedState.Database[string(keyCommittedLSN)] = appliedLSN + } + } + + // If singular Raft cluster is enabled, the Raft handler starts to insert raft log entries. We need to + // offset the expected LSN to account for LSN shifting. + if testhelper.IsRaftEnabled() && raftEntryRecorder.Len() > 0 { + if !tc.expectedState.NotOffsetDatabaseInRaft { + appliedLSN, exist := tc.expectedState.Database[string(keyAppliedLSN)] + if exist { + // If expected applied LSN is present, offset expected LSN. + appliedLSN := appliedLSN.(*gitalypb.LSN) + tc.expectedState.Database[string(keyAppliedLSN)] = raftEntryRecorder.Offset(storage.LSN(appliedLSN.GetValue())).ToProto() + + committedLSN := tc.expectedState.Database[string(keyCommittedLSN)].(*gitalypb.LSN) + tc.expectedState.Database[string(keyCommittedLSN)] = raftEntryRecorder.Offset(storage.LSN(committedLSN.GetValue())).ToProto() + } else { + // Otherwise, the test expects no applied log entry in cases such as invalid transactions. + // Regardless, raft log entries are applied successfully. + if tc.expectedState.Database == nil { + tc.expectedState.Database = DatabaseState{} + } + tc.expectedState.Database[string(keyAppliedLSN)] = raftEntryRecorder.Latest().ToProto() + tc.expectedState.Database[string(keyCommittedLSN)] = raftEntryRecorder.Latest().ToProto() + } + } + tc.expectedState.Database[string(raftmgr.KeyHardState)] = &anypb.Any{} + tc.expectedState.Database[string(raftmgr.KeyConfState)] = &anypb.Any{} + } RequireDatabase(t, ctx, database, tc.expectedState.Database) expectedRepositories := tc.expectedState.Repositories @@ -1553,6 +1631,17 @@ func runTransactionTest(t *testing.T, ctx context.Context, tc transactionTestCas require.NoError(t, transactionManager.CloseSnapshots()) RequireRepositories(t, ctx, setup.Config, setup.Config.Storages[0].Path, storageScopedFactory.Build, expectedRepositories) + expectedConsumers := tc.expectedState.Consumers + if testhelper.IsRaftEnabled() { + expectedConsumers.HighWaterMark = raftEntryRecorder.Offset(expectedConsumers.HighWaterMark) + // If the expected manager position is equal to 0, it means the test asserts the starting position of the + // consumer or consumer errors. Otherwise, it should be strictly positive as LSN starts at 1. + if expectedConsumers.ManagerPosition != 0 { + expectedConsumers.ManagerPosition = raftEntryRecorder.Offset(expectedConsumers.ManagerPosition) + } + } + RequireConsumer(t, transactionManager.consumer, transactionManager.consumerPos, expectedConsumers) + expectedDirectory := tc.expectedState.Directory if expectedDirectory == nil { // Set the base state as the default so we don't have to repeat it in every test case but it @@ -1563,8 +1652,7 @@ func runTransactionTest(t *testing.T, ctx context.Context, tc transactionTestCas } } - RequireConsumer(t, transactionManager.consumer, transactionManager.consumerPos, tc.expectedState.Consumers) - + expectedDirectory = modifyDirectoryStateForRaft(t, expectedDirectory, transactionManager) testhelper.RequireDirectoryState(t, stateDir, "", expectedDirectory) expectedStagingDirState := testhelper.DirectoryState{ @@ -1582,6 +1670,48 @@ func runTransactionTest(t *testing.T, ctx context.Context, tc transactionTestCas testhelper.RequireDirectoryState(t, transactionManager.stagingDirectory, "", expectedStagingDirState) } +func modifyDirectoryStateForRaft(t *testing.T, expectedDirectory testhelper.DirectoryState, transactionManager *TransactionManager) testhelper.DirectoryState { + if !testhelper.IsRaftEnabled() { + return expectedDirectory + } + + raftEntryRecorder := transactionManager.raftManager.EntryRecorder + newExpectedDirectory := testhelper.DirectoryState{} + lsnRegex := regexp.MustCompile(`/wal/(\d+)\/?.*`) + for path, state := range expectedDirectory { + matches := lsnRegex.FindStringSubmatch(path) + if len(matches) == 0 { + // If no LSN found, retain the original entry + newExpectedDirectory[path] = state + continue + } + // Extract and offset the LSN + lsnStr := matches[1] + lsn, err := strconv.ParseUint(lsnStr, 10, 64) + require.NoError(t, err) + + offsetLSN := raftEntryRecorder.Offset(storage.LSN(lsn)) + offsetPath := strings.Replace(path, fmt.Sprintf("/%s", lsnStr), fmt.Sprintf("/%s", offsetLSN.String()), 1) + newExpectedDirectory[offsetPath] = state + + if entry, isEntry := state.Content.(*gitalypb.LogEntry); isEntry { + entry.Metadata = raftEntryRecorder.Metadata(offsetLSN) + } + } + + // Insert Raft-specific log entries into the new directory structure + raftEntries := raftEntryRecorder.FromRaft() + for lsn, raftEntry := range raftEntries { + if lsn >= transactionManager.oldestLSN { + newExpectedDirectory[fmt.Sprintf("/wal/%s", lsn)] = testhelper.DirectoryEntry{Mode: mode.Directory} + newExpectedDirectory[fmt.Sprintf("/wal/%s/MANIFEST", lsn)] = manifestDirectoryEntry(raftEntry) + } + } + + // Update expected directory + return newExpectedDirectory +} + func checkManagerError(t *testing.T, ctx context.Context, managerErrChannel chan error, mgr *TransactionManager) (bool, error) { t.Helper() diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager.go index 0f83e047636..6a9e6a2c131 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager.go @@ -34,6 +34,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/gitstorage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/mode" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/snapshot" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/wal" "gitlab.com/gitlab-org/gitaly/v16/internal/log" @@ -342,6 +343,12 @@ func (mgr *TransactionManager) Begin(ctx context.Context, opts storage.BeginOpti return nil, errWritableAllRepository } + if mgr.raftManager != nil && !opts.WithoutRaftReady { + if err := mgr.raftManager.WaitReady(); err != nil { + return nil, fmt.Errorf("waiting for Raft to be ready: %w", err) + } + } + span, _ := tracing.StartSpanIfHasParent(ctx, "transaction.Begin", nil) span.SetTag("write", opts.Write) span.SetTag("relativePath", relativePath) @@ -387,20 +394,8 @@ func (mgr *TransactionManager) Begin(ctx context.Context, opts storage.BeginOpti mgr.mutex.Lock() removedAnyEntry = mgr.cleanCommittedEntry(entry) mgr.mutex.Unlock() - - // Signal the manager this transaction finishes. The purpose of this signaling is to wake it up - // and clean up stale entries in the database. The manager scans and removes leading empty - // entries. We signal only if the transaction modifies the in-memory committed entry. - // This signal queue is buffered. If the queue is full, the manager hasn't woken up. The - // next scan will cover the work of the prior one. So, no need to let the transaction wait. - // ┌─ 1st signal ┌─ The manager scans til here - // □ □ □ □ □ □ □ □ □ □ ■ ■ ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ■ ■ ⧅ ⧅ ⧅ ⧅ ■ - // └─ 2nd signal if removedAnyEntry { - select { - case mgr.completedQueue <- struct{}{}: - default: - } + mgr.NotifyNewCommittedEntry() } } }() @@ -1057,6 +1052,24 @@ type TransactionManager struct { // metrics stores reporters which facilitate metric recording of transactional operations. metrics ManagerMetrics + + // raftManager controls cross-network replication using the Raft consensus algorithm. Raft manager follows the + // life cycle of Transaction Manager. Major interactions between Raft Manager and Transaction Manager are + // summarized in the below graph: + // + // Without Raft + // TransactionManager New Transaction ┌────┬───────┐ + // │ ▼ │ ▼ ▼ + // └─►Initialize───►txn.Commit()─►Verify─┤ Append Commit─►Apply ──...──► Stop + // │ │ ▲ ▲ │ + // ▼ ▼ │ │ ▼ + // ┌── Run()──────────...───────────Propose──┴───────┘ ───────...─────── Stop + // │ ▲ │ + // Raft Manager Conf Change │ │ + // ┌──┴─▼─────┐ + // │Raft Group├◄───► Network + // └──────────┘ + raftManager *raftmgr.Manager } type testHooks struct { @@ -1082,6 +1095,7 @@ func NewTransactionManager( repositoryFactory localrepo.StorageScopedFactory, metrics ManagerMetrics, consumer LogConsumer, + raftManager *raftmgr.Manager, ) *TransactionManager { ctx, cancel := context.WithCancel(context.Background()) @@ -1112,6 +1126,7 @@ func NewTransactionManager( consumer: consumer, consumerPos: consumerPos, acknowledgedQueue: make(chan struct{}, 1), + raftManager: raftManager, testHooks: testHooks{ beforeInitialization: func() {}, @@ -2188,9 +2203,19 @@ func (mgr *TransactionManager) run(ctx context.Context) (returnedErr error) { return fmt.Errorf("initialize: %w", err) } + if mgr.raftManager != nil { + if err := mgr.raftManager.Run(ctx, mgr); err != nil { + return fmt.Errorf("starting raft manager: %w", err) + } + } + for { // We prioritize applying committed log entries to the partition first. - if mgr.appliedLSN < mgr.committedLSN { + mgr.mutex.Lock() + shouldApply := mgr.appliedLSN < mgr.committedLSN + mgr.mutex.Unlock() + + if shouldApply { lsn := mgr.appliedLSN + 1 if err := mgr.applyLogEntry(ctx, lsn); err != nil { @@ -2212,7 +2237,9 @@ func (mgr *TransactionManager) run(ctx context.Context) (returnedErr error) { if err := mgr.deleteLogEntry(ctx, mgr.oldestLSN); err != nil { return fmt.Errorf("deleting log entry: %w", err) } + mgr.mutex.Lock() mgr.oldestLSN++ + mgr.mutex.Unlock() continue } @@ -2226,6 +2253,12 @@ func (mgr *TransactionManager) run(ctx context.Context) (returnedErr error) { // logging it. func (mgr *TransactionManager) processTransaction(ctx context.Context) (returnedErr error) { var transaction *Transaction + + var raftDone <-chan struct{} + if mgr.raftManager != nil { + raftDone = mgr.raftManager.Done() + } + select { case transaction = <-mgr.admissionQueue: defer trace.StartRegion(ctx, "processTransaction").End() @@ -2244,6 +2277,7 @@ func (mgr *TransactionManager) processTransaction(ctx context.Context) (returned case <-mgr.acknowledgedQueue: return nil case <-ctx.Done(): + case <-raftDone: } // Return if the manager was stopped. The select is indeterministic so this guarantees @@ -2252,6 +2286,13 @@ func (mgr *TransactionManager) processTransaction(ctx context.Context) (returned return err } + // If Raft stops, we should not continue processing more entries. + if mgr.raftManager != nil { + if err := mgr.raftManager.Err(); err != nil { + return fmt.Errorf("raft error: %w", err) + } + } + span, ctx := tracing.StartSpanIfHasParent(ctx, "transaction.processTransaction", nil) defer span.Finish() @@ -2447,7 +2488,12 @@ func (mgr *TransactionManager) verifyObjectsExist(ctx context.Context, repositor } // Close stops the transaction processing causing Run to return. -func (mgr *TransactionManager) Close() { mgr.close() } +func (mgr *TransactionManager) Close() { + if mgr.raftManager != nil { + mgr.raftManager.Stop() + } + mgr.close() +} // CloseSnapshots closes any remaining snapshots in the cache. Caller of Run() should // call it after there are no more active transactions and no new transactions will be @@ -2723,6 +2769,11 @@ func packFilePath(walFiles string) string { return filepath.Join(walFiles, "transaction.pack") } +// LSNDirPath returns the path to WAL dir of a particular LSN. +func (mgr *TransactionManager) LSNDirPath(lsn storage.LSN) string { + return walFilesPathForLSN(mgr.stateDirectory, lsn) +} + // verifyAlternateUpdate verifies the staged alternate update. func (mgr *TransactionManager) verifyAlternateUpdate(ctx context.Context, transaction *Transaction) (string, error) { defer trace.StartRegion(ctx, "verifyAlternateUpdate").End() @@ -3085,7 +3136,7 @@ func (mgr *TransactionManager) stagePackedRefs(ctx context.Context, tx *Transact } if containsReferenceDeletions { - if err := mgr.walkCommittedEntries(tx, func(entry *gitalypb.LogEntry, dependencies map[git.ObjectID]struct{}) error { + if err := mgr.walkCommittedEntries(tx, func(entry *gitalypb.LogEntry, _ map[git.ObjectID]struct{}) error { if entry.GetHousekeeping().GetPackRefs() != nil { return errConcurrentReferencePacking } @@ -3209,7 +3260,7 @@ func (mgr *TransactionManager) verifyHousekeeping(ctx context.Context, transacti defer mgr.mutex.Unlock() // Check for any concurrent housekeeping between this transaction's snapshot LSN and the latest committed LSN. - if err := mgr.walkCommittedEntries(transaction, func(entry *gitalypb.LogEntry, objectDependencies map[git.ObjectID]struct{}) error { + if err := mgr.walkCommittedEntries(transaction, func(entry *gitalypb.LogEntry, _ map[git.ObjectID]struct{}) error { if entry.GetHousekeeping() != nil { return errHousekeepingConflictConcurrent } @@ -3341,7 +3392,7 @@ func (mgr *TransactionManager) verifyPackRefsFiles(ctx context.Context, transact packRefs := transaction.runHousekeeping.packRefs // Check for any concurrent ref deletion between this transaction's snapshot LSN to the end. - if err := mgr.walkCommittedEntries(transaction, func(entry *gitalypb.LogEntry, objectDependencies map[git.ObjectID]struct{}) error { + if err := mgr.walkCommittedEntries(transaction, func(entry *gitalypb.LogEntry, _ map[git.ObjectID]struct{}) error { for _, refTransaction := range entry.GetReferenceTransactions() { for _, change := range refTransaction.GetChanges() { // We handle HEAD updates through the git-update-ref, but since @@ -3579,20 +3630,124 @@ func (mgr *TransactionManager) applyReferenceTransaction(ctx context.Context, ch // manifest file into the log entry's directory. Second, it sends the log entry to other cluster members if needed. // Afterwards it moves the log entry's directory from the staging area to its final place in the write-ahead log. func (mgr *TransactionManager) proposeLogEntry(ctx context.Context, objectDependencies map[git.ObjectID]struct{}, logEntry *gitalypb.LogEntry, logEntryPath string) error { - nextLSN := mgr.appendedLSN + 1 - if err := mgr.appendLogEntry(ctx, nextLSN, logEntry, logEntryPath); err != nil { - return fmt.Errorf("append log entry: %w", err) + if mgr.raftManager != nil { + if err := mgr.raftManager.Propose(ctx, logEntry, logEntryPath); err != nil { + return err + } + } else { + nextLSN := mgr.committedLSN + 1 + if err := mgr.AppendLogEntry(ctx, nextLSN, logEntry, logEntryPath); err != nil { + return fmt.Errorf("append log entry: %w", err) + } + if err := mgr.CommitLogEntry(ctx, nextLSN); err != nil { + return fmt.Errorf("commit log entry: %w", err) + } } - if err := mgr.commitLogEntry(ctx, nextLSN, objectDependencies); err != nil { - return fmt.Errorf("commit log entry: %w", err) + + mgr.mutex.Lock() + // Tracking object dependencies is only relevant to repacking task that runs in parallel with other operations. + // When the task finishes, the Transaction Manager needs to verify the existence of all dependent objects of + // entries since the time the repacking task started. The current node must be primary when the task started. + // This node need to backfill object dependencies back to the committed entries list in this code path. In + // contrast, replicas don't need to do so. If this node wins an election, the next log entry uses the snapshot + // from at the latest committed entry; hence prior object dependencies are not relevant. + if elm := mgr.committedEntries.Back(); elm != nil { + entry := elm.Value.(*committedEntry) + entry.objectDependencies = objectDependencies } + mgr.mutex.Unlock() + return nil } -// appendLogEntry appends the transaction to the write-ahead log. -func (mgr *TransactionManager) appendLogEntry(ctx context.Context, nextLSN storage.LSN, logEntry *gitalypb.LogEntry, logEntryPath string) error { +// InsertLogEntry inserts a KV log entry into the WAL at the specified LSN. The caller can manipulate the data +// by passing a modification function. It's responsibility of the caller NOT TO append/insert any log entries +// while this function is running. All adjacent log entries at and after inserting position are wiped. +func (mgr *TransactionManager) InsertLogEntry( + ctx context.Context, + lsn storage.LSN, + txnFunc func(keyvalue.ReadWriter) error, + metadata *gitalypb.LogEntry_Metadata, +) (_ *gitalypb.LogEntry, returnedErr error) { + txn, err := mgr.Begin(ctx, storage.BeginOptions{ + Write: true, + RelativePaths: []string{}, + WithoutRaftReady: true, + }) + if err != nil { + return nil, fmt.Errorf("initializing transaction: %w", err) + } + + if txnFunc != nil { + if err := txnFunc(txn.KV()); err != nil { + defer func() { + if rollbackErr := txn.Rollback(ctx); rollbackErr != nil { + returnedErr = errors.Join(returnedErr, fmt.Errorf("rollback transaction: %w", rollbackErr)) + } + }() + return nil, fmt.Errorf("mutate KV transaction: %w", err) + } + } + + transaction := txn.(*Transaction) + defer func() { + if err := transaction.finish(); err != nil && returnedErr == nil { + returnedErr = fmt.Errorf("finish transaction: %w", err) + } + }() + defer trace.StartRegion(ctx, "commit").End() + defer prometheus.NewTimer(transaction.metrics.commitDuration(transaction.write)).ObserveDuration() + + if err = transaction.updateState(transactionStateCommit); err != nil { + return nil, err + } + transaction.admitted = true + + if err = mgr.prepareCommit(ctx, transaction); err != nil { + return nil, fmt.Errorf("preparing commit: %w", err) + } + + logEntry, logEntryPath, err := mgr.packageLogEntry(ctx, transaction) + if err != nil { + return nil, fmt.Errorf("packaging log entry: %w", err) + } + + if err := mgr.AppendLogEntry(ctx, lsn, logEntry, logEntryPath); err != nil { + return nil, fmt.Errorf("appending log entry: %w", err) + } + return logEntry, nil +} + +// AppendLogEntry appends the transaction to the write-ahead log. +func (mgr *TransactionManager) AppendLogEntry(ctx context.Context, lsn storage.LSN, logEntry *gitalypb.LogEntry, logEntryPath string) (err error) { defer trace.StartRegion(ctx, "appendLogEntry").End() + select { + case <-mgr.ctx.Done(): + return mgr.ctx.Err() + default: + } + + // ┌─ committedLSN + // ■ ■ ■ ■ ■ ■ ■ ■ ■ ■ □ □ ⧅ ⧅ ⧅ ⧅ ⧅ + // └─ appliedLSN └─ lsn └─ appendedLSN + // + // When a new log entry is appended, it needs to be confirmed by peers. If the number of confirmations + // (including this node) reach quorum, the log entry is moved to the committed phase. In contrast, that + // log entry and following ones are discarded in favor of more up-to-date entries. + var removeRange storage.LSN + mgr.mutex.Lock() + if lsn <= mgr.committedLSN { + return fmt.Errorf("out-of-order log entry appending, position %s taken", lsn.String()) + } + if lsn <= mgr.appendedLSN { + removeRange = mgr.appendedLSN + } + mgr.mutex.Unlock() + if err := mgr.removeOsboleteLogEntries(lsn, removeRange); err != nil { + return fmt.Errorf("removing obsolete log entries: %w", err) + } + manifestBytes, err := proto.Marshal(logEntry) if err != nil { return fmt.Errorf("marshal manifest: %w", err) @@ -3619,10 +3774,10 @@ func (mgr *TransactionManager) appendLogEntry(ctx context.Context, nextLSN stora return fmt.Errorf("synchronizing WAL directory: %w", err) } - mgr.testHooks.beforeAppendLogEntry(nextLSN) + mgr.testHooks.beforeAppendLogEntry(lsn) // Move the log entry from the staging directory into its place in the log. - destinationPath := walFilesPathForLSN(mgr.stateDirectory, nextLSN) + destinationPath := walFilesPathForLSN(mgr.stateDirectory, lsn) if err := os.Rename(logEntryPath, destinationPath); err != nil { return fmt.Errorf("move wal files: %w", err) } @@ -3646,44 +3801,119 @@ func (mgr *TransactionManager) appendLogEntry(ctx context.Context, nextLSN stora // After this latch block, the transaction is committed and all subsequent transactions // are guaranteed to read it. mgr.mutex.Lock() - mgr.appendedLSN = nextLSN + mgr.appendedLSN = lsn mgr.appendedEntries[mgr.appendedLSN] = logEntry mgr.mutex.Unlock() return nil } -// commitLogEntry commits the transaction to the write-ahead log. -func (mgr *TransactionManager) commitLogEntry(ctx context.Context, nextLSN storage.LSN, objectDependencies map[git.ObjectID]struct{}) error { +// ReadLogEntry reads and returns the log entry stored at the specified LSN within the WAL. It first looks into the +// appended entries before attempting to load it from disk. +func (mgr *TransactionManager) ReadLogEntry(lsn storage.LSN) (*gitalypb.LogEntry, error) { + mgr.mutex.Lock() + defer mgr.mutex.Unlock() + + if lsn < mgr.oldestLSN || lsn > mgr.appendedLSN { + return nil, fmt.Errorf("log entry not found") + } else if lsn > mgr.committedLSN { + // Quick-access from appended entries map. + return mgr.appendedEntries[lsn], nil + } + + return mgr.readLogEntry(lsn) +} + +// FirstLSN retrieves the first LSN that is still accessible in the WAL. +func (mgr *TransactionManager) FirstLSN() storage.LSN { + mgr.mutex.Lock() + defer mgr.mutex.Unlock() + + return mgr.oldestLSN +} + +// LastLSN returns the last LSN that has been appended to the WAL, indicating the latest position. +func (mgr *TransactionManager) LastLSN() storage.LSN { + mgr.mutex.Lock() + defer mgr.mutex.Unlock() + + return mgr.appendedLSN +} + +func (mgr *TransactionManager) removeOsboleteLogEntries(from storage.LSN, to storage.LSN) error { + // Remove log entries reversely to prevent holes in between. + for lsn := to; lsn >= from; lsn-- { + path := walFilesPathForLSN(mgr.stateDirectory, lsn) + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + continue + } + return err + } + if err := os.RemoveAll(path); err != nil { + return err + } + } + return nil +} + +// CommitLogEntry commits the transaction to the write-ahead log. +func (mgr *TransactionManager) CommitLogEntry(ctx context.Context, lsn storage.LSN) (err error) { defer trace.StartRegion(ctx, "commitLogEntry").End() - mgr.testHooks.beforeCommitLogEntry(nextLSN) + mgr.testHooks.beforeCommitLogEntry(lsn) + + select { + case <-mgr.ctx.Done(): + return mgr.ctx.Err() + default: + } // Persist committed LSN before updating internal states. - if err := mgr.storeCommittedLSN(nextLSN); err != nil { + if err := mgr.storeCommittedLSN(lsn); err != nil { return fmt.Errorf("persisting committed entry: %w", err) } + if lsn <= mgr.committedLSN { + return fmt.Errorf("out-of-order log entry committing, position %s taken", lsn.String()) + } + mgr.mutex.Lock() - mgr.committedLSN = nextLSN - mgr.snapshotLocks[nextLSN] = &snapshotLock{applied: make(chan struct{})} - if _, exist := mgr.appendedEntries[nextLSN]; !exist { + mgr.committedLSN = lsn + mgr.snapshotLocks[lsn] = &snapshotLock{applied: make(chan struct{})} + if _, exist := mgr.appendedEntries[lsn]; !exist { mgr.mutex.Unlock() - return fmt.Errorf("log entry %s not found in the appended list", nextLSN) + return fmt.Errorf("log entry %s not found in the appended list", lsn) } - delete(mgr.appendedEntries, nextLSN) - mgr.committedEntries.PushBack(&committedEntry{ - lsn: nextLSN, - objectDependencies: objectDependencies, - }) + delete(mgr.appendedEntries, lsn) + mgr.committedEntries.PushBack(&committedEntry{lsn: lsn}) mgr.mutex.Unlock() if mgr.consumer != nil { - mgr.consumer.NotifyNewTransactions(mgr.storageName, mgr.partitionID, mgr.lowWaterMark(), nextLSN) + mgr.consumer.NotifyNewTransactions(mgr.storageName, mgr.partitionID, mgr.lowWaterMark(), lsn) } return nil } +// NotifyNewCommittedEntry notifies the Transaction Manager of a new committed log entry. Most of the time, the +// processing loop of the transaction manager receives completion signal from admission queue. After processing, +// committing, then applying a log entry, the processing goroutine is put to idle state until there a new transaction +// commits. This flow works fine if Raft is not enabled. In contrast, when this node acts as a replica, Raft needs to +// notify the processing goroutine manually. +func (mgr *TransactionManager) NotifyNewCommittedEntry() { + // The purpose of this signaling is to wake it up and clean up stale entries in the database. The manager scans and + // removes leading empty entries. We signal only if the transaction modifies the in-memory committed entry. This + // signal queue is buffered. If the queue is full, the manager hasn't woken up. The next scan will cover the work + // of the prior one. So, no need to let the transaction wait. + // ┌─ 1st signal ┌─ The manager scans til here + // □ □ □ □ □ □ □ □ □ □ ■ ■ ⧅ ⧅ ⧅ ⧅ ⧅ ⧅ ■ ■ ⧅ ⧅ ⧅ ⧅ ■ + // └─ 2nd signal + select { + case mgr.completedQueue <- struct{}{}: + default: + } +} + // applyLogEntry reads a log entry at the given LSN and applies it to the repository. func (mgr *TransactionManager) applyLogEntry(ctx context.Context, lsn storage.LSN) error { defer trace.StartRegion(ctx, "applyLogEntry").End() diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go index d2467b3dab2..a791926902b 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_hook_test.go @@ -8,6 +8,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/mode" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" ) @@ -21,6 +22,8 @@ type hookContext struct { manager *TransactionManager // lsn stores the LSN context when the hook is triggered. lsn storage.LSN + // raftManager allows the access to the Raft manager inside TransactionManager. + raftManager *raftmgr.Manager } // installHooks takes the hooks in the test setup and configures them in the TransactionManager. @@ -39,7 +42,8 @@ func installHooks(mgr *TransactionManager, inflightTransactions *sync.WaitGroup, runHook := source *destination = func() { runHook(hookContext{ - manager: mgr, + manager: mgr, + raftManager: mgr.raftManager, }) } } @@ -55,8 +59,9 @@ func installHooks(mgr *TransactionManager, inflightTransactions *sync.WaitGroup, runHook := source *destination = func(lsn storage.LSN) { runHook(hookContext{ - manager: mgr, - lsn: lsn, + manager: mgr, + lsn: lsn, + raftManager: mgr.raftManager, }) } } diff --git a/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go b/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go index 725447712ff..de03e9128aa 100644 --- a/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go +++ b/internal/gitaly/storage/storagemgr/partition/transaction_manager_test.go @@ -28,11 +28,13 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/mode" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/snapshot" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" ) // errSimulatedCrash is used in the tests to simulate a crash at a certain point during @@ -41,8 +43,12 @@ var errSimulatedCrash = errors.New("simulated crash") // simulateCrashHook returns a hook function that panics with errSimulatedCrash. var simulateCrashHook = func() func(hookContext) { - return func(hookContext) { - panic(errSimulatedCrash) + return func(c hookContext) { + if !testhelper.IsRaftEnabled() || + c.raftManager == nil || + !c.raftManager.EntryRecorder.IsFromRaft(c.lsn) { + panic(errSimulatedCrash) + } } } @@ -441,55 +447,70 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio ExpectedError: context.Canceled, }, }, - expectedState: StateAssertion{ - Database: DatabaseState{ - string(keyAppliedLSN): storage.LSN(1).ToProto(), + expectedState: testhelper.WithOrWithoutRaft( + StateAssertion{ + Database: DatabaseState{ + string(keyAppliedLSN): storage.LSN(1).ToProto(), + string(keyCommittedLSN): storage.LSN(1).ToProto(), + }, + NotOffsetDatabaseInRaft: true, }, - Repositories: RepositoryStates{ - setup.RelativePath: { - DefaultBranch: "refs/heads/main", - References: gittest.FilesOrReftables( - &ReferencesState{ - FilesBackend: &FilesBackendState{ - LooseReferences: map[git.ReferenceName]git.ObjectID{ - "refs/heads/main": setup.Commits.First.OID, + StateAssertion{ + Database: DatabaseState{ + string(keyAppliedLSN): storage.LSN(1).ToProto(), + string(keyCommittedLSN): storage.LSN(1).ToProto(), + }, + Repositories: RepositoryStates{ + setup.RelativePath: { + DefaultBranch: "refs/heads/main", + References: gittest.FilesOrReftables( + &ReferencesState{ + FilesBackend: &FilesBackendState{ + LooseReferences: map[git.ReferenceName]git.ObjectID{ + "refs/heads/main": setup.Commits.First.OID, + }, }, - }, - }, &ReferencesState{ - ReftableBackend: &ReftableBackendState{ - Tables: []ReftableTable{ - { - MinIndex: 1, - MaxIndex: 1, - References: []git.Reference{ - { - Name: "HEAD", - Target: "refs/heads/main", - IsSymbolic: true, + }, &ReferencesState{ + ReftableBackend: &ReftableBackendState{ + Tables: []ReftableTable{ + { + MinIndex: 1, + MaxIndex: 1, + References: []git.Reference{ + { + Name: "HEAD", + Target: "refs/heads/main", + IsSymbolic: true, + }, }, }, - }, - { - MinIndex: 2, - MaxIndex: 2, - References: []git.Reference{ - { - Name: "refs/heads/main", - Target: setup.Commits.First.OID.String(), + { + MinIndex: 2, + MaxIndex: 2, + References: []git.Reference{ + { + Name: "refs/heads/main", + Target: setup.Commits.First.OID.String(), + }, }, }, }, }, }, - }, - ), + ), + }, }, }, - }, + ), } }(), { - desc: "commit returns if transaction processing stops before transaction acceptance", + desc: "commit returns if context is canceled before admission", + skip: func(t *testing.T) { + testhelper.SkipWithRaft(t, `The hook is installed before appending log entry, before + recorder is activated. Hence, it's not feasible to differentiate between normal + entries and Raft internal entries`) + }, steps: steps{ StartManager{ Hooks: testTransactionHooks{ @@ -508,6 +529,10 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio ExpectedError: storage.ErrTransactionProcessingStopped, }, }, + expectedState: StateAssertion{ + Database: DatabaseState{}, + NotOffsetDatabaseInRaft: true, + }, }, { desc: "commit returns if transaction processing stops after transaction acceptance", @@ -532,11 +557,18 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio }, }, expectedState: StateAssertion{ - Database: DatabaseState{ + Database: testhelper.WithOrWithoutRaft( // The process crashes before apply but after it's committed. So, only // committedLSN is persisted. - string(keyCommittedLSN): storage.LSN(1).ToProto(), - }, + DatabaseState{ + string(keyAppliedLSN): storage.LSN(2).ToProto(), + string(keyCommittedLSN): storage.LSN(3).ToProto(), + }, + DatabaseState{ + string(keyCommittedLSN): storage.LSN(1).ToProto(), + }, + ), + NotOffsetDatabaseInRaft: true, Directory: gittest.FilesOrReftables(testhelper.DirectoryState{ "/": {Mode: mode.Directory}, "/wal": {Mode: mode.Directory}, @@ -1141,7 +1173,8 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio }, expectedState: StateAssertion{ Database: DatabaseState{ - string(keyAppliedLSN): storage.LSN(1).ToProto(), + string(keyAppliedLSN): storage.LSN(1).ToProto(), + string(keyCommittedLSN): storage.LSN(1).ToProto(), }, Repositories: RepositoryStates{ setup.RelativePath: { @@ -1205,7 +1238,8 @@ func generateCommonTests(t *testing.T, ctx context.Context, setup testTransactio }, expectedState: StateAssertion{ Database: DatabaseState{ - string(keyAppliedLSN): storage.LSN(1).ToProto(), + string(keyAppliedLSN): storage.LSN(1).ToProto(), + string(keyCommittedLSN): storage.LSN(1).ToProto(), }, Repositories: RepositoryStates{ setup.RelativePath: { @@ -1720,13 +1754,18 @@ type expectedCommittedEntry struct { } func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []transactionTestCase { - assertCommittedEntries := func(t *testing.T, manager *TransactionManager, expected []*expectedCommittedEntry, actualList *list.List) { + assertCommittedEntries := func(t *testing.T, manager *TransactionManager, expected []*expectedCommittedEntry, actualList *list.List, offsetLSN bool) { require.Equal(t, len(expected), actualList.Len()) i := 0 for elm := actualList.Front(); elm != nil; elm = elm.Next() { actual := elm.Value.(*committedEntry) - require.Equal(t, expected[i].lsn, actual.lsn) + if testhelper.IsRaftEnabled() && offsetLSN { + recorder := manager.raftManager.EntryRecorder + require.Equal(t, recorder.Offset(expected[i].lsn), actual.lsn) + } else { + require.Equal(t, expected[i].lsn, actual.lsn) + } require.Equal(t, expected[i].snapshotReaders, actual.snapshotReaders) if expected[i].entry != nil { @@ -1752,6 +1791,10 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t } } } + if testhelper.IsRaftEnabled() { + recorder := manager.raftManager.EntryRecorder + expectedEntry.Metadata = recorder.Metadata(recorder.Offset(expected[i].lsn)) + } testhelper.ProtoEqual(t, expectedEntry, actualEntry) } @@ -1767,7 +1810,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t steps: steps{ StartManager{}, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries) + assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries, true) }), }, }, @@ -1785,7 +1828,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t lsn: 0, snapshotReaders: 1, }, - }, tm.committedEntries) + }, tm.committedEntries, true) }), Commit{ TransactionID: 1, @@ -1794,7 +1837,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t }, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries) + assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries, true) }), Begin{ TransactionID: 2, @@ -1807,7 +1850,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t lsn: 1, snapshotReaders: 1, }, - }, tm.committedEntries) + }, tm.committedEntries, true) }), Commit{ TransactionID: 2, @@ -1816,7 +1859,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t }, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries) + assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries, true) }), }, expectedState: StateAssertion{ @@ -1906,7 +1949,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t lsn: 1, snapshotReaders: 2, }, - }, tm.committedEntries) + }, tm.committedEntries, true) }), Commit{ TransactionID: 2, @@ -1924,7 +1967,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t lsn: 2, entry: refChangeLogEntry(setup, "refs/heads/branch-1", setup.Commits.First.OID), }, - }, tm.committedEntries) + }, tm.committedEntries, true) }), Begin{ TransactionID: 4, @@ -1942,7 +1985,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t snapshotReaders: 1, entry: refChangeLogEntry(setup, "refs/heads/branch-1", setup.Commits.First.OID), }, - }, tm.committedEntries) + }, tm.committedEntries, true) }), Commit{ TransactionID: 3, @@ -1961,13 +2004,13 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t lsn: 3, entry: refChangeLogEntry(setup, "refs/heads/branch-2", setup.Commits.First.OID), }, - }, tm.committedEntries) + }, tm.committedEntries, true) }), Rollback{ TransactionID: 4, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries) + assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries, true) }), }, expectedState: StateAssertion{ @@ -2051,7 +2094,15 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t TransactionID: 1, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries) + assertCommittedEntries( + t, tm, + testhelper.WithOrWithoutRaft( + []*expectedCommittedEntry{{lsn: 1}}, + []*expectedCommittedEntry{}, + ), + tm.committedEntries, + false, + ) }), Begin{ TransactionID: 2, @@ -2062,7 +2113,15 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t TransactionID: 2, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - assertCommittedEntries(t, tm, []*expectedCommittedEntry{}, tm.committedEntries) + assertCommittedEntries( + t, tm, + testhelper.WithOrWithoutRaft( + []*expectedCommittedEntry{{lsn: 1}}, + []*expectedCommittedEntry{}, + ), + tm.committedEntries, + false, + ) }), }, expectedState: StateAssertion{ @@ -2121,12 +2180,23 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t ExpectedError: storage.ErrTransactionProcessingStopped, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - RequireDatabase(t, ctx, tm.db, DatabaseState{ - string(keyAppliedLSN): storage.LSN(3).ToProto(), - }) + RequireDatabase(t, ctx, tm.db, + testhelper.WithOrWithoutRaft( + DatabaseState{ + string(keyAppliedLSN): storage.LSN(5).ToProto(), + string(keyCommittedLSN): storage.LSN(5).ToProto(), + string(raftmgr.KeyHardState): &anypb.Any{}, + string(raftmgr.KeyConfState): &anypb.Any{}, + }, + DatabaseState{ + string(keyAppliedLSN): storage.LSN(3).ToProto(), + string(keyCommittedLSN): storage.LSN(3).ToProto(), + }, + ), + ) // Transaction 2 and 3 are left-over. testhelper.RequireDirectoryState(t, tm.stateDirectory, "", - gittest.FilesOrReftables(testhelper.DirectoryState{ + gittest.FilesOrReftables(modifyDirectoryStateForRaft(t, testhelper.DirectoryState{ "/": {Mode: mode.Directory}, "/wal": {Mode: mode.Directory}, "/wal/0000000000002": {Mode: mode.Directory}, @@ -2135,7 +2205,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t "/wal/0000000000003": {Mode: mode.Directory}, "/wal/0000000000003/MANIFEST": manifestDirectoryEntry(refChangeLogEntry(setup, "refs/heads/branch-2", setup.Commits.First.OID)), "/wal/0000000000003/1": {Mode: mode.File, Content: []byte(setup.Commits.First.OID + "\n")}, - }, buildReftableDirectory(map[int][]git.ReferenceUpdates{ + }, tm), buildReftableDirectory(map[int][]git.ReferenceUpdates{ 2: {{"refs/heads/branch-1": git.ReferenceUpdate{NewOID: setup.Commits.First.OID}}}, 3: {{"refs/heads/branch-2": git.ReferenceUpdate{NewOID: setup.Commits.First.OID}}}, }))) @@ -2145,12 +2215,25 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { // When the manager finishes initialization, the left-over log entries are // cleaned up. - RequireDatabase(t, ctx, tm.db, DatabaseState{ - string(keyAppliedLSN): storage.LSN(3).ToProto(), - }) - require.Equal(t, tm.appliedLSN, storage.LSN(3)) - require.Equal(t, tm.committedLSN, storage.LSN(3)) - require.Equal(t, tm.appendedLSN, storage.LSN(3)) + RequireDatabase(t, ctx, tm.db, + testhelper.WithOrWithoutRaft( + DatabaseState{ + string(keyAppliedLSN): storage.LSN(5).ToProto(), + string(keyCommittedLSN): storage.LSN(5).ToProto(), + string(raftmgr.KeyHardState): &anypb.Any{}, + string(raftmgr.KeyConfState): &anypb.Any{}, + }, + DatabaseState{ + string(keyAppliedLSN): storage.LSN(3).ToProto(), + string(keyCommittedLSN): storage.LSN(3).ToProto(), + }, + ), + ) + + finalLSN := testhelper.WithOrWithoutRaft(storage.LSN(5), storage.LSN(3)) + require.Equal(t, tm.appliedLSN, finalLSN) + require.Equal(t, tm.committedLSN, finalLSN) + require.Equal(t, tm.appendedLSN, finalLSN) }), }, expectedState: StateAssertion{ @@ -2225,6 +2308,7 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t desc: "transaction manager cleans up left-over committed entries when appliedLSN < committedLSN", skip: func(t *testing.T) { testhelper.SkipWithReftable(t, "test requires manual log addition") + testhelper.SkipWithRaft(t, "test requires manual log addition without going through Raft") }, steps: steps{ StartManager{}, @@ -2271,6 +2355,9 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t ExpectedError: storage.ErrTransactionProcessingStopped, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { + backupCtx := tm.ctx + tm.ctx = testhelper.Context(t) + // Insert an out-of-band log-entry directly into the database for easier test // setup. It's a bit tricky to simulate committed log entries and un-processed // committed log entries at the same time. @@ -2279,6 +2366,8 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t require.NoError(t, os.WriteFile(filepath.Join(logEntryPath, "1"), []byte(setup.Commits.First.OID+"\n"), mode.File)) require.NoError(t, tm.proposeLogEntry(ctx, map[git.ObjectID]struct{}{setup.Commits.First.OID: {}}, refChangeLogEntry(setup, "refs/heads/branch-3", setup.Commits.First.OID), logEntryPath)) + tm.ctx = backupCtx + RequireDatabase(t, ctx, tm.db, DatabaseState{ string(keyAppliedLSN): storage.LSN(3).ToProto(), // Because this is an out-of-band insertion, the committedLSN is updated @@ -2306,7 +2395,8 @@ func generateCommittedEntriesTests(t *testing.T, setup testTransactionSetup) []t // When the manager finishes initialization, the left-over log entries are // cleaned up. RequireDatabase(t, ctx, tm.db, DatabaseState{ - string(keyAppliedLSN): storage.LSN(4).ToProto(), + string(keyAppliedLSN): storage.LSN(4).ToProto(), + string(keyCommittedLSN): storage.LSN(4).ToProto(), }) require.Equal(t, tm.appliedLSN, storage.LSN(4)) require.Equal(t, tm.committedLSN, storage.LSN(4)) @@ -2350,18 +2440,22 @@ func generateAppendedEntriesTests(t *testing.T, setup testTransactionSetup) []tr } } + offsetIfNeeded := func(tm *TransactionManager, lsn storage.LSN) storage.LSN { + if !testhelper.IsRaftEnabled() || tm.raftManager == nil { + return lsn + } + recorder := tm.raftManager.EntryRecorder + return recorder.Offset(lsn) + } + return []transactionTestCase{ { desc: "manager has just initialized", steps: steps{ - StartManager{Hooks: testTransactionHooks{ - BeforeCommitLogEntry: func(c hookContext) { - assert.Fail(t, "there shouldn't be any committed entry") - }, - }}, + StartManager{}, AssertManager{}, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - require.Equal(t, storage.LSN(0), tm.appendedLSN) + require.Equal(t, offsetIfNeeded(tm, 0), tm.appendedLSN) assertAppendedEntries(t, tm, nil) }), CloseManager{}, @@ -2369,6 +2463,9 @@ func generateAppendedEntriesTests(t *testing.T, setup testTransactionSetup) []tr }, { desc: "appended entries are removed after committed", + skip: func(t *testing.T) { + testhelper.SkipWithRaft(t, "this test is not table if Raft inserts internal entries") + }, steps: steps{ StartManager{Hooks: testTransactionHooks{ BeforeCommitLogEntry: func(c hookContext) { @@ -2401,8 +2498,8 @@ func generateAppendedEntriesTests(t *testing.T, setup testTransactionSetup) []tr }, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - require.Equal(t, storage.LSN(1), tm.appendedLSN) - require.Equal(t, storage.LSN(1), tm.committedLSN) + require.Equal(t, offsetIfNeeded(tm, 1), tm.appendedLSN) + require.Equal(t, offsetIfNeeded(tm, 1), tm.committedLSN) assertAppendedEntries(t, tm, nil) }), Begin{ @@ -2411,8 +2508,8 @@ func generateAppendedEntriesTests(t *testing.T, setup testTransactionSetup) []tr ExpectedSnapshotLSN: 1, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - require.Equal(t, storage.LSN(1), tm.appendedLSN) - require.Equal(t, storage.LSN(1), tm.committedLSN) + require.Equal(t, offsetIfNeeded(tm, 1), tm.appendedLSN) + require.Equal(t, offsetIfNeeded(tm, 1), tm.committedLSN) assertAppendedEntries(t, tm, nil) }), Commit{ @@ -2422,8 +2519,8 @@ func generateAppendedEntriesTests(t *testing.T, setup testTransactionSetup) []tr }, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - require.Equal(t, storage.LSN(2), tm.appendedLSN) - require.Equal(t, storage.LSN(2), tm.committedLSN) + require.Equal(t, offsetIfNeeded(tm, 2), tm.appendedLSN) + require.Equal(t, offsetIfNeeded(tm, 2), tm.committedLSN) assertAppendedEntries(t, tm, nil) }), CloseManager{}, @@ -2492,6 +2589,9 @@ func generateAppendedEntriesTests(t *testing.T, setup testTransactionSetup) []tr }, { desc: "transaction manager crashes after appending", + skip: func(t *testing.T) { + testhelper.SkipWithRaft(t, "this test is not table if Raft inserts internal entries") + }, steps: steps{ StartManager{ Hooks: testTransactionHooks{ @@ -2509,8 +2609,8 @@ func generateAppendedEntriesTests(t *testing.T, setup testTransactionSetup) []tr RelativePaths: []string{setup.RelativePath}, }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { - require.Equal(t, storage.LSN(0), tm.appendedLSN) - require.Equal(t, storage.LSN(0), tm.committedLSN) + require.Equal(t, offsetIfNeeded(tm, 0), tm.appendedLSN) + require.Equal(t, offsetIfNeeded(tm, 0), tm.committedLSN) assertAppendedEntries(t, tm, nil) }), Commit{ @@ -2525,7 +2625,7 @@ func generateAppendedEntriesTests(t *testing.T, setup testTransactionSetup) []tr }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { // Appended entries are persisted. - testhelper.RequireDirectoryState(t, tm.stateDirectory, "", gittest.FilesOrReftables(testhelper.DirectoryState{ + testhelper.RequireDirectoryState(t, tm.stateDirectory, "", gittest.FilesOrReftables(modifyDirectoryStateForRaft(t, testhelper.DirectoryState{ "/": {Mode: mode.Directory}, "/wal": {Mode: mode.Directory}, "/wal/0000000000001": {Mode: mode.Directory}, @@ -2553,7 +2653,7 @@ func generateAppendedEntriesTests(t *testing.T, setup testTransactionSetup) []tr }, }), "/wal/0000000000001/1": {Mode: mode.File, Content: []byte(setup.Commits.First.OID + "\n")}, - }, buildReftableDirectory(map[int][]git.ReferenceUpdates{ + }, tm), buildReftableDirectory(map[int][]git.ReferenceUpdates{ 1: {{"refs/heads/branch-1": git.ReferenceUpdate{NewOID: setup.Commits.First.OID}}}, }))) }), @@ -2562,13 +2662,15 @@ func generateAppendedEntriesTests(t *testing.T, setup testTransactionSetup) []tr // No-op, just to ensure the manager is initialized. AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { // Both of them are set to 0 now. - require.Equal(t, storage.LSN(0), tm.appendedLSN) - require.Equal(t, storage.LSN(0), tm.committedLSN) + require.Equal(t, offsetIfNeeded(tm, 0), tm.appendedLSN) + require.Equal(t, offsetIfNeeded(tm, 0), tm.committedLSN) // Appended entries are removed now. - testhelper.RequireDirectoryState(t, tm.stateDirectory, "", testhelper.DirectoryState{ - "/": {Mode: mode.Directory}, - "/wal": {Mode: mode.Directory}, - }) + testhelper.RequireDirectoryState(t, tm.stateDirectory, "", modifyDirectoryStateForRaft(t, + testhelper.DirectoryState{ + "/": {Mode: mode.Directory}, + "/wal": {Mode: mode.Directory}, + }, + tm)) assertAppendedEntries(t, tm, nil) }), Begin{ @@ -2583,8 +2685,8 @@ func generateAppendedEntriesTests(t *testing.T, setup testTransactionSetup) []tr }, AdhocAssertion(func(t *testing.T, ctx context.Context, tm *TransactionManager) { // Apply new commit only. The prior commit was rejected. - require.Equal(t, storage.LSN(1), tm.appendedLSN) - require.Equal(t, storage.LSN(1), tm.committedLSN) + require.Equal(t, offsetIfNeeded(tm, 1), tm.appendedLSN) + require.Equal(t, offsetIfNeeded(tm, 1), tm.committedLSN) assertAppendedEntries(t, tm, nil) }), }, @@ -2736,7 +2838,7 @@ func BenchmarkTransactionManager(b *testing.B) { // Valid partition IDs are >=1. testPartitionID := storage.PartitionID(i + 1) - manager := NewTransactionManager(testPartitionID, logger, database, storageName, storagePath, stateDir, stagingDir, cmdFactory, repositoryFactory, m, nil) + manager := NewTransactionManager(testPartitionID, logger, database, storageName, storagePath, stateDir, stagingDir, cmdFactory, repositoryFactory, m, nil, nil) managers = append(managers, manager) diff --git a/internal/testhelper/testhelper.go b/internal/testhelper/testhelper.go index 452fe9843e0..57f74b68961 100644 --- a/internal/testhelper/testhelper.go +++ b/internal/testhelper/testhelper.go @@ -57,6 +57,30 @@ func IsWALEnabled() bool { return ok } +// IsRaftEnabled returns whether Raft single cluster is enabled in this testing run. +func IsRaftEnabled() bool { + _, ok := os.LookupEnv("GITALY_TEST_RAFT") + if ok && !IsWALEnabled() { + panic("GITALY_TEST_WAL must be enabled") + } + return ok +} + +// WithOrWithoutRaft returns a value correspondingly to if Raft is enabled or not. +func WithOrWithoutRaft[T any](raftVal, noRaftVal T) T { + if IsRaftEnabled() { + return raftVal + } + return noRaftVal +} + +// SkipWithRaft skips the test if Raft is enabled in this testing run. +func SkipWithRaft(tb testing.TB, reason string) { + if IsRaftEnabled() { + tb.Skip(reason) + } +} + // SkipWithWAL skips the test if write-ahead logging is enabled in this testing run. A reason // should be provided either as a description or a link to an issue to explain why the test is // skipped. -- GitLab From 58aa28457453836544bb2aa158b9350c80a44bf8 Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Thu, 24 Oct 2024 11:59:03 +0700 Subject: [PATCH 18/19] raft: Embed Raft Manager factory to Transaction Manager factory Integrating the Raft consensus algorithm necessitated the creation of a Raft Manager for Gitaly. This commit embeds the Raft Manager factory into the Transaction Manager factory, allowing Gitaly to instantiate Raft Managers conditionally based on configuration settings. Test modifications were also made to include this new parameter. The parameter is set to nil to ensure a seamless integration without impacting existing test cases. Later commits will add extra factory setups, case by case. --- internal/cli/gitaly/serve.go | 10 +++++++ internal/cli/gitaly/subcmd_recovery.go | 1 + internal/cli/gitaly/subcmd_recovery_test.go | 1 + .../housekeeping/manager/testhelper_test.go | 1 + internal/git/objectpool/fetch_test.go | 1 + internal/gitaly/storage/raftmgr/manager.go | 3 ++ .../storage/storagemgr/partition/factory.go | 30 +++++++++++++------ internal/testhelper/testserver/gitaly.go | 1 + 8 files changed, 39 insertions(+), 9 deletions(-) diff --git a/internal/cli/gitaly/serve.go b/internal/cli/gitaly/serve.go index 00e5dfd4e4d..4d3be3b3e38 100644 --- a/internal/cli/gitaly/serve.go +++ b/internal/cli/gitaly/serve.go @@ -37,6 +37,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue/databasemgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/mdfile" nodeimpl "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/node" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/partition" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/snapshot" @@ -404,6 +405,13 @@ func run(appCtx *cli.Context, cfg config.Cfg, logger log.Logger) error { logConsumer = walArchiver } + var raftManagerFactory raftmgr.RaftManagerFactory + if cfg.Raft.Enabled { + raftManagerFactory = func(ptnID storage.PartitionID, storageName string, db keyvalue.Transactioner, logger log.Logger) (*raftmgr.Manager, error) { + return raftmgr.NewManager(ptnID, storageName, cfg.Raft, db, logger) + } + } + nodeMgr, err := nodeimpl.NewManager( cfg.Storages, storagemgr.NewFactory( @@ -414,6 +422,7 @@ func run(appCtx *cli.Context, cfg config.Cfg, logger log.Logger) error { localrepo.NewFactory(logger, locator, gitCmdFactory, catfileCache), partitionMetrics, logConsumer, + raftManagerFactory, ), 2, storageMetrics, @@ -460,6 +469,7 @@ func run(appCtx *cli.Context, cfg config.Cfg, logger log.Logger) error { localrepo.NewFactory(logger, locator, gitCmdFactory, catfileCache), partitionMetrics, nil, + nil, ), // In recovery mode we don't want to keep inactive partitions active. The cache // however can't be disabled so simply set it to one. diff --git a/internal/cli/gitaly/subcmd_recovery.go b/internal/cli/gitaly/subcmd_recovery.go index 9d0337ac5d2..a46c1648718 100644 --- a/internal/cli/gitaly/subcmd_recovery.go +++ b/internal/cli/gitaly/subcmd_recovery.go @@ -126,6 +126,7 @@ func recoveryStatusAction(ctx *cli.Context) (returnErr error) { localrepo.NewFactory(logger, locator, gitCmdFactory, catfileCache), partitionMetrics, nil, + nil, ), 1, storageMetrics, diff --git a/internal/cli/gitaly/subcmd_recovery_test.go b/internal/cli/gitaly/subcmd_recovery_test.go index e0d78a56b00..d70e2896ecc 100644 --- a/internal/cli/gitaly/subcmd_recovery_test.go +++ b/internal/cli/gitaly/subcmd_recovery_test.go @@ -168,6 +168,7 @@ Relative paths: localrepo.NewFactory(logger, locator, gitCmdFactory, catfileCache), partition.NewMetrics(housekeeping.NewMetrics(cfg.Prometheus), snapshot.NewMetrics()), nil, + nil, ), 1, storagemgr.NewMetrics(cfg.Prometheus), diff --git a/internal/git/housekeeping/manager/testhelper_test.go b/internal/git/housekeeping/manager/testhelper_test.go index ffdbf029bd0..4f2a8547ca5 100644 --- a/internal/git/housekeeping/manager/testhelper_test.go +++ b/internal/git/housekeeping/manager/testhelper_test.go @@ -113,6 +113,7 @@ func testWithAndWithoutTransaction(t *testing.T, ctx context.Context, desc strin snapshot.NewMetrics(), ), nil, + nil, ), storagemgr.DefaultMaxInactivePartitions, storagemgr.NewMetrics(cfg.Prometheus), diff --git a/internal/git/objectpool/fetch_test.go b/internal/git/objectpool/fetch_test.go index b502b45925a..89f7a454fb0 100644 --- a/internal/git/objectpool/fetch_test.go +++ b/internal/git/objectpool/fetch_test.go @@ -431,6 +431,7 @@ func testWithAndWithoutTransaction(t *testing.T, ctx context.Context, testFunc f localRepoFactory, partition.NewMetrics(housekeeping.NewMetrics(cfg.Prometheus), snapshot.NewMetrics()), nil, + nil, ), storagemgr.DefaultMaxInactivePartitions, storagemgr.NewMetrics(cfg.Prometheus), diff --git a/internal/gitaly/storage/raftmgr/manager.go b/internal/gitaly/storage/raftmgr/manager.go index d1e0a89e0e9..ec06e2e3e79 100644 --- a/internal/gitaly/storage/raftmgr/manager.go +++ b/internal/gitaly/storage/raftmgr/manager.go @@ -90,6 +90,9 @@ func WithRecordTransport() OptionFunc { } } +// RaftManagerFactory defines a function type that creates a new Manager instance for a particular parittion. +type RaftManagerFactory func(storage.PartitionID, string, keyvalue.Transactioner, log.Logger) (*Manager, error) + // Manager orchestrates the Raft consensus protocol, managing configuration, // state synchronization, and communication between distributed nodes. type Manager struct { diff --git a/internal/gitaly/storage/storagemgr/partition/factory.go b/internal/gitaly/storage/storagemgr/partition/factory.go index 3baad57b7c8..d72f306796e 100644 --- a/internal/gitaly/storage/storagemgr/partition/factory.go +++ b/internal/gitaly/storage/storagemgr/partition/factory.go @@ -8,6 +8,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr" "gitlab.com/gitlab-org/gitaly/v16/internal/log" ) @@ -27,10 +28,11 @@ type LogConsumer interface { // Factory is factory type that can create new partitions. type Factory struct { - cmdFactory gitcmd.CommandFactory - repoFactory localrepo.Factory - metrics Metrics - logConsumer LogConsumer + cmdFactory gitcmd.CommandFactory + repoFactory localrepo.Factory + metrics Metrics + logConsumer LogConsumer + raftManagerFactory raftmgr.RaftManagerFactory } // New returns a new Partition instance. @@ -55,6 +57,14 @@ func (f Factory) New( panic(fmt.Errorf("building a partition for a non-existent storage: %q", storageName)) } + var raftManager *raftmgr.Manager + if f.raftManagerFactory != nil { + raftManager, err = f.raftManagerFactory(partitionID, storageName, db, logger) + if err != nil { + panic(fmt.Errorf("creating raft manager: %w", err)) + } + } + return NewTransactionManager( partitionID, logger, @@ -67,7 +77,7 @@ func (f Factory) New( repoFactory, f.metrics.Scope(storageName), f.logConsumer, - nil, + raftManager, ) } @@ -77,11 +87,13 @@ func NewFactory( repoFactory localrepo.Factory, metrics Metrics, logConsumer LogConsumer, + raftManagerFactory raftmgr.RaftManagerFactory, ) Factory { return Factory{ - cmdFactory: cmdFactory, - repoFactory: repoFactory, - metrics: metrics, - logConsumer: logConsumer, + cmdFactory: cmdFactory, + repoFactory: repoFactory, + metrics: metrics, + logConsumer: logConsumer, + raftManagerFactory: raftManagerFactory, } } diff --git a/internal/testhelper/testserver/gitaly.go b/internal/testhelper/testserver/gitaly.go index 5a582d14111..4a2ba98b1ff 100644 --- a/internal/testhelper/testserver/gitaly.go +++ b/internal/testhelper/testserver/gitaly.go @@ -361,6 +361,7 @@ func (gsd *gitalyServerDeps) createDependencies(tb testing.TB, ctx context.Conte snapshot.NewMetrics(), ), nil, + nil, ), storagemgr.DefaultMaxInactivePartitions, storagemgr.NewMetrics(cfg.Prometheus), -- GitLab From 059e682f35a8256050ddf98fc66aac89546b7408 Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Thu, 31 Oct 2024 15:01:08 +0700 Subject: [PATCH 19/19] raft: Add test-raft job to the test matrices This commit creates new test target. This test target initiates Raft factory and adds it to gRPC server dependency. Along the way, some tests are skipped because there are no reliable ways to assert the results when Raft is enabled. --- .gitlab-ci.yml | 8 ++++---- Makefile | 6 ++++++ internal/backup/partition_backup_test.go | 6 +++++- internal/cli/gitalybackup/partition_test.go | 3 +++ .../gitaly/service/partition/backup_partition_test.go | 3 +++ internal/testhelper/testcfg/gitaly.go | 5 +++++ internal/testhelper/testserver/gitaly.go | 11 ++++++++++- 7 files changed, 36 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dcf6e3375bc..e3f40deeb69 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -256,7 +256,7 @@ test: - GO_VERSION: !reference [.versions, go_supported] TEST_TARGET: test - TEST_TARGET: - [test-with-praefect, race-go, test-wal, test-with-praefect-wal] + [test-with-praefect, race-go, test-wal, test-raft, test-with-praefect-wal] # We also verify that things work as expected with a non-bundled Git # version matching our minimum required Git version. - TEST_TARGET: test @@ -312,7 +312,7 @@ test:nightly: parallel: matrix: - GIT_VERSION: ["master", "next"] - TEST_TARGET: [test, test-with-praefect] + TEST_TARGET: [test, test-with-praefect, test-wal, test-raft] rules: - if: '$CI_PIPELINE_SOURCE == "schedule"' allow_failure: false @@ -331,7 +331,7 @@ test:sha256: parallel: matrix: - TEST_TARGET: - [test, test-with-praefect, test-wal, test-with-praefect-wal] + [test, test-with-praefect, test-wal, test-raft, test-with-praefect-wal] TEST_WITH_SHA256: "YesPlease" test:fips: @@ -354,7 +354,7 @@ test:fips: - test "$(cat /proc/sys/crypto/fips_enabled)" = "1" || (echo "System is not running in FIPS mode" && exit 1) parallel: matrix: - - TEST_TARGET: [test, test-with-praefect] + - TEST_TARGET: [test, test-with-praefect, test-wal, test-raft] FIPS_MODE: "YesPlease" GO_VERSION: !reference [.versions, go_supported] rules: diff --git a/Makefile b/Makefile index 1aacac7a184..d5f13194652 100644 --- a/Makefile +++ b/Makefile @@ -406,6 +406,12 @@ test-with-praefect: test-go test-wal: export GITALY_TEST_WAL = YesPlease test-wal: test-go +.PHONY: test-raft +## Run Go tests with write-ahead logging + Raft enabled. +test-raft: export GITALY_TEST_WAL = YesPlease +test-raft: export GITALY_TEST_RAFT = YesPlease +test-raft: test-go + .PHONY: test-with-praefect-wal ## Run Go tests with write-ahead logging and Praefect enabled. test-with-praefect-wal: export GITALY_TEST_WAL = YesPlease diff --git a/internal/backup/partition_backup_test.go b/internal/backup/partition_backup_test.go index 44a3cba1b5d..5d2e65784e7 100644 --- a/internal/backup/partition_backup_test.go +++ b/internal/backup/partition_backup_test.go @@ -110,8 +110,12 @@ func TestPartitionBackup_CreateSuccess(t *testing.T) { require.NoError(t, err) + testhelper.SkipWithRaft(t, `The test asserts the existence of backup files based on the latest + LSN. When Raft is not enabled, the LSN is not static. The test should fetch the latest + LSN instead https://gitlab.com/gitlab-org/gitaly/-/issues/6459`) + lsn := testhelper.WithOrWithoutRaft(storage.LSN(3), storage.LSN(1)) for _, expectedArchive := range tc.expectedArchives { - tarPath := filepath.Join(backupRoot, cfg.Storages[0].Name, expectedArchive, storage.LSN(1).String()) + ".tar" + tarPath := filepath.Join(backupRoot, cfg.Storages[0].Name, expectedArchive, lsn.String()+".tar") tar, err := os.Open(tarPath) require.NoError(t, err) testhelper.MustClose(t, tar) diff --git a/internal/cli/gitalybackup/partition_test.go b/internal/cli/gitalybackup/partition_test.go index 29e0bc7f24e..c74c953bb5d 100644 --- a/internal/cli/gitalybackup/partition_test.go +++ b/internal/cli/gitalybackup/partition_test.go @@ -109,6 +109,9 @@ func TestPartitionSubcommand_Create(t *testing.T) { } require.NoError(t, err) + testhelper.SkipWithRaft(t, `The test asserts the existence of backup files based on the latest + LSN. When Raft is not enabled, the LSN is not static. The test should fetch the latest + LSN instead https://gitlab.com/gitlab-org/gitaly/-/issues/6459`) lsn := storage.LSN(1) tarPath := filepath.Join(path, cfg.Storages[0].Name, "2", lsn.String()) + ".tar" tar, err := os.Open(tarPath) diff --git a/internal/gitaly/service/partition/backup_partition_test.go b/internal/gitaly/service/partition/backup_partition_test.go index 501680df67f..a75ef5092c2 100644 --- a/internal/gitaly/service/partition/backup_partition_test.go +++ b/internal/gitaly/service/partition/backup_partition_test.go @@ -136,6 +136,9 @@ func TestBackupPartition(t *testing.T) { require.NoError(t, err) testhelper.ProtoEqual(t, &gitalypb.BackupPartitionResponse{}, resp) + testhelper.SkipWithRaft(t, `The test asserts the existence of backup files based on the latest + LSN. When Raft is not enabled, the LSN is not static. The test should fetch the latest + LSN instead https://gitlab.com/gitlab-org/gitaly/-/issues/6459`) lsn := storage.LSN(2) tarPath := filepath.Join(backupRoot, data.storageName, data.partitionID, lsn.String()) + ".tar" tar, err := os.Open(tarPath) diff --git a/internal/testhelper/testcfg/gitaly.go b/internal/testhelper/testcfg/gitaly.go index 726092838af..d11d478518b 100644 --- a/internal/testhelper/testcfg/gitaly.go +++ b/internal/testhelper/testcfg/gitaly.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/google/uuid" "github.com/pelletier/go-toml/v2" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" @@ -172,6 +173,10 @@ func (gc *GitalyCfgBuilder) Build(tb testing.TB) config.Cfg { } cfg.Transactions.Enabled = testhelper.IsWALEnabled() + if testhelper.IsRaftEnabled() { + cfg.Transactions.Enabled = true + cfg.Raft = config.DefaultRaftConfig(uuid.New().String()) + } // We force the use of bundled (embedded) binaries unless we're specifically executing tests // against a custom version of Git. diff --git a/internal/testhelper/testserver/gitaly.go b/internal/testhelper/testserver/gitaly.go index 4a2ba98b1ff..3ee0773f946 100644 --- a/internal/testhelper/testserver/gitaly.go +++ b/internal/testhelper/testserver/gitaly.go @@ -6,6 +6,7 @@ import ( "os" "testing" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" gitalyauth "gitlab.com/gitlab-org/gitaly/v16/auth" @@ -30,6 +31,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue/databasemgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/mode" nodeimpl "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/node" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/partition" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr/snapshot" @@ -348,6 +350,13 @@ func (gsd *gitalyServerDeps) createDependencies(tb testing.TB, ctx context.Conte require.NoError(tb, err) tb.Cleanup(dbMgr.Close) + var raftManagerFactory raftmgr.RaftManagerFactory + if testhelper.IsRaftEnabled() { + cfg.Raft = config.DefaultRaftConfig(uuid.New().String()) + raftManagerFactory = func(ptnID storage.PartitionID, storageName string, db keyvalue.Transactioner, logger log.Logger) (*raftmgr.Manager, error) { + return raftmgr.NewManager(ptnID, storageName, cfg.Raft, db, logger) + } + } nodeMgr, err := nodeimpl.NewManager( cfg.Storages, storagemgr.NewFactory( @@ -361,7 +370,7 @@ func (gsd *gitalyServerDeps) createDependencies(tb testing.TB, ctx context.Conte snapshot.NewMetrics(), ), nil, - nil, + raftManagerFactory, ), storagemgr.DefaultMaxInactivePartitions, storagemgr.NewMetrics(cfg.Prometheus), -- GitLab