diff --git a/internal/gitaly/storage/partition_id.go b/internal/gitaly/storage/partition_id.go index 4c61b0e4a0dd012341409eb9c68c2484cbb52229..3f29dce799137d75dc1a70c12b333dba1693cdfc 100644 --- a/internal/gitaly/storage/partition_id.go +++ b/internal/gitaly/storage/partition_id.go @@ -24,3 +24,10 @@ func (id *PartitionID) UnmarshalBinary(data []byte) { func (id PartitionID) String() string { return strconv.FormatUint(uint64(id), 10) } + +// AtoPartitionID converts a string to a PartitionID +func AtoPartitionID(str string) (PartitionID, error) { + num, err := strconv.ParseInt(str, 10, 64) + + return PartitionID(num), err +} diff --git a/internal/gitaly/storage/storagemgr/partition/partition_restructure_migration.go b/internal/gitaly/storage/storagemgr/partition/partition_restructure_migration.go index ef2950d2cf1c96edc46df8ab84f64eb3fd4c5e08..6335c67c4b3513d0ba6ad17ecfd04469fafcb1e8 100644 --- a/internal/gitaly/storage/storagemgr/partition/partition_restructure_migration.go +++ b/internal/gitaly/storage/storagemgr/partition/partition_restructure_migration.go @@ -2,7 +2,6 @@ package partition import ( "context" - "crypto/sha256" "errors" "fmt" "io/fs" @@ -14,6 +13,7 @@ import ( "github.com/dgraph-io/badger/v4" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/keyvalue" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/raftmgr" "gitlab.com/gitlab-org/gitaly/v16/internal/safe" ) @@ -158,7 +158,10 @@ func (m *RaftPartitionMigrator) partitionRestructureMigration() error { } // It matched, third capture group will be partitionID partitionID := matches[3] - _, newWalDir := pathForMigratedDir(m.storageName, m.partitionsDir, partitionID) + _, newWalDir, err := pathForMigratedDir(m.storageName, m.partitionsDir, partitionID) + if err != nil { + return err + } // Add dir to be synced dirsToSync[newWalDir] = struct{}{} @@ -508,10 +511,14 @@ func cleanupNewPartitionStructure(partitionsDir string) error { return nil } -func pathForMigratedDir(storageName, partitionsBase, partitionID string) (relativePath string, absolutePath string) { - targetPartitionName := storage.GetRaftPartitionName(storageName, partitionID) +func pathForMigratedDir(storageName, partitionsBase, partitionIDstr string) (string, string, error) { + targetPartitionName := storage.GetRaftPartitionName(storageName, partitionIDstr) // Generate hash for new path - hash := fmt.Sprintf("%x", sha256.Sum256([]byte(targetPartitionName))) + partitionID, err := storage.AtoPartitionID(partitionIDstr) + if err != nil { + return "", "", err + } + hash := raftmgr.NewPartitionKey(storageName, partitionID).GetValue() hashedPath := fmt.Sprintf("%s/%s/%s", hash[:2], hash[2:4], targetPartitionName) // Determine the base of the new path @@ -520,7 +527,7 @@ func pathForMigratedDir(storageName, partitionsBase, partitionID string) (relati partitionsBase, hashedPath, "wal", - ) + ), nil } func (m *RaftPartitionMigrator) updateMigrationInDB() error { diff --git a/internal/gitaly/storage/storagemgr/partition/partition_restructure_migration_test.go b/internal/gitaly/storage/storagemgr/partition/partition_restructure_migration_test.go index 8206a935ad4e8d1836f5c293a6cbd346f7db4cbd..c5c34b4ef76ced1b62db6abc528b73d77f509fab 100644 --- a/internal/gitaly/storage/storagemgr/partition/partition_restructure_migration_test.go +++ b/internal/gitaly/storage/storagemgr/partition/partition_restructure_migration_test.go @@ -71,7 +71,8 @@ func TestPartitionRestructureMigration(t *testing.T) { // Run the migration require.NoError(t, migrator.partitionRestructureMigration()) - _, expectedNewWalPath := pathForMigratedDir("testStorage", partitionsDir, tc.partitionID) + _, expectedNewWalPath, err := pathForMigratedDir("testStorage", partitionsDir, tc.partitionID) + require.NoError(t, err) // Verify the directory structure exists if it should if tc.numberOfFiles > 0 { // Loop through each sequence directory based on numberOfFiles @@ -102,25 +103,25 @@ func TestPartitionRestructureMigration(t *testing.T) { "/59/94/12345/wal/0000000000000002": {Mode: mode.Directory}, "/59/94/12345/wal/0000000000000002/MANIFEST": {Mode: mode.File, Content: content}, "/59/94/12345/wal/0000000000000002/RAFT": {Mode: mode.File, Content: content}, - "/a8": {Mode: mode.Directory}, - "/a8/42": {Mode: mode.Directory}, - "/a8/42/testStorage_12345": {Mode: mode.Directory}, - "/a8/42/testStorage_12345/wal": {Mode: mode.Directory}, - "/a8/42/testStorage_12345/wal/0000000000000001": {Mode: mode.Directory}, - "/a8/42/testStorage_12345/wal/0000000000000001/RAFT": {Mode: mode.File, Content: content}, - "/a8/42/testStorage_12345/wal/0000000000000001/MANIFEST": {Mode: mode.File, Content: content}, - "/a8/42/testStorage_12345/wal/0000000000000002": {Mode: mode.Directory}, - "/a8/42/testStorage_12345/wal/0000000000000002/RAFT": {Mode: mode.File, Content: content}, - "/a8/42/testStorage_12345/wal/0000000000000002/MANIFEST": {Mode: mode.File, Content: content}, + "/53": {Mode: mode.Directory}, + "/53/1a": {Mode: mode.Directory}, + "/53/1a/testStorage_12345": {Mode: mode.Directory}, + "/53/1a/testStorage_12345/wal": {Mode: mode.Directory}, + "/53/1a/testStorage_12345/wal/0000000000000001": {Mode: mode.Directory}, + "/53/1a/testStorage_12345/wal/0000000000000001/RAFT": {Mode: mode.File, Content: content}, + "/53/1a/testStorage_12345/wal/0000000000000001/MANIFEST": {Mode: mode.File, Content: content}, + "/53/1a/testStorage_12345/wal/0000000000000002": {Mode: mode.Directory}, + "/53/1a/testStorage_12345/wal/0000000000000002/RAFT": {Mode: mode.File, Content: content}, + "/53/1a/testStorage_12345/wal/0000000000000002/MANIFEST": {Mode: mode.File, Content: content}, }) } } else { testhelper.RequireDirectoryState(t, partitionsDir, "", testhelper.DirectoryState{ "/": {Mode: mode.Directory}, - "/08": {Mode: mode.Directory}, - "/08/10": {Mode: mode.Directory}, - "/08/10/testStorage_99999": {Mode: mode.Directory}, - "/08/10/testStorage_99999/wal": {Mode: mode.Directory}, + "/37": {Mode: mode.Directory}, + "/37/3c": {Mode: mode.Directory}, + "/37/3c/testStorage_99999": {Mode: mode.Directory}, + "/37/3c/testStorage_99999/wal": {Mode: mode.Directory}, "/fd": {Mode: mode.Directory}, "/fd/5f": {Mode: mode.Directory}, "/fd/5f/99999": {Mode: mode.Directory}, @@ -231,12 +232,12 @@ func TestCleanupOldPartitionStructure(t *testing.T) { "/59/94/12345/wal": {Mode: mode.Directory}, "/59/94/12345/wal/0000000000000001": {Mode: mode.Directory}, "/59/94/12345/wal/0000000000000001/RAFT": {Mode: mode.File, Content: content}, - "/a8": {Mode: mode.Directory}, - "/a8/42": {Mode: mode.Directory}, - "/a8/42/testStorage_12345": {Mode: mode.Directory}, - "/a8/42/testStorage_12345/wal": {Mode: mode.Directory}, - "/a8/42/testStorage_12345/wal/0000000000000001": {Mode: mode.Directory}, - "/a8/42/testStorage_12345/wal/0000000000000001/RAFT": {Mode: mode.File, Content: content}, + "/53": {Mode: mode.Directory}, + "/53/1a": {Mode: mode.Directory}, + "/53/1a/testStorage_12345": {Mode: mode.Directory}, + "/53/1a/testStorage_12345/wal": {Mode: mode.Directory}, + "/53/1a/testStorage_12345/wal/0000000000000001": {Mode: mode.Directory}, + "/53/1a/testStorage_12345/wal/0000000000000001/RAFT": {Mode: mode.File, Content: content}, }) // Run the cleanup @@ -247,12 +248,12 @@ func TestCleanupOldPartitionStructure(t *testing.T) { "/": {Mode: mode.Directory}, "/59": {Mode: mode.Directory}, "/59/94": {Mode: mode.Directory}, // parent of partition 12345 still exists - "/a8": {Mode: mode.Directory}, - "/a8/42": {Mode: mode.Directory}, - "/a8/42/testStorage_12345": {Mode: mode.Directory}, - "/a8/42/testStorage_12345/wal": {Mode: mode.Directory}, - "/a8/42/testStorage_12345/wal/0000000000000001": {Mode: mode.Directory}, - "/a8/42/testStorage_12345/wal/0000000000000001/RAFT": {Mode: mode.File, Content: content}, + "/53": {Mode: mode.Directory}, + "/53/1a": {Mode: mode.Directory}, + "/53/1a/testStorage_12345": {Mode: mode.Directory}, + "/53/1a/testStorage_12345/wal": {Mode: mode.Directory}, + "/53/1a/testStorage_12345/wal/0000000000000001": {Mode: mode.Directory}, + "/53/1a/testStorage_12345/wal/0000000000000001/RAFT": {Mode: mode.File, Content: content}, }) } @@ -479,18 +480,18 @@ func TestPartitionMigrator_Forward(t *testing.T) { "/xx/yy": {Mode: mode.Directory}, // parent of partition 123 still remains "/zz": {Mode: mode.Directory}, "/zz/yy": {Mode: mode.Directory}, // parent of partition 456 still remains - "/43": {Mode: mode.Directory}, - "/43/02": {Mode: mode.Directory}, - "/43/02/testStorage_456": {Mode: mode.Directory}, - "/43/02/testStorage_456/wal": {Mode: mode.Directory}, - "/43/02/testStorage_456/wal/0000000000000001": {Mode: mode.Directory}, - "/43/02/testStorage_456/wal/0000000000000001/RAFT": {Mode: mode.File, Content: content}, - "/7e": {Mode: mode.Directory}, - "/7e/8d": {Mode: mode.Directory}, - "/7e/8d/testStorage_123": {Mode: mode.Directory}, - "/7e/8d/testStorage_123/wal": {Mode: mode.Directory}, - "/7e/8d/testStorage_123/wal/0000000000000001": {Mode: mode.Directory}, - "/7e/8d/testStorage_123/wal/0000000000000001/RAFT": {Mode: mode.File, Content: content}, + "/9e": {Mode: mode.Directory}, + "/9e/8a": {Mode: mode.Directory}, + "/9e/8a/testStorage_456": {Mode: mode.Directory}, + "/9e/8a/testStorage_456/wal": {Mode: mode.Directory}, + "/9e/8a/testStorage_456/wal/0000000000000001": {Mode: mode.Directory}, + "/9e/8a/testStorage_456/wal/0000000000000001/RAFT": {Mode: mode.File, Content: content}, + "/16": {Mode: mode.Directory}, + "/16/87": {Mode: mode.Directory}, + "/16/87/testStorage_123": {Mode: mode.Directory}, + "/16/87/testStorage_123/wal": {Mode: mode.Directory}, + "/16/87/testStorage_123/wal/0000000000000001": {Mode: mode.Directory}, + "/16/87/testStorage_123/wal/0000000000000001/RAFT": {Mode: mode.File, Content: content}, }) assertKeyExists(t, migrator.db) @@ -543,11 +544,11 @@ func TestPartitionMigrator_Forward(t *testing.T) { "/xx": {Mode: mode.Directory}, "/xx/yy": {Mode: mode.Directory}, "/xx/yy/123": {Mode: mode.Directory}, // expected to not be cleaned up - "/7e": {Mode: mode.Directory}, - "/7e/8d": {Mode: mode.Directory}, - "/7e/8d/testStorage_123": {Mode: mode.Directory}, - "/7e/8d/testStorage_123/wal": {Mode: mode.Directory}, - "/7e/8d/testStorage_123/wal/0000000000000001": {Mode: mode.Directory}, + "/16": {Mode: mode.Directory}, + "/16/87": {Mode: mode.Directory}, + "/16/87/testStorage_123": {Mode: mode.Directory}, + "/16/87/testStorage_123/wal": {Mode: mode.Directory}, + "/16/87/testStorage_123/wal/0000000000000001": {Mode: mode.Directory}, }) assertKeyAbsent(t, migrator.db) migrated, err := migrator.CheckMigrationStatus() @@ -560,11 +561,11 @@ func TestPartitionMigrator_Forward(t *testing.T) { "/": {Mode: mode.Directory}, "/xx": {Mode: mode.Directory}, "/xx/yy": {Mode: mode.Directory}, - "/7e": {Mode: mode.Directory}, - "/7e/8d": {Mode: mode.Directory}, - "/7e/8d/testStorage_123": {Mode: mode.Directory}, - "/7e/8d/testStorage_123/wal": {Mode: mode.Directory}, - "/7e/8d/testStorage_123/wal/0000000000000001": {Mode: mode.Directory}, + "/16": {Mode: mode.Directory}, + "/16/87": {Mode: mode.Directory}, + "/16/87/testStorage_123": {Mode: mode.Directory}, + "/16/87/testStorage_123/wal": {Mode: mode.Directory}, + "/16/87/testStorage_123/wal/0000000000000001": {Mode: mode.Directory}, }) assertKeyExists(t, migrator.db) migrated, err = migrator.CheckMigrationStatus()