From 51ee1ba6c9abb6e1c75624a211e5aeff84703ff9 Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Fri, 10 Oct 2025 17:21:17 +0700 Subject: [PATCH 1/4] cli: Add JSON output format to cluster info command The cluster info command currently outputs data only in a human-readable TUI format with colored tables. This format is not suitable for programmatic consumption or automation workflows that need to parse cluster statistics and partition details. This commit adds a --format flag accepting "text" (default) or "json" values. The JSON output uses protojson marshaling to preserve protobuf field names in lowerCamelCase format. The implementation separates data fetching from presentation, allowing both text and JSON formats to share the same RPC calls without duplication. The JSON structure mirrors the protobuf definitions with cluster_info and optional partitions array, ensuring consistency with the gRPC API. --- internal/cli/gitaly/subcmd_cluster_info.go | 123 ++++- .../cli/gitaly/subcmd_cluster_info_test.go | 426 ++++++++++++++++-- 2 files changed, 489 insertions(+), 60 deletions(-) diff --git a/internal/cli/gitaly/subcmd_cluster_info.go b/internal/cli/gitaly/subcmd_cluster_info.go index 2abfe89b683..b53f0ab20e0 100644 --- a/internal/cli/gitaly/subcmd_cluster_info.go +++ b/internal/cli/gitaly/subcmd_cluster_info.go @@ -2,6 +2,7 @@ package gitaly import ( "context" + "encoding/json" "fmt" "io" "os" @@ -13,6 +14,7 @@ import ( "github.com/mattn/go-isatty" "github.com/urfave/cli/v3" "gitlab.com/gitlab-org/gitaly/v18/proto/go/gitalypb" + "google.golang.org/protobuf/encoding/protojson" ) const ( @@ -20,13 +22,14 @@ const ( flagClusterInfoStorage = "storage" flagClusterInfoListPartitions = "list-partitions" flagClusterInfoNoColor = "no-color" + flagClusterInfoFormat = "format" ) func newClusterInfoCommand() *cli.Command { return &cli.Command{ Name: "info", Usage: "display cluster statistics and overview", - UsageText: `gitaly cluster info --config [--list-partitions] [--storage ] + UsageText: `gitaly cluster info --config [--list-partitions] [--storage ] [--format ] Examples: # Show cluster statistics only (default) @@ -36,7 +39,13 @@ Examples: gitaly cluster info --config config.toml --list-partitions # Filter by storage (shows partition overview for that storage) - gitaly cluster info --config config.toml --storage storage-1 --list-partitions`, + gitaly cluster info --config config.toml --storage storage-1 --list-partitions + + # Output as JSON for programmatic consumption + gitaly cluster info --config config.toml --format json + + # Output JSON with partition details + gitaly cluster info --config config.toml --format json --list-partitions`, Description: `Display cluster-wide information including: - Cluster statistics (total partitions, replicas, health) - shown by default - Per-storage statistics (leader and replica counts) @@ -45,6 +54,10 @@ Examples: By default, only cluster statistics are displayed. Use --list-partitions to see a partition overview table. Use --storage to filter partitions by storage. +Output formats: + - text (default): Human-readable colored tables and statistics + - json: Machine-readable JSON for automation and scripting + For detailed partition information, use 'gitaly cluster get-partition'.`, Flags: []cli.Flag{ &cli.StringFlag{ @@ -65,6 +78,11 @@ For detailed partition information, use 'gitaly cluster get-partition'.`, Name: flagClusterInfoNoColor, Usage: "disable colored output", }, + &cli.StringFlag{ + Name: flagClusterInfoFormat, + Usage: "output format: 'text' (default) or 'json'", + Value: "text", + }, }, Action: clusterInfoAction, } @@ -77,9 +95,12 @@ func clusterInfoAction(ctx context.Context, cmd *cli.Command) error { storage := cmd.String(flagClusterInfoStorage) listPartitions := cmd.Bool(flagClusterInfoListPartitions) noColor := cmd.Bool(flagClusterInfoNoColor) + format := cmd.String(flagClusterInfoFormat) - // Configure color output - colorOutput := setupColorOutput(cmd.Writer, noColor) + // Validate format + if format != "text" && format != "json" { + return fmt.Errorf("invalid format %q: must be 'text' or 'json'", format) + } // Create Raft client using shared helper raftClient, cleanup, err := loadConfigAndCreateRaftClient(ctx, configPath) @@ -88,17 +109,29 @@ func clusterInfoAction(ctx context.Context, cmd *cli.Command) error { } defer cleanup() - // Display cluster info with optimized RPC calls - return displayClusterInfo(ctx, cmd.Writer, raftClient, storage, listPartitions, colorOutput) + // Fetch cluster data + clusterInfoResp, partitions, err := fetchClusterData(ctx, raftClient, storage, listPartitions) + if err != nil { + return err + } + + // Output based on format + if format == "json" { + return outputClusterInfoJSON(cmd.Writer, clusterInfoResp, partitions, storage, listPartitions) + } + + // Configure color output for text format + colorOutput := setupColorOutput(cmd.Writer, noColor) + return outputClusterInfoText(cmd.Writer, clusterInfoResp, partitions, storage, listPartitions, colorOutput) } -// displayClusterInfo calls GetClusterInfo and optionally GetPartitions RPCs, processes the data, and displays the results -func displayClusterInfo(ctx context.Context, writer io.Writer, client gitalypb.RaftServiceClient, storage string, listPartitions bool, colorOutput *colorOutput) error { +// fetchClusterData retrieves cluster information and optionally partition details from the Raft service +func fetchClusterData(ctx context.Context, client gitalypb.RaftServiceClient, storage string, listPartitions bool) (*gitalypb.RaftClusterInfoResponse, []*gitalypb.GetPartitionsResponse, error) { // Step 1: Always get cluster statistics using GetClusterInfo RPC var clusterInfoReq *gitalypb.RaftClusterInfoRequest clusterInfoResp, err := client.GetClusterInfo(ctx, clusterInfoReq) if err != nil { - return fmt.Errorf("failed to retrieve cluster information - verify server is running and Raft is enabled: %w", err) + return nil, nil, fmt.Errorf("failed to retrieve cluster information - verify server is running and Raft is enabled: %w", err) } // Step 2: Only get partition details if needed (when --list-partitions is set or --storage filter is provided) @@ -120,22 +153,82 @@ func displayClusterInfo(ctx context.Context, writer io.Writer, client gitalypb.R partitionsStream, err := client.GetPartitions(ctx, partitionsReq) if err != nil { - return fmt.Errorf("failed to retrieve partition details - verify server is running and storage %q exists: %w", storage, err) + return nil, nil, fmt.Errorf("failed to retrieve partition details - verify server is running and storage %q exists: %w", storage, err) } // Collect all partition responses using helper function partitionResponses, err = collectPartitionResponses(partitionsStream) if err != nil { - return err + return nil, nil, err + } + } + + return clusterInfoResp, partitionResponses, nil +} + +// outputClusterInfoJSON outputs cluster information in JSON format +func outputClusterInfoJSON(writer io.Writer, clusterInfoResp *gitalypb.RaftClusterInfoResponse, partitions []*gitalypb.GetPartitionsResponse, storage string, listPartitions bool) error { + // Include partitions in JSON if they were fetched (either via --list-partitions or --storage filter) + includePartitions := listPartitions || storage != "" + + // Sort partitions by partition key for consistent output + if includePartitions && len(partitions) > 0 { + sortPartitionsByKey(partitions) + } + + // Configure protojson marshaler + marshaler := protojson.MarshalOptions{ + EmitUnpopulated: false, // Omit fields with default values + UseProtoNames: false, // Use lowerCamelCase JSON field names + } + + // Convert protobuf to JSON, then to map for standard JSON marshaling + clusterInfoBytes, err := marshaler.Marshal(clusterInfoResp) + if err != nil { + return fmt.Errorf("failed to marshal cluster info: %w", err) + } + + var clusterInfoMap map[string]interface{} + if err := json.Unmarshal(clusterInfoBytes, &clusterInfoMap); err != nil { + return fmt.Errorf("failed to unmarshal cluster info: %w", err) + } + + // Build the output structure + output := map[string]interface{}{ + "clusterInfo": clusterInfoMap, + } + + // Add partitions if requested + if includePartitions && len(partitions) > 0 { + var partitionsArray []map[string]interface{} + for _, partition := range partitions { + partitionBytes, err := marshaler.Marshal(partition) + if err != nil { + return fmt.Errorf("failed to marshal partition: %w", err) + } + + var partitionMap map[string]interface{} + if err := json.Unmarshal(partitionBytes, &partitionMap); err != nil { + return fmt.Errorf("failed to unmarshal partition: %w", err) + } + + partitionsArray = append(partitionsArray, partitionMap) } + output["partitions"] = partitionsArray } - // Step 3: Display results using the RPC responses - return displayFormattedResults(writer, clusterInfoResp, partitionResponses, storage, listPartitions, colorOutput) + // Marshal with indentation for human-friendly output + jsonBytes, err := json.MarshalIndent(output, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal JSON output: %w", err) + } + + fmt.Fprintf(writer, "%s\n", string(jsonBytes)) + return nil } -// displayFormattedResults displays cluster information using tabular format -func displayFormattedResults(writer io.Writer, clusterInfoResp *gitalypb.RaftClusterInfoResponse, partitions []*gitalypb.GetPartitionsResponse, storage string, listPartitions bool, colorOutput *colorOutput) error { +// outputClusterInfoText displays cluster information in human-readable text format +func outputClusterInfoText(writer io.Writer, clusterInfoResp *gitalypb.RaftClusterInfoResponse, partitions []*gitalypb.GetPartitionsResponse, storage string, listPartitions bool, colorOutput *colorOutput) error { fmt.Fprintf(writer, "%s\n\n", colorOutput.formatHeader("=== Gitaly Cluster Information ===")) // Display cluster statistics overview diff --git a/internal/cli/gitaly/subcmd_cluster_info_test.go b/internal/cli/gitaly/subcmd_cluster_info_test.go index ef7bfab10fe..be97f0ab601 100644 --- a/internal/cli/gitaly/subcmd_cluster_info_test.go +++ b/internal/cli/gitaly/subcmd_cluster_info_test.go @@ -2,7 +2,6 @@ package gitaly import ( "bytes" - "fmt" "strings" "testing" @@ -107,16 +106,6 @@ storage-3 0 2 Use --list-partitions to display partition overview table. `, }, - { - name: "basic cluster info without partitions (robust check)", - setupServer: func(t *testing.T) (string, func()) { - return setupRaftServerForPartition(t, setupTestData) - }, - args: []string{}, - expectError: false, - // This test uses content validation instead of exact string matching - expectedOutput: "ROBUST_CHECK", - }, { name: "cluster info with show partitions flag", setupServer: func(t *testing.T) (string, func()) { @@ -185,6 +174,386 @@ PARTITION KEY LEADER REP ae3928eb528786e728edb0583f06ec25d4d0f41f3ad6105a8c2777790d8cfc98 storage-2 storage-1, storage-2, storage-3 (filtered: storage-1) 3/3 150 150 1 repos `, }, + { + name: "cluster info with JSON format (basic)", + setupServer: func(t *testing.T) (string, func()) { + return setupRaftServerForPartition(t, setupTestData) + }, + args: []string{"--format", "json"}, + expectError: false, + expectedOutput: `{ + "clusterInfo": { + "clusterId": "test-cluster", + "statistics": { + "healthyPartitions": 2, + "healthyReplicas": 6, + "storageStats": { + "storage-1": { + "leaderCount": 1, + "replicaCount": 2 + }, + "storage-2": { + "leaderCount": 1, + "replicaCount": 2 + }, + "storage-3": { + "replicaCount": 2 + } + }, + "totalPartitions": 2, + "totalReplicas": 6 + } + } +} +`, + }, + { + name: "cluster info with JSON format and list-partitions", + setupServer: func(t *testing.T) (string, func()) { + return setupRaftServerForPartition(t, setupTestData) + }, + args: []string{"--format", "json", "--list-partitions"}, + expectError: false, + expectedOutput: `{ + "clusterInfo": { + "clusterId": "test-cluster", + "statistics": { + "healthyPartitions": 2, + "healthyReplicas": 6, + "storageStats": { + "storage-1": { + "leaderCount": 1, + "replicaCount": 2 + }, + "storage-2": { + "leaderCount": 1, + "replicaCount": 2 + }, + "storage-3": { + "replicaCount": 2 + } + }, + "totalPartitions": 2, + "totalReplicas": 6 + } + }, + "partitions": [ + { + "clusterId": "test-cluster", + "index": "100", + "leaderId": "1", + "partitionKey": { + "value": "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad" + }, + "relativePath": "@hashed/ab/cd/repo1.git", + "relativePaths": [ + "@hashed/ab/cd/repo1.git" + ], + "replicas": [ + { + "isHealthy": true, + "isLeader": true, + "lastIndex": "100", + "matchIndex": "100", + "replicaId": { + "memberId": "1", + "metadata": { + "address": "gitaly-1.example.com:8075" + }, + "partitionKey": { + "value": "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad" + }, + "storageName": "storage-1", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "leader" + }, + { + "isHealthy": true, + "lastIndex": "100", + "matchIndex": "100", + "replicaId": { + "memberId": "2", + "metadata": { + "address": "gitaly-2.example.com:8075" + }, + "partitionKey": { + "value": "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad" + }, + "storageName": "storage-2", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "follower" + }, + { + "isHealthy": true, + "lastIndex": "100", + "matchIndex": "100", + "replicaId": { + "memberId": "3", + "metadata": { + "address": "gitaly-3.example.com:8075" + }, + "partitionKey": { + "value": "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad" + }, + "storageName": "storage-3", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "follower" + } + ], + "term": "5" + }, + { + "clusterId": "test-cluster", + "index": "150", + "leaderId": "5", + "partitionKey": { + "value": "ae3928eb528786e728edb0583f06ec25d4d0f41f3ad6105a8c2777790d8cfc98" + }, + "relativePath": "@hashed/ef/gh/repo2.git", + "relativePaths": [ + "@hashed/ef/gh/repo2.git" + ], + "replicas": [ + { + "isHealthy": true, + "lastIndex": "150", + "matchIndex": "150", + "replicaId": { + "memberId": "4", + "metadata": { + "address": "gitaly-1.example.com:8075" + }, + "partitionKey": { + "value": "ae3928eb528786e728edb0583f06ec25d4d0f41f3ad6105a8c2777790d8cfc98" + }, + "storageName": "storage-1", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "follower" + }, + { + "isHealthy": true, + "isLeader": true, + "lastIndex": "150", + "matchIndex": "150", + "replicaId": { + "memberId": "5", + "metadata": { + "address": "gitaly-2.example.com:8075" + }, + "partitionKey": { + "value": "ae3928eb528786e728edb0583f06ec25d4d0f41f3ad6105a8c2777790d8cfc98" + }, + "storageName": "storage-2", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "leader" + }, + { + "isHealthy": true, + "lastIndex": "150", + "matchIndex": "150", + "replicaId": { + "memberId": "6", + "metadata": { + "address": "gitaly-3.example.com:8075" + }, + "partitionKey": { + "value": "ae3928eb528786e728edb0583f06ec25d4d0f41f3ad6105a8c2777790d8cfc98" + }, + "storageName": "storage-3", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "follower" + } + ], + "term": "6" + } + ] +} +`, + }, + { + name: "cluster info with JSON format and storage filter", + setupServer: func(t *testing.T) (string, func()) { + return setupRaftServerForPartition(t, setupTestData) + }, + args: []string{"--format", "json", "--storage", storageOne}, + expectError: false, + expectedOutput: `{ + "clusterInfo": { + "clusterId": "test-cluster", + "statistics": { + "healthyPartitions": 2, + "healthyReplicas": 6, + "storageStats": { + "storage-1": { + "leaderCount": 1, + "replicaCount": 2 + }, + "storage-2": { + "leaderCount": 1, + "replicaCount": 2 + }, + "storage-3": { + "replicaCount": 2 + } + }, + "totalPartitions": 2, + "totalReplicas": 6 + } + }, + "partitions": [ + { + "clusterId": "test-cluster", + "index": "100", + "leaderId": "1", + "partitionKey": { + "value": "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad" + }, + "relativePath": "@hashed/ab/cd/repo1.git", + "relativePaths": [ + "@hashed/ab/cd/repo1.git" + ], + "replicas": [ + { + "isHealthy": true, + "isLeader": true, + "lastIndex": "100", + "matchIndex": "100", + "replicaId": { + "memberId": "1", + "metadata": { + "address": "gitaly-1.example.com:8075" + }, + "partitionKey": { + "value": "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad" + }, + "storageName": "storage-1", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "leader" + }, + { + "isHealthy": true, + "lastIndex": "100", + "matchIndex": "100", + "replicaId": { + "memberId": "2", + "metadata": { + "address": "gitaly-2.example.com:8075" + }, + "partitionKey": { + "value": "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad" + }, + "storageName": "storage-2", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "follower" + }, + { + "isHealthy": true, + "lastIndex": "100", + "matchIndex": "100", + "replicaId": { + "memberId": "3", + "metadata": { + "address": "gitaly-3.example.com:8075" + }, + "partitionKey": { + "value": "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad" + }, + "storageName": "storage-3", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "follower" + } + ], + "term": "5" + }, + { + "clusterId": "test-cluster", + "index": "150", + "leaderId": "5", + "partitionKey": { + "value": "ae3928eb528786e728edb0583f06ec25d4d0f41f3ad6105a8c2777790d8cfc98" + }, + "relativePath": "@hashed/ef/gh/repo2.git", + "relativePaths": [ + "@hashed/ef/gh/repo2.git" + ], + "replicas": [ + { + "isHealthy": true, + "lastIndex": "150", + "matchIndex": "150", + "replicaId": { + "memberId": "4", + "metadata": { + "address": "gitaly-1.example.com:8075" + }, + "partitionKey": { + "value": "ae3928eb528786e728edb0583f06ec25d4d0f41f3ad6105a8c2777790d8cfc98" + }, + "storageName": "storage-1", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "follower" + }, + { + "isHealthy": true, + "isLeader": true, + "lastIndex": "150", + "matchIndex": "150", + "replicaId": { + "memberId": "5", + "metadata": { + "address": "gitaly-2.example.com:8075" + }, + "partitionKey": { + "value": "ae3928eb528786e728edb0583f06ec25d4d0f41f3ad6105a8c2777790d8cfc98" + }, + "storageName": "storage-2", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "leader" + }, + { + "isHealthy": true, + "lastIndex": "150", + "matchIndex": "150", + "replicaId": { + "memberId": "6", + "metadata": { + "address": "gitaly-3.example.com:8075" + }, + "partitionKey": { + "value": "ae3928eb528786e728edb0583f06ec25d4d0f41f3ad6105a8c2777790d8cfc98" + }, + "storageName": "storage-3", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "follower" + } + ], + "term": "6" + } + ] +} +`, + }, + { + name: "cluster info with invalid format", + setupServer: func(t *testing.T) (string, func()) { + return setupRaftServerForPartition(t, setupTestData) + }, + args: []string{"--format", "yaml"}, + expectError: true, + expectedOutput: "invalid format \"yaml\": must be 'text' or 'json'", + }, } for _, tc := range tests { @@ -211,45 +580,12 @@ ae3928eb528786e728edb0583f06ec25d4d0f41f3ad6105a8c2777790d8cfc98 storage-2 sto require.Contains(t, err.Error(), tc.expectedOutput) } else { require.NoError(t, err, "Command should execute successfully") - - // Check if this is a robust test case - if tc.expectedOutput == "ROBUST_CHECK" { - // Use content-based validation instead of exact string matching - assertClusterStatistics(t, actualOutput, 2, 6) - assertOutputContains(t, actualOutput, []string{ - "=== Gitaly Cluster Information ===", - "=== Per-Storage Statistics ===", - "storage-1", - "storage-2", - "storage-3", - "Use --list-partitions to display partition overview table.", - }) - } else { - require.Equal(t, tc.expectedOutput, actualOutput, "Output should match exactly") - } + require.Equal(t, tc.expectedOutput, actualOutput, "Output should match exactly") } }) } } -// assertOutputContains checks that output contains all expected content parts without enforcing exact formatting -func assertOutputContains(t *testing.T, output string, expectedParts []string) { - t.Helper() - for _, part := range expectedParts { - require.Contains(t, output, part, "Output should contain: %q", part) - } -} - -// assertClusterStatistics validates cluster statistics in output without exact formatting -func assertClusterStatistics(t *testing.T, output string, expectedPartitions, expectedReplicas int) { - t.Helper() - require.Contains(t, output, "=== Cluster Statistics ===") - require.Contains(t, output, fmt.Sprintf("Total Partitions: %d", expectedPartitions)) - require.Contains(t, output, fmt.Sprintf("Total Replicas: %d", expectedReplicas)) - require.Contains(t, output, fmt.Sprintf("Healthy Partitions: %d", expectedPartitions)) - require.Contains(t, output, fmt.Sprintf("Healthy Replicas: %d", expectedReplicas)) -} - // setupTestData populates the Raft cluster with test partition data func setupTestData(t *testing.T, cfg any, node *raftmgr.Node) { // Set up mock routing table entries -- GitLab From a4f658d7a43519163d8a9f215a950fc650dabd1e Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Fri, 10 Oct 2025 17:49:34 +0700 Subject: [PATCH 2/4] cli: Add JSON output format to cluster get-partition command The cluster get-partition command previously only supported human-readable text output with colored tables. This made it difficult to consume partition information programmatically for automation scripts, monitoring systems, or integration with other tools. This commit adds a --format flag with two values: "text" (default) and "json". The JSON format outputs partition details using protojson marshaling, ensuring consistency with the protobuf definitions. The implementation maintains full backward compatibility by keeping text as the default format. The code was refactored into three distinct functions: - fetchPartitionData() retrieves partition data via RPC - outputPartitionDetailsText() formats output as human-readable text - outputPartitionDetailsJSON() formats output as machine-readable JSON The JSON output includes complete partition metadata: clusterId, partitionKey, term, index, leaderId, replicas array, and relativePaths. Both --partition-key and --relative-path filters work correctly with JSON output. --- .../gitaly/subcmd_cluster_get_partition.go | 105 +++++++++-- .../subcmd_cluster_get_partition_test.go | 168 ++++++++++++++++++ 2 files changed, 257 insertions(+), 16 deletions(-) diff --git a/internal/cli/gitaly/subcmd_cluster_get_partition.go b/internal/cli/gitaly/subcmd_cluster_get_partition.go index 972bec1d6ce..d2a636ecfa7 100644 --- a/internal/cli/gitaly/subcmd_cluster_get_partition.go +++ b/internal/cli/gitaly/subcmd_cluster_get_partition.go @@ -2,6 +2,7 @@ package gitaly import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -9,6 +10,7 @@ import ( "github.com/urfave/cli/v3" "gitlab.com/gitlab-org/gitaly/v18/proto/go/gitalypb" + "google.golang.org/protobuf/encoding/protojson" ) const ( @@ -16,20 +18,24 @@ const ( flagGetPartitionPartitionKey = "partition-key" flagGetPartitionRelativePath = "relative-path" flagGetPartitionNoColor = "no-color" + flagGetPartitionFormat = "format" ) func newClusterGetPartitionCommand() *cli.Command { return &cli.Command{ Name: "get-partition", Usage: "display detailed partition information", - UsageText: `gitaly cluster get-partition --config [--partition-key ] [--relative-path ] + UsageText: `gitaly cluster get-partition --config [--partition-key ] [--relative-path ] [--format ] Examples: # Get detailed info for a specific partition by key (64-character SHA256 hex) gitaly cluster get-partition --config config.toml --partition-key abc123... # Get partition info for a specific repository path - gitaly cluster get-partition --config config.toml --relative-path @hashed/ab/cd/abcd...`, + gitaly cluster get-partition --config config.toml --relative-path @hashed/ab/cd/abcd... + + # Output as JSON for programmatic consumption + gitaly cluster get-partition --config config.toml --partition-key abc123... --format json`, Description: `Display detailed information about specific partitions including: - Partition key and replica topology - Leader/follower status for each replica @@ -38,7 +44,11 @@ Examples: Use --partition-key to filter by a specific partition key, or --relative-path to find the partition containing a specific repository. When using --relative-path, the output -shows the partition that contains the specified repository.`, +shows the partition that contains the specified repository. + +Output formats: + - text (default): Human-readable colored tables and detailed information + - json: Machine-readable JSON for automation and scripting`, Flags: []cli.Flag{ &cli.StringFlag{ Name: flagGetPartitionConfig, @@ -58,6 +68,11 @@ shows the partition that contains the specified repository.`, Name: flagGetPartitionNoColor, Usage: "disable colored output", }, + &cli.StringFlag{ + Name: flagGetPartitionFormat, + Usage: "output format: 'text' (default) or 'json'", + Value: "text", + }, }, Action: getPartitionAction, } @@ -70,9 +85,12 @@ func getPartitionAction(ctx context.Context, cmd *cli.Command) error { partitionKey := cmd.String(flagGetPartitionPartitionKey) relativePath := cmd.String(flagGetPartitionRelativePath) noColor := cmd.Bool(flagGetPartitionNoColor) + format := cmd.String(flagGetPartitionFormat) - // Configure color output - colorOutput := setupColorOutput(cmd.Writer, noColor) + // Validate format + if format != "text" && format != "json" { + return fmt.Errorf("invalid format %q: must be 'text' or 'json'", format) + } // Validate that at least one filter is provided if partitionKey == "" && relativePath == "" { @@ -96,12 +114,24 @@ func getPartitionAction(ctx context.Context, cmd *cli.Command) error { } defer cleanup() - // Display partition details - return displayPartitionDetails(ctx, cmd.Writer, raftClient, partitionKey, relativePath, colorOutput) + // Fetch partition data + partitions, err := fetchPartitionData(ctx, raftClient, partitionKey, relativePath) + if err != nil { + return err + } + + // Output based on format + if format == "json" { + return outputPartitionDetailsJSON(cmd.Writer, partitions) + } + + // Configure color output for text format + colorOutput := setupColorOutput(cmd.Writer, noColor) + return outputPartitionDetailsText(cmd.Writer, partitions, partitionKey, relativePath, colorOutput) } -// displayPartitionDetails calls RPCs and displays detailed partition information -func displayPartitionDetails(ctx context.Context, writer io.Writer, client gitalypb.RaftServiceClient, partitionKey, relativePath string, colorOutput *colorOutput) error { +// fetchPartitionData retrieves partition details from the Raft service +func fetchPartitionData(ctx context.Context, client gitalypb.RaftServiceClient, partitionKey, relativePath string) ([]*gitalypb.GetPartitionsResponse, error) { // Get partition details using GetPartitions RPC partitionsReq := &gitalypb.GetPartitionsRequest{ IncludeRelativePaths: true, @@ -118,21 +148,64 @@ func displayPartitionDetails(ctx context.Context, writer io.Writer, client gital partitionsStream, err := client.GetPartitions(ctx, partitionsReq) if err != nil { - return fmt.Errorf("failed to retrieve partition information - verify server is running and Raft is enabled: %w", err) + return nil, fmt.Errorf("failed to retrieve partition information - verify server is running and Raft is enabled: %w", err) } - // Step 2: Collect all partition responses using helper function + // Collect all partition responses using helper function partitionResponses, err := collectPartitionResponses(partitionsStream) if err != nil { - return err + return nil, err + } + + return partitionResponses, nil +} + +// outputPartitionDetailsJSON outputs partition details in JSON format +func outputPartitionDetailsJSON(writer io.Writer, partitions []*gitalypb.GetPartitionsResponse) error { + // Sort partitions by partition key for consistent output + if len(partitions) > 0 { + sortPartitionsByKey(partitions) + } + + // Configure protojson marshaler + marshaler := protojson.MarshalOptions{ + EmitUnpopulated: false, // Omit fields with default values + UseProtoNames: false, // Use lowerCamelCase JSON field names } - // Step 3: Display results - return displayFormattedPartitionDetails(writer, partitionResponses, partitionKey, relativePath, colorOutput) + // Build the partitions array + var partitionsArray []map[string]interface{} + for _, partition := range partitions { + partitionBytes, err := marshaler.Marshal(partition) + if err != nil { + return fmt.Errorf("failed to marshal partition: %w", err) + } + + var partitionMap map[string]interface{} + if err := json.Unmarshal(partitionBytes, &partitionMap); err != nil { + return fmt.Errorf("failed to unmarshal partition: %w", err) + } + + partitionsArray = append(partitionsArray, partitionMap) + } + + // Build the output structure + output := map[string]interface{}{ + "partitions": partitionsArray, + } + + // Marshal with indentation for human-friendly output + jsonBytes, err := json.MarshalIndent(output, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal JSON output: %w", err) + } + + fmt.Fprintf(writer, "%s\n", string(jsonBytes)) + return nil } -// displayFormattedPartitionDetails displays detailed partition information -func displayFormattedPartitionDetails(writer io.Writer, partitions []*gitalypb.GetPartitionsResponse, partitionKey, relativePath string, colorOutput *colorOutput) error { +// outputPartitionDetailsText displays detailed partition information in human-readable text format +func outputPartitionDetailsText(writer io.Writer, partitions []*gitalypb.GetPartitionsResponse, partitionKey, relativePath string, colorOutput *colorOutput) error { // Display detailed partition information if len(partitions) > 0 { // Sort partitions by partition key for consistent output diff --git a/internal/cli/gitaly/subcmd_cluster_get_partition_test.go b/internal/cli/gitaly/subcmd_cluster_get_partition_test.go index 29e190c1a27..e9f8e1b93ff 100644 --- a/internal/cli/gitaly/subcmd_cluster_get_partition_test.go +++ b/internal/cli/gitaly/subcmd_cluster_get_partition_test.go @@ -176,6 +176,174 @@ REPOSITORY PATH }, args: []string{"--partition-key", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, expectedOutput: `No partitions found matching the specified criteria. +`, + }, + { + name: "invalid format value", + setupServer: func(t *testing.T) (string, func()) { + cfg := testcfg.Build(t) + return testcfg.WriteTemporaryGitalyConfigFile(t, cfg), func() {} + }, + args: []string{"--partition-key", "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad", "--format", "yaml"}, + expectError: true, + expectedOutput: "invalid format \"yaml\": must be 'text' or 'json'", + }, + { + name: "get partition with JSON format and partition key", + setupServer: func(t *testing.T) (string, func()) { + return setupRaftServerForPartition(t, setupTestDataForPartition) + }, + args: []string{"--partition-key", "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad", "--format", "json"}, + expectedOutput: `{ + "partitions": [ + { + "clusterId": "test-cluster", + "index": "100", + "leaderId": "1", + "partitionKey": { + "value": "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad" + }, + "relativePath": "@hashed/ab/cd/repo1.git", + "relativePaths": [ + "@hashed/ab/cd/repo1.git" + ], + "replicas": [ + { + "isHealthy": true, + "isLeader": true, + "lastIndex": "100", + "matchIndex": "100", + "replicaId": { + "memberId": "1", + "metadata": { + "address": "gitaly-1.example.com:8075" + }, + "partitionKey": { + "value": "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad" + }, + "storageName": "storage-1", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "leader" + }, + { + "isHealthy": true, + "lastIndex": "100", + "matchIndex": "100", + "replicaId": { + "memberId": "2", + "metadata": { + "address": "gitaly-2.example.com:8075" + }, + "partitionKey": { + "value": "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad" + }, + "storageName": "storage-2", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "follower" + }, + { + "isHealthy": true, + "lastIndex": "100", + "matchIndex": "100", + "replicaId": { + "memberId": "3", + "metadata": { + "address": "gitaly-3.example.com:8075" + }, + "partitionKey": { + "value": "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad" + }, + "storageName": "storage-3", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "follower" + } + ], + "term": "5" + } + ] +} +`, + }, + { + name: "get partition with JSON format and relative path", + setupServer: func(t *testing.T) (string, func()) { + return setupRaftServerForPartition(t, setupTestDataForPartition) + }, + args: []string{"--relative-path", "@hashed/ab/cd/repo1.git", "--format", "json"}, + expectedOutput: `{ + "partitions": [ + { + "clusterId": "test-cluster", + "index": "100", + "leaderId": "1", + "partitionKey": { + "value": "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad" + }, + "relativePath": "@hashed/ab/cd/repo1.git", + "relativePaths": [ + "@hashed/ab/cd/repo1.git" + ], + "replicas": [ + { + "isHealthy": true, + "isLeader": true, + "lastIndex": "100", + "matchIndex": "100", + "replicaId": { + "memberId": "1", + "metadata": { + "address": "gitaly-1.example.com:8075" + }, + "partitionKey": { + "value": "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad" + }, + "storageName": "storage-1", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "leader" + }, + { + "isHealthy": true, + "lastIndex": "100", + "matchIndex": "100", + "replicaId": { + "memberId": "2", + "metadata": { + "address": "gitaly-2.example.com:8075" + }, + "partitionKey": { + "value": "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad" + }, + "storageName": "storage-2", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "follower" + }, + { + "isHealthy": true, + "lastIndex": "100", + "matchIndex": "100", + "replicaId": { + "memberId": "3", + "metadata": { + "address": "gitaly-3.example.com:8075" + }, + "partitionKey": { + "value": "1ae75994b13cfe1d19983e0d7eeac7b4a7077bd9c4a26e3421c1acd3d683a4ad" + }, + "storageName": "storage-3", + "type": "REPLICA_TYPE_VOTER" + }, + "state": "follower" + } + ], + "term": "5" + } + ] +} `, }, } -- GitLab From f6599ce4d55e0ea1965bb3fcc8d2902ce648a9ae Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Tue, 14 Oct 2025 16:17:04 +0700 Subject: [PATCH 3/4] raft: ClusterID generation if already exists If GITALY_TEST_RAFT env variable is set, Gitaly test suite injects a Raft config filled with default values into testing Gitaly server. ClusterID is generated randomly. Some tests fetching cluster ID fail due to this reason. This commit lets Gitaly ignore ID generation if it already exists. --- internal/testhelper/testserver/gitaly.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/testhelper/testserver/gitaly.go b/internal/testhelper/testserver/gitaly.go index b995c1c68c9..7193a01ae45 100644 --- a/internal/testhelper/testserver/gitaly.go +++ b/internal/testhelper/testserver/gitaly.go @@ -380,7 +380,11 @@ func (gsd *gitalyServerDeps) createDependencies(tb testing.TB, ctx context.Conte var raftFactory raftmgr.RaftReplicaFactory var raftNode *raftmgr.Node if testhelper.IsRaftEnabled() && !testhelper.IsPraefectEnabled() { - cfg.Raft = config.DefaultRaftConfig(uuid.New().String()) + clusterID := cfg.Raft.ClusterID + if clusterID == "" { + clusterID = uuid.New().String() + } + cfg.Raft = config.DefaultRaftConfig(clusterID) // Speed up initial election overhead in the test setup cfg.Raft.ElectionTicks = 5 cfg.Raft.RTTMilliseconds = 100 -- GitLab From da08394439e259deb7b4a40211412eadbb8f1410 Mon Sep 17 00:00:00 2001 From: Quang-Minh Nguyen Date: Mon, 20 Oct 2025 17:36:03 +0700 Subject: [PATCH 4/4] cli: Refactor cluster commands with structured output types This commit addresses MR feedback to establish an explicit contract for JSON output by introducing structured Go types. The previous implementation scattered JSON marshaling across command files, making the output structure implicit and harder for clients to rely on when consuming the JSON API. The refactoring introduces ClusterInfoOutput and PartitionDetailsOutput types that encapsulate output data and provide ToJSON() and ToText() methods. This design makes the JSON contract explicit through Go type definitions, ensuring clients can depend on a stable output structure. All output rendering logic is centralized in cluster_output.go for better maintainability. The change also eliminates redundant sortPartitionsByKey() evaluation in subcmd_cluster_info.go by moving sorting into the output type methods where it's needed. --- internal/cli/gitaly/cluster_output.go | 375 ++++++++++++++++++ .../gitaly/subcmd_cluster_get_partition.go | 132 +----- internal/cli/gitaly/subcmd_cluster_info.go | 246 +----------- 3 files changed, 390 insertions(+), 363 deletions(-) create mode 100644 internal/cli/gitaly/cluster_output.go diff --git a/internal/cli/gitaly/cluster_output.go b/internal/cli/gitaly/cluster_output.go new file mode 100644 index 00000000000..a262f71cafc --- /dev/null +++ b/internal/cli/gitaly/cluster_output.go @@ -0,0 +1,375 @@ +package gitaly + +import ( + "encoding/json" + "fmt" + "io" + "sort" + "strings" + "text/tabwriter" + + "gitlab.com/gitlab-org/gitaly/v18/proto/go/gitalypb" + "google.golang.org/protobuf/encoding/protojson" +) + +// ClusterInfoOutput represents the structured output for the cluster info command. +// This type defines the contract for JSON output, making it easier to maintain +// compatibility with clients consuming the JSON data. +type ClusterInfoOutput struct { + ClusterInfo *gitalypb.RaftClusterInfoResponse + Partitions []*gitalypb.GetPartitionsResponse +} + +// ToJSON outputs the cluster info in JSON format +func (o *ClusterInfoOutput) ToJSON(writer io.Writer) error { + // Sort partitions by partition key for consistent output + if len(o.Partitions) > 0 { + sortPartitionsByKey(o.Partitions) + } + + // Configure protojson marshaler + marshaler := protojson.MarshalOptions{ + EmitUnpopulated: false, // Omit fields with default values + UseProtoNames: false, // Use lowerCamelCase JSON field names + } + + // Convert protobuf to JSON, then to map for standard JSON marshaling + clusterInfoBytes, err := marshaler.Marshal(o.ClusterInfo) + if err != nil { + return fmt.Errorf("failed to marshal cluster info: %w", err) + } + + var clusterInfoMap map[string]interface{} + if err := json.Unmarshal(clusterInfoBytes, &clusterInfoMap); err != nil { + return fmt.Errorf("failed to unmarshal cluster info: %w", err) + } + + // Build the output structure + output := map[string]interface{}{ + "clusterInfo": clusterInfoMap, + } + + // Add partitions if present + if len(o.Partitions) > 0 { + var partitionsArray []map[string]interface{} + for _, partition := range o.Partitions { + partitionBytes, err := marshaler.Marshal(partition) + if err != nil { + return fmt.Errorf("failed to marshal partition: %w", err) + } + + var partitionMap map[string]interface{} + if err := json.Unmarshal(partitionBytes, &partitionMap); err != nil { + return fmt.Errorf("failed to unmarshal partition: %w", err) + } + + partitionsArray = append(partitionsArray, partitionMap) + } + output["partitions"] = partitionsArray + } + + // Marshal with indentation for human-friendly output + jsonBytes, err := json.MarshalIndent(output, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal JSON output: %w", err) + } + + fmt.Fprintf(writer, "%s\n", string(jsonBytes)) + return nil +} + +// ToText outputs the cluster info in human-readable text format +func (o *ClusterInfoOutput) ToText(writer io.Writer, colorOutput *colorOutput, storage string, listPartitions bool) error { + fmt.Fprintf(writer, "%s\n\n", colorOutput.formatHeader("=== Gitaly Cluster Information ===")) + + // Display cluster statistics overview + if o.ClusterInfo.GetStatistics() != nil { + if err := displayClusterStatistics(writer, o.ClusterInfo.GetStatistics(), storage, colorOutput); err != nil { + return err + } + } + + // Display partition overview table if requested + if listPartitions || storage != "" { + if len(o.Partitions) > 0 { + // Sort partitions by partition key for consistent output + sortPartitionsByKey(o.Partitions) + + return displayPartitionTable(writer, o.Partitions, storage, colorOutput) + } else if storage != "" { + fmt.Fprintf(writer, "No partitions found for storage: %s\n", storage) + } + } else { + // Suggest showing partitions if not displayed + fmt.Fprintf(writer, "%s\n", colorOutput.formatInfo("Use --list-partitions to display partition overview table.")) + } + + return nil +} + +// PartitionDetailsOutput represents the structured output for the get-partition command. +// This type defines the contract for JSON output, making it easier to maintain +// compatibility with clients consuming the JSON data. +type PartitionDetailsOutput struct { + Partitions []*gitalypb.GetPartitionsResponse +} + +// ToJSON outputs the partition details in JSON format +func (o *PartitionDetailsOutput) ToJSON(writer io.Writer) error { + // Sort partitions by partition key for consistent output + if len(o.Partitions) > 0 { + sortPartitionsByKey(o.Partitions) + } + + // Configure protojson marshaler + marshaler := protojson.MarshalOptions{ + EmitUnpopulated: false, // Omit fields with default values + UseProtoNames: false, // Use lowerCamelCase JSON field names + } + + // Build the partitions array + var partitionsArray []map[string]interface{} + for _, partition := range o.Partitions { + partitionBytes, err := marshaler.Marshal(partition) + if err != nil { + return fmt.Errorf("failed to marshal partition: %w", err) + } + + var partitionMap map[string]interface{} + if err := json.Unmarshal(partitionBytes, &partitionMap); err != nil { + return fmt.Errorf("failed to unmarshal partition: %w", err) + } + + partitionsArray = append(partitionsArray, partitionMap) + } + + // Build the output structure + output := map[string]interface{}{ + "partitions": partitionsArray, + } + + // Marshal with indentation for human-friendly output + jsonBytes, err := json.MarshalIndent(output, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal JSON output: %w", err) + } + + fmt.Fprintf(writer, "%s\n", string(jsonBytes)) + return nil +} + +// ToText outputs the partition details in human-readable text format +func (o *PartitionDetailsOutput) ToText(writer io.Writer, colorOutput *colorOutput, partitionKey, relativePath string) error { + // Display detailed partition information + if len(o.Partitions) > 0 { + // Sort partitions by partition key for consistent output + sortPartitionsByKey(o.Partitions) + + if relativePath != "" { + fmt.Fprintf(writer, "%s\n\n", colorOutput.formatHeader(fmt.Sprintf("=== Partition Details for Repository: %s ===", relativePath))) + } else if partitionKey != "" { + fmt.Fprintf(writer, "%s\n\n", colorOutput.formatHeader(fmt.Sprintf("=== Partition Details for Key: %s ===", partitionKey))) + } + + for i, partition := range o.Partitions { + if i > 0 { + fmt.Fprintf(writer, "\n") + } + + fmt.Fprintf(writer, "Partition: %s\n\n", colorOutput.formatInfo(partition.GetPartitionKey().GetValue())) + + // Display replicas in tabular format + if len(partition.GetReplicas()) > 0 { + tw := tabwriter.NewWriter(writer, 0, 0, 2, ' ', 0) + + fmt.Fprintf(tw, "STORAGE\tROLE\tHEALTH\tLAST INDEX\tMATCH INDEX\n") + fmt.Fprintf(tw, "-------\t----\t------\t----------\t-----------\n") + + for _, replica := range partition.GetReplicas() { + var role string + if replica.GetIsLeader() { + role = colorOutput.formatInfo("Leader") + } else { + role = "Follower" + } + var health string + if replica.GetIsHealthy() { + health = colorOutput.formatHealthy("Healthy") + } else { + health = colorOutput.formatUnhealthy("Unhealthy") + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%d\t%d\n", + replica.GetReplicaId().GetStorageName(), + role, + health, + replica.GetLastIndex(), + replica.GetMatchIndex()) + } + + _ = tw.Flush() + fmt.Fprintf(writer, "\n") + } + + // Display repositories in tabular format + if len(partition.GetRelativePaths()) > 0 { + fmt.Fprintf(writer, "%s\n\n", colorOutput.formatHeader("Repositories:")) + + tw := tabwriter.NewWriter(writer, 0, 0, 2, ' ', 0) + + fmt.Fprintf(tw, "REPOSITORY PATH\n") + fmt.Fprintf(tw, "---------------\n") + + for _, path := range partition.GetRelativePaths() { + fmt.Fprintf(tw, "%s\n", path) + } + + _ = tw.Flush() + } + } + } else { + fmt.Fprintf(writer, "No partitions found matching the specified criteria.\n") + } + + return nil +} + +// displayClusterStatistics displays cluster-wide statistics in a readable format +func displayClusterStatistics(writer io.Writer, stats *gitalypb.ClusterStatistics, storageFilter string, colorOutput *colorOutput) error { + // Display overall cluster health at the top + partitionHealth := colorOutput.formatHealthStatus(int(stats.GetHealthyPartitions()), int(stats.GetTotalPartitions())) + replicaHealth := colorOutput.formatHealthStatus(int(stats.GetHealthyReplicas()), int(stats.GetTotalReplicas())) + + fmt.Fprintf(writer, "%s\n\n", colorOutput.formatHeader("=== Cluster Health Summary ===")) + fmt.Fprintf(writer, " Partitions: %s\n", partitionHealth) + fmt.Fprintf(writer, " Replicas: %s\n\n", replicaHealth) + + fmt.Fprintf(writer, "%s\n", colorOutput.formatHeader("=== Cluster Statistics ===")) + fmt.Fprintf(writer, " Total Partitions: %s\n", colorOutput.formatInfo(fmt.Sprintf("%d", stats.GetTotalPartitions()))) + fmt.Fprintf(writer, " Total Replicas: %s\n", colorOutput.formatInfo(fmt.Sprintf("%d", stats.GetTotalReplicas()))) + fmt.Fprintf(writer, " Healthy Partitions: %s\n", colorOutput.formatInfo(fmt.Sprintf("%d", stats.GetHealthyPartitions()))) + fmt.Fprintf(writer, " Healthy Replicas: %s\n", colorOutput.formatInfo(fmt.Sprintf("%d", stats.GetHealthyReplicas()))) + fmt.Fprintf(writer, "\n") + + if len(stats.GetStorageStats()) > 0 { + fmt.Fprintf(writer, "%s\n\n", colorOutput.formatHeader("=== Per-Storage Statistics ===")) + + // Filter storage names if a storage filter is specified + var storageNames []string + if storageFilter != "" { + // Only show the filtered storage if it exists + if _, exists := stats.GetStorageStats()[storageFilter]; exists { + storageNames = append(storageNames, storageFilter) + } + } else { + // Show all storages if no filter is specified + for storageName := range stats.GetStorageStats() { + storageNames = append(storageNames, storageName) + } + sort.Strings(storageNames) + } + + // Create table writer for storage statistics + tw := tabwriter.NewWriter(writer, 0, 0, 2, ' ', 0) + + // Table headers + fmt.Fprintf(tw, "STORAGE\tLEADER COUNT\tREPLICA COUNT\n") + fmt.Fprintf(tw, "-------\t------------\t-------------\n") + + for _, storageName := range storageNames { + storageStat := stats.GetStorageStats()[storageName] + fmt.Fprintf(tw, "%s\t%d\t%d\n", + storageName, + storageStat.GetLeaderCount(), + storageStat.GetReplicaCount()) + } + + // Flush the table and add spacing + if err := tw.Flush(); err != nil { + return fmt.Errorf("failed to flush storage statistics table: %w", err) + } + fmt.Fprintf(writer, "\n") + } + return nil +} + +// displayPartitionTable displays partitions in a tabular format +func displayPartitionTable(writer io.Writer, partitions []*gitalypb.GetPartitionsResponse, storageFilter string, colorOutput *colorOutput) error { + fmt.Fprintf(writer, "%s\n\n", colorOutput.formatHeader("=== Partition Overview ===")) + + // Create table writer + tw := tabwriter.NewWriter(writer, 0, 0, 2, ' ', 0) + + // Table headers + fmt.Fprintf(tw, "PARTITION KEY\tLEADER\tREPLICAS\tHEALTH\tLAST INDEX\tMATCH INDEX\tREPOSITORIES\n") + fmt.Fprintf(tw, "-------------\t------\t--------\t------\t----------\t-----------\t------------\n") + + for _, partition := range partitions { + // Find leader and collect replica info + leader := "None" + healthyReplicas := 0 + totalReplicas := len(partition.GetReplicas()) + var replicaStorages []string + var filteredReplicas []string + var leaderLastIndex, leaderMatchIndex uint64 + + for _, replica := range partition.GetReplicas() { + storageName := replica.GetReplicaId().GetStorageName() + if replica.GetIsLeader() { + leader = colorOutput.formatInfo(storageName) + leaderLastIndex = replica.GetLastIndex() + leaderMatchIndex = replica.GetMatchIndex() + } + if replica.GetIsHealthy() { + healthyReplicas++ + } + replicaStorages = append(replicaStorages, storageName) + + // Collect replicas matching storage filter + if storageFilter != "" && storageName == storageFilter { + filteredReplicas = append(filteredReplicas, storageName) + } + } + + // Format replica list showing storage names + replicasStr := strings.Join(replicaStorages, ", ") + if storageFilter != "" && len(filteredReplicas) > 0 { + // Show all replicas but highlight the filtered ones + replicasStr = fmt.Sprintf("%s (filtered: %s)", replicasStr, strings.Join(filteredReplicas, ", ")) + } + + // Format health status with color + var healthStr string + if healthyReplicas == totalReplicas && totalReplicas > 0 { + healthStr = colorOutput.formatHealthy(fmt.Sprintf("%d/%d", healthyReplicas, totalReplicas)) + } else if healthyReplicas == 0 { + healthStr = colorOutput.formatUnhealthy(fmt.Sprintf("%d/%d", healthyReplicas, totalReplicas)) + } else { + healthStr = colorOutput.formatWarning(fmt.Sprintf("%d/%d", healthyReplicas, totalReplicas)) + } + + // Format last index and match index + lastIndexStr := "N/A" + matchIndexStr := "N/A" + if leader != "None" { + lastIndexStr = fmt.Sprintf("%d", leaderLastIndex) + matchIndexStr = fmt.Sprintf("%d", leaderMatchIndex) + } + + // Format repository count + repoCount := len(partition.GetRelativePaths()) + repoStr := fmt.Sprintf("%d repos", repoCount) + + // Display full partition key + partitionKeyDisplay := partition.GetPartitionKey().GetValue() + + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", + partitionKeyDisplay, leader, replicasStr, healthStr, lastIndexStr, matchIndexStr, repoStr) + } + + // Flush the partition table + if err := tw.Flush(); err != nil { + return fmt.Errorf("failed to flush partition overview table: %w", err) + } + + return nil +} diff --git a/internal/cli/gitaly/subcmd_cluster_get_partition.go b/internal/cli/gitaly/subcmd_cluster_get_partition.go index d2a636ecfa7..f1745dc090d 100644 --- a/internal/cli/gitaly/subcmd_cluster_get_partition.go +++ b/internal/cli/gitaly/subcmd_cluster_get_partition.go @@ -2,15 +2,11 @@ package gitaly import ( "context" - "encoding/json" "errors" "fmt" - "io" - "text/tabwriter" "github.com/urfave/cli/v3" "gitlab.com/gitlab-org/gitaly/v18/proto/go/gitalypb" - "google.golang.org/protobuf/encoding/protojson" ) const ( @@ -120,14 +116,19 @@ func getPartitionAction(ctx context.Context, cmd *cli.Command) error { return err } + // Prepare output structure + output := &PartitionDetailsOutput{ + Partitions: partitions, + } + // Output based on format if format == "json" { - return outputPartitionDetailsJSON(cmd.Writer, partitions) + return output.ToJSON(cmd.Writer) } // Configure color output for text format colorOutput := setupColorOutput(cmd.Writer, noColor) - return outputPartitionDetailsText(cmd.Writer, partitions, partitionKey, relativePath, colorOutput) + return output.ToText(cmd.Writer, colorOutput, partitionKey, relativePath) } // fetchPartitionData retrieves partition details from the Raft service @@ -159,122 +160,3 @@ func fetchPartitionData(ctx context.Context, client gitalypb.RaftServiceClient, return partitionResponses, nil } - -// outputPartitionDetailsJSON outputs partition details in JSON format -func outputPartitionDetailsJSON(writer io.Writer, partitions []*gitalypb.GetPartitionsResponse) error { - // Sort partitions by partition key for consistent output - if len(partitions) > 0 { - sortPartitionsByKey(partitions) - } - - // Configure protojson marshaler - marshaler := protojson.MarshalOptions{ - EmitUnpopulated: false, // Omit fields with default values - UseProtoNames: false, // Use lowerCamelCase JSON field names - } - - // Build the partitions array - var partitionsArray []map[string]interface{} - for _, partition := range partitions { - partitionBytes, err := marshaler.Marshal(partition) - if err != nil { - return fmt.Errorf("failed to marshal partition: %w", err) - } - - var partitionMap map[string]interface{} - if err := json.Unmarshal(partitionBytes, &partitionMap); err != nil { - return fmt.Errorf("failed to unmarshal partition: %w", err) - } - - partitionsArray = append(partitionsArray, partitionMap) - } - - // Build the output structure - output := map[string]interface{}{ - "partitions": partitionsArray, - } - - // Marshal with indentation for human-friendly output - jsonBytes, err := json.MarshalIndent(output, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal JSON output: %w", err) - } - - fmt.Fprintf(writer, "%s\n", string(jsonBytes)) - return nil -} - -// outputPartitionDetailsText displays detailed partition information in human-readable text format -func outputPartitionDetailsText(writer io.Writer, partitions []*gitalypb.GetPartitionsResponse, partitionKey, relativePath string, colorOutput *colorOutput) error { - // Display detailed partition information - if len(partitions) > 0 { - // Sort partitions by partition key for consistent output - sortPartitionsByKey(partitions) - - if relativePath != "" { - fmt.Fprintf(writer, "%s\n\n", colorOutput.formatHeader(fmt.Sprintf("=== Partition Details for Repository: %s ===", relativePath))) - } else if partitionKey != "" { - fmt.Fprintf(writer, "%s\n\n", colorOutput.formatHeader(fmt.Sprintf("=== Partition Details for Key: %s ===", partitionKey))) - } - - for i, partition := range partitions { - if i > 0 { - fmt.Fprintf(writer, "\n") - } - - fmt.Fprintf(writer, "Partition: %s\n\n", colorOutput.formatInfo(partition.GetPartitionKey().GetValue())) - - // Display replicas in tabular format - if len(partition.GetReplicas()) > 0 { - tw := tabwriter.NewWriter(writer, 0, 0, 2, ' ', 0) - - fmt.Fprintf(tw, "STORAGE\tROLE\tHEALTH\tLAST INDEX\tMATCH INDEX\n") - fmt.Fprintf(tw, "-------\t----\t------\t----------\t-----------\n") - - for _, replica := range partition.GetReplicas() { - var role string - if replica.GetIsLeader() { - role = colorOutput.formatInfo("Leader") - } else { - role = "Follower" - } - var health string - if replica.GetIsHealthy() { - health = colorOutput.formatHealthy("Healthy") - } else { - health = colorOutput.formatUnhealthy("Unhealthy") - } - fmt.Fprintf(tw, "%s\t%s\t%s\t%d\t%d\n", - replica.GetReplicaId().GetStorageName(), - role, - health, - replica.GetLastIndex(), - replica.GetMatchIndex()) - } - - _ = tw.Flush() - fmt.Fprintf(writer, "\n") - } - - // Display repositories in tabular format - if len(partition.GetRelativePaths()) > 0 { - fmt.Fprintf(writer, "%s\n\n", colorOutput.formatHeader("Repositories:")) - - tw := tabwriter.NewWriter(writer, 0, 0, 2, ' ', 0) - - fmt.Fprintf(tw, "REPOSITORY PATH\n") - fmt.Fprintf(tw, "---------------\n") - - for _, path := range partition.GetRelativePaths() { - fmt.Fprintf(tw, "%s\n", path) - } - - _ = tw.Flush() - } - } - } else { - fmt.Fprintf(writer, "No partitions found matching the specified criteria.\n") - } - - return nil -} diff --git a/internal/cli/gitaly/subcmd_cluster_info.go b/internal/cli/gitaly/subcmd_cluster_info.go index b53f0ab20e0..4c43eb7ba9d 100644 --- a/internal/cli/gitaly/subcmd_cluster_info.go +++ b/internal/cli/gitaly/subcmd_cluster_info.go @@ -2,19 +2,14 @@ package gitaly import ( "context" - "encoding/json" "fmt" "io" "os" - "sort" - "strings" - "text/tabwriter" "github.com/fatih/color" "github.com/mattn/go-isatty" "github.com/urfave/cli/v3" "gitlab.com/gitlab-org/gitaly/v18/proto/go/gitalypb" - "google.golang.org/protobuf/encoding/protojson" ) const ( @@ -115,14 +110,20 @@ func clusterInfoAction(ctx context.Context, cmd *cli.Command) error { return err } + // Prepare output structure + output := &ClusterInfoOutput{ + ClusterInfo: clusterInfoResp, + Partitions: partitions, + } + // Output based on format if format == "json" { - return outputClusterInfoJSON(cmd.Writer, clusterInfoResp, partitions, storage, listPartitions) + return output.ToJSON(cmd.Writer) } // Configure color output for text format colorOutput := setupColorOutput(cmd.Writer, noColor) - return outputClusterInfoText(cmd.Writer, clusterInfoResp, partitions, storage, listPartitions, colorOutput) + return output.ToText(cmd.Writer, colorOutput, storage, listPartitions) } // fetchClusterData retrieves cluster information and optionally partition details from the Raft service @@ -166,237 +167,6 @@ func fetchClusterData(ctx context.Context, client gitalypb.RaftServiceClient, st return clusterInfoResp, partitionResponses, nil } -// outputClusterInfoJSON outputs cluster information in JSON format -func outputClusterInfoJSON(writer io.Writer, clusterInfoResp *gitalypb.RaftClusterInfoResponse, partitions []*gitalypb.GetPartitionsResponse, storage string, listPartitions bool) error { - // Include partitions in JSON if they were fetched (either via --list-partitions or --storage filter) - includePartitions := listPartitions || storage != "" - - // Sort partitions by partition key for consistent output - if includePartitions && len(partitions) > 0 { - sortPartitionsByKey(partitions) - } - - // Configure protojson marshaler - marshaler := protojson.MarshalOptions{ - EmitUnpopulated: false, // Omit fields with default values - UseProtoNames: false, // Use lowerCamelCase JSON field names - } - - // Convert protobuf to JSON, then to map for standard JSON marshaling - clusterInfoBytes, err := marshaler.Marshal(clusterInfoResp) - if err != nil { - return fmt.Errorf("failed to marshal cluster info: %w", err) - } - - var clusterInfoMap map[string]interface{} - if err := json.Unmarshal(clusterInfoBytes, &clusterInfoMap); err != nil { - return fmt.Errorf("failed to unmarshal cluster info: %w", err) - } - - // Build the output structure - output := map[string]interface{}{ - "clusterInfo": clusterInfoMap, - } - - // Add partitions if requested - if includePartitions && len(partitions) > 0 { - var partitionsArray []map[string]interface{} - for _, partition := range partitions { - partitionBytes, err := marshaler.Marshal(partition) - if err != nil { - return fmt.Errorf("failed to marshal partition: %w", err) - } - - var partitionMap map[string]interface{} - if err := json.Unmarshal(partitionBytes, &partitionMap); err != nil { - return fmt.Errorf("failed to unmarshal partition: %w", err) - } - - partitionsArray = append(partitionsArray, partitionMap) - } - output["partitions"] = partitionsArray - } - - // Marshal with indentation for human-friendly output - jsonBytes, err := json.MarshalIndent(output, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal JSON output: %w", err) - } - - fmt.Fprintf(writer, "%s\n", string(jsonBytes)) - return nil -} - -// outputClusterInfoText displays cluster information in human-readable text format -func outputClusterInfoText(writer io.Writer, clusterInfoResp *gitalypb.RaftClusterInfoResponse, partitions []*gitalypb.GetPartitionsResponse, storage string, listPartitions bool, colorOutput *colorOutput) error { - fmt.Fprintf(writer, "%s\n\n", colorOutput.formatHeader("=== Gitaly Cluster Information ===")) - - // Display cluster statistics overview - if clusterInfoResp.GetStatistics() != nil { - if err := displayClusterStatistics(writer, clusterInfoResp.GetStatistics(), storage, colorOutput); err != nil { - return err - } - } - - // Display partition overview table if requested - if listPartitions || storage != "" { - if len(partitions) > 0 { - // Sort partitions by partition key for consistent output - sortPartitionsByKey(partitions) - - return displayPartitionTable(writer, partitions, storage, colorOutput) - } else if storage != "" { - fmt.Fprintf(writer, "No partitions found for storage: %s\n", storage) - } - } else { - // Suggest showing partitions if not displayed - fmt.Fprintf(writer, "%s\n", colorOutput.formatInfo("Use --list-partitions to display partition overview table.")) - } - - return nil -} - -// displayClusterStatistics displays cluster-wide statistics in a readable format -func displayClusterStatistics(writer io.Writer, stats *gitalypb.ClusterStatistics, storageFilter string, colorOutput *colorOutput) error { - // Display overall cluster health at the top - partitionHealth := colorOutput.formatHealthStatus(int(stats.GetHealthyPartitions()), int(stats.GetTotalPartitions())) - replicaHealth := colorOutput.formatHealthStatus(int(stats.GetHealthyReplicas()), int(stats.GetTotalReplicas())) - - fmt.Fprintf(writer, "%s\n\n", colorOutput.formatHeader("=== Cluster Health Summary ===")) - fmt.Fprintf(writer, " Partitions: %s\n", partitionHealth) - fmt.Fprintf(writer, " Replicas: %s\n\n", replicaHealth) - - fmt.Fprintf(writer, "%s\n", colorOutput.formatHeader("=== Cluster Statistics ===")) - fmt.Fprintf(writer, " Total Partitions: %s\n", colorOutput.formatInfo(fmt.Sprintf("%d", stats.GetTotalPartitions()))) - fmt.Fprintf(writer, " Total Replicas: %s\n", colorOutput.formatInfo(fmt.Sprintf("%d", stats.GetTotalReplicas()))) - fmt.Fprintf(writer, " Healthy Partitions: %s\n", colorOutput.formatInfo(fmt.Sprintf("%d", stats.GetHealthyPartitions()))) - fmt.Fprintf(writer, " Healthy Replicas: %s\n", colorOutput.formatInfo(fmt.Sprintf("%d", stats.GetHealthyReplicas()))) - fmt.Fprintf(writer, "\n") - - if len(stats.GetStorageStats()) > 0 { - fmt.Fprintf(writer, "%s\n\n", colorOutput.formatHeader("=== Per-Storage Statistics ===")) - - // Filter storage names if a storage filter is specified - var storageNames []string - if storageFilter != "" { - // Only show the filtered storage if it exists - if _, exists := stats.GetStorageStats()[storageFilter]; exists { - storageNames = append(storageNames, storageFilter) - } - } else { - // Show all storages if no filter is specified - for storageName := range stats.GetStorageStats() { - storageNames = append(storageNames, storageName) - } - sort.Strings(storageNames) - } - - // Create table writer for storage statistics - tw := tabwriter.NewWriter(writer, 0, 0, 2, ' ', 0) - - // Table headers - fmt.Fprintf(tw, "STORAGE\tLEADER COUNT\tREPLICA COUNT\n") - fmt.Fprintf(tw, "-------\t------------\t-------------\n") - - for _, storageName := range storageNames { - storageStat := stats.GetStorageStats()[storageName] - fmt.Fprintf(tw, "%s\t%d\t%d\n", - storageName, - storageStat.GetLeaderCount(), - storageStat.GetReplicaCount()) - } - - // Flush the table and add spacing - if err := tw.Flush(); err != nil { - return fmt.Errorf("failed to flush storage statistics table: %w", err) - } - fmt.Fprintf(writer, "\n") - } - return nil -} - -// displayPartitionTable displays partitions in a tabular format -func displayPartitionTable(writer io.Writer, partitions []*gitalypb.GetPartitionsResponse, storageFilter string, colorOutput *colorOutput) error { - fmt.Fprintf(writer, "%s\n\n", colorOutput.formatHeader("=== Partition Overview ===")) - - // Create table writer - tw := tabwriter.NewWriter(writer, 0, 0, 2, ' ', 0) - - // Table headers - fmt.Fprintf(tw, "PARTITION KEY\tLEADER\tREPLICAS\tHEALTH\tLAST INDEX\tMATCH INDEX\tREPOSITORIES\n") - fmt.Fprintf(tw, "-------------\t------\t--------\t------\t----------\t-----------\t------------\n") - - for _, partition := range partitions { - // Find leader and collect replica info - leader := "None" - healthyReplicas := 0 - totalReplicas := len(partition.GetReplicas()) - var replicaStorages []string - var filteredReplicas []string - var leaderLastIndex, leaderMatchIndex uint64 - - for _, replica := range partition.GetReplicas() { - storageName := replica.GetReplicaId().GetStorageName() - if replica.GetIsLeader() { - leader = colorOutput.formatInfo(storageName) - leaderLastIndex = replica.GetLastIndex() - leaderMatchIndex = replica.GetMatchIndex() - } - if replica.GetIsHealthy() { - healthyReplicas++ - } - replicaStorages = append(replicaStorages, storageName) - - // Collect replicas matching storage filter - if storageFilter != "" && storageName == storageFilter { - filteredReplicas = append(filteredReplicas, storageName) - } - } - - // Format replica list showing storage names - replicasStr := strings.Join(replicaStorages, ", ") - if storageFilter != "" && len(filteredReplicas) > 0 { - // Show all replicas but highlight the filtered ones - replicasStr = fmt.Sprintf("%s (filtered: %s)", replicasStr, strings.Join(filteredReplicas, ", ")) - } - - // Format health status with color - var healthStr string - if healthyReplicas == totalReplicas && totalReplicas > 0 { - healthStr = colorOutput.formatHealthy(fmt.Sprintf("%d/%d", healthyReplicas, totalReplicas)) - } else if healthyReplicas == 0 { - healthStr = colorOutput.formatUnhealthy(fmt.Sprintf("%d/%d", healthyReplicas, totalReplicas)) - } else { - healthStr = colorOutput.formatWarning(fmt.Sprintf("%d/%d", healthyReplicas, totalReplicas)) - } - - // Format last index and match index - lastIndexStr := "N/A" - matchIndexStr := "N/A" - if leader != "None" { - lastIndexStr = fmt.Sprintf("%d", leaderLastIndex) - matchIndexStr = fmt.Sprintf("%d", leaderMatchIndex) - } - - // Format repository count - repoCount := len(partition.GetRelativePaths()) - repoStr := fmt.Sprintf("%d repos", repoCount) - - // Display full partition key - partitionKeyDisplay := partition.GetPartitionKey().GetValue() - - fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", - partitionKeyDisplay, leader, replicasStr, healthStr, lastIndexStr, matchIndexStr, repoStr) - } - - // Flush the partition table - if err := tw.Flush(); err != nil { - return fmt.Errorf("failed to flush partition overview table: %w", err) - } - - return nil -} - // colorOutput holds color configuration for terminal output type colorOutput struct { enabled bool -- GitLab