diff --git a/internal/gitaly/service/operations/tags_test.go b/internal/gitaly/service/operations/tags_test.go
index ccd03e157a88d018f69fcb88e606877998475797..d90d42d769e1171d0ba5cd13660c071e31a3234b 100644
--- a/internal/gitaly/service/operations/tags_test.go
+++ b/internal/gitaly/service/operations/tags_test.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"os/exec"
 	"path/filepath"
 	"testing"
 
@@ -38,7 +37,7 @@ func testSuccessfulUserDeleteTagRequest(t *testing.T, ctx context.Context) {
 
 	tagNameInput := "to-be-deleted-soon-tag"
 
-	defer exec.Command(config.Config.Git.BinPath, "-C", testRepoPath, "tag", "-d", tagNameInput).Run()
+	defer testhelper.MayFailRunCommand(t, nil, "git", "-C", testRepoPath, "tag", "-d", tagNameInput)
 
 	testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "tag", tagNameInput)
 
@@ -73,7 +72,6 @@ func testSuccessfulGitHooksForUserDeleteTagRequest(t *testing.T, ctx context.Con
 	defer cleanupFn()
 
 	tagNameInput := "to-be-déleted-soon-tag"
-	defer exec.Command(config.Config.Git.BinPath, "-C", testRepoPath, "tag", "-d", tagNameInput).Run()
 
 	request := &gitalypb.UserDeleteTagRequest{
 		Repository: testRepo,
@@ -84,6 +82,7 @@ func testSuccessfulGitHooksForUserDeleteTagRequest(t *testing.T, ctx context.Con
 	for _, hookName := range GitlabHooks {
 		t.Run(hookName, func(t *testing.T) {
 			testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "tag", tagNameInput)
+			defer testhelper.MayFailRunCommand(t, nil, "git", "-C", testRepoPath, "tag", "-d", tagNameInput)
 
 			hookOutputTempPath, cleanup := testhelper.WriteEnvToCustomHook(t, testRepoPath, hookName)
 			defer cleanup()
@@ -239,7 +238,7 @@ func testSuccessfulUserCreateTagRequest(t *testing.T, ctx context.Context) {
 			require.NoError(t, err, "error from calling RPC")
 			require.Empty(t, response.PreReceiveError, "PreReceiveError must be empty, signalling the push was accepted")
 
-			defer exec.Command(config.Config.Git.BinPath, "-C", testRepoPath, "tag", "-d", inputTagName).Run()
+			defer testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "tag", "-d", inputTagName)
 
 			responseOk := &gitalypb.UserCreateTagResponse{
 				Tag: testCase.expectedTag,
@@ -360,7 +359,7 @@ func TestSuccessfulUserCreateTagRequestToNonCommit(t *testing.T) {
 				Tag: testCase.expectedTag,
 			}
 			response, err := client.UserCreateTag(ctx, request)
-			defer exec.Command(config.Config.Git.BinPath, "-C", testRepoPath, "tag", "-d", inputTagName).Run()
+			defer testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "tag", "-d", inputTagName)
 
 			// Fake up *.Id for annotated tags
 			if len(testCase.expectedTag.Id) == 0 {
@@ -450,7 +449,7 @@ func TestSuccessfulUserCreateTagNestedTags(t *testing.T) {
 					Message:        []byte(tagMessage),
 				}
 				response, err := client.UserCreateTag(ctx, request)
-				defer exec.Command(config.Config.Git.BinPath, "-C", testRepoPath, "tag", "-d", tagName).Run()
+				defer testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "tag", "-d", tagName)
 				require.NoError(t, err)
 
 				createdID := testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "rev-parse", tagName)
@@ -523,7 +522,7 @@ func testUserDeleteTagsuccessfulDeletionOfPrefixedTag(t *testing.T, ctx context.
 	for _, testCase := range testCases {
 		t.Run(testCase.desc, func(t *testing.T) {
 			testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "tag", testCase.tagNameInput, testCase.tagCommit)
-			defer exec.Command(config.Config.Git.BinPath, "-C", testRepoPath, "tag", "-d", testCase.tagNameInput).Run()
+			defer testhelper.MayFailRunCommand(t, nil, "git", "-C", testRepoPath, "tag", "-d", testCase.tagNameInput)
 
 			request := &gitalypb.UserDeleteTagRequest{
 				Repository: testRepo,
@@ -571,7 +570,7 @@ func TestUserCreateTagsuccessfulCreationOfPrefixedTag(t *testing.T) {
 
 	for _, testCase := range testCases {
 		t.Run(testCase.desc, func(t *testing.T) {
-			defer exec.Command(config.Config.Git.BinPath, "-C", testRepoPath, "tag", "-d", testCase.tagNameInput).Run()
+			defer testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "tag", "-d", testCase.tagNameInput)
 
 			request := &gitalypb.UserCreateTagRequest{
 				Repository:     testRepo,
@@ -635,7 +634,7 @@ func testSuccessfulGitHooksForUserCreateTagRequest(t *testing.T, ctx context.Con
 
 	for _, hookName := range GitlabHooks {
 		t.Run(hookName, func(t *testing.T) {
-			defer exec.Command(config.Config.Git.BinPath, "-C", testRepoPath, "tag", "-d", tagName).Run()
+			defer testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "tag", "-d", tagName)
 
 			hookOutputTempPath, cleanup := testhelper.WriteEnvToCustomHook(t, testRepoPath, hookName)
 			defer cleanup()
@@ -732,7 +731,7 @@ func testFailedUserDeleteTagDueToHooks(t *testing.T, ctx context.Context) {
 
 	tagNameInput := "to-be-deleted-soon-tag"
 	testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "tag", tagNameInput)
-	defer exec.Command(config.Config.Git.BinPath, "-C", testRepoPath, "tag", "-d", tagNameInput).Run()
+	defer testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "tag", "-d", tagNameInput)
 
 	request := &gitalypb.UserDeleteTagRequest{
 		Repository: testRepo,
@@ -976,7 +975,7 @@ func testTagHookOutput(t *testing.T, ctx context.Context) {
 				require.False(t, createResponse.Exists)
 				require.Equal(t, testCase.output, createResponse.PreReceiveError)
 
-				defer exec.Command(config.Config.Git.BinPath, "-C", testRepoPath, "tag", "-d", tagNameInput).Run()
+				defer testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "tag", "-d", tagNameInput)
 				testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "tag", tagNameInput)
 
 				deleteResponse, err := client.UserDeleteTag(ctx, deleteRequest)
diff --git a/internal/testhelper/testhelper.go b/internal/testhelper/testhelper.go
index d36ff70f0174981408048a80526aa7d718ea4429..6cfca0210de9da1b0455730f1d464966a0b314e4 100644
--- a/internal/testhelper/testhelper.go
+++ b/internal/testhelper/testhelper.go
@@ -87,8 +87,7 @@ func GitalyServersMetadata(t testing.TB, serverSocketPath string) metadata.MD {
 	return metadata.Pairs("gitaly-servers", base64.StdEncoding.EncodeToString(gitalyServersJSON))
 }
 
-// MustRunCommand runs a command with an optional standard input and returns the standard output, or fails.
-func MustRunCommand(t testing.TB, stdin io.Reader, name string, args ...string) []byte {
+func testRunCommandInternal(t testing.TB, failOk bool, stdin io.Reader, name string, args ...string) []byte {
 	if t != nil {
 		t.Helper()
 	}
@@ -111,7 +110,7 @@ func MustRunCommand(t testing.TB, stdin io.Reader, name string, args ...string)
 	}
 
 	output, err := cmd.Output()
-	if err != nil {
+	if err != nil && !failOk {
 		stderr := err.(*exec.ExitError).Stderr
 		if t == nil {
 			log.Print(name, args)
@@ -127,6 +126,21 @@ func MustRunCommand(t testing.TB, stdin io.Reader, name string, args ...string)
 	return output
 }
 
+// MustRunCommand runs a command with an optional standard input and returns the standard output, or fails.
+func MustRunCommand(t testing.TB, stdin io.Reader, name string, args ...string) []byte {
+	return testRunCommandInternal(t, false, stdin, name, args...)
+}
+
+// MayFailRunCommand is MustRunCommand which ignores failures.
+//
+// Useful e.g. for normally redundant teardown if a test doesn't die,
+// but which should run in a "defer" block to cleanup things for the
+// next test if the current test were to die in the middle of its own
+// setup/testing/teardown.
+func MayFailRunCommand(t testing.TB, stdin io.Reader, name string, args ...string) []byte {
+	return testRunCommandInternal(t, true, stdin, name, args...)
+}
+
 // GetTemporaryGitalySocketFileName will return a unique, useable socket file name
 func GetTemporaryGitalySocketFileName() string {
 	if testDirectory == "" {