Skip to content

test(testutil): retry terraform provider cache population on transient failures #26196

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
jscottmiller wants to merge 3 commits into
base: main
Choose a base branch
from
+37 −7
Draft

test(testutil): retry terraform provider cache population on transient failures #26196

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b60f6f0
test(testutil): retry terraform provider cache population on transien…
jscottmiller Jun 9, 2026
ff41a97
test(testutil): widen provider cache retry window
jscottmiller Jun 9, 2026
b5582fc
test(testutil): tighten cache retry comment
jscottmiller Jun 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 37 additions & 7 deletions testutil/terraform_cache.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ package testutil

import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
Expand All @@ -13,8 +14,11 @@ import (
"slices"
"strings"
"testing"
"time"

"github.com/stretchr/testify/require"

"github.com/coder/retry"
)

const (
Expand Down Expand Up @@ -85,17 +89,43 @@ func WriteTFCliConfig(t *testing.T, dir string) string {
return cliConfigPath
}

// runCmdMaxAttempts is the number of times runCmd attempts a command before
// failing the test. These commands populate the provider cache from the
// network, which intermittently returns transient registry/GitHub 5xx errors.
// Retrying any non-zero exit absorbs those flakes, and is safe because
// `terraform init` and `terraform providers mirror` are idempotent.
//
// The window is wide (5 attempts over ~1 minute via retry.New(5s, 30s)) because
// registry/GitHub incidents last seconds to minutes, not a single request. It
// is cheap because this path runs only on a cache miss (see DownloadTFProviders).
const runCmdMaxAttempts = 5

func runCmd(t *testing.T, dir string, args ...string) {
t.Helper()

stdout, stderr := bytes.NewBuffer(nil), bytes.NewBuffer(nil)
cmd := exec.Command(args[0], args[1:]...) //#nosec
cmd.Dir = dir
cmd.Stdout = stdout
cmd.Stderr = stderr
if err := cmd.Run(); err != nil {
t.Fatalf("failed to run %s: %s\nstdout: %s\nstderr: %s", strings.Join(args, " "), err, stdout.String(), stderr.String())
ctx := context.Background()
var (
attempt int
lastErr error
lastStdout, lastStderr string
)
for r := retry.New(5*time.Second, 30*time.Second); attempt < runCmdMaxAttempts && r.Wait(ctx); attempt++ {
stdout, stderr := bytes.NewBuffer(nil), bytes.NewBuffer(nil)
cmd := exec.Command(args[0], args[1:]...) //#nosec
cmd.Dir = dir
cmd.Stdout = stdout
cmd.Stderr = stderr
err := cmd.Run()
if err == nil {
return
}
lastErr = err
lastStdout, lastStderr = stdout.String(), stderr.String()
t.Logf("attempt %d/%d to run %s failed: %s\nstderr: %s",
attempt+1, runCmdMaxAttempts, strings.Join(args, " "), err, lastStderr)
}
t.Fatalf("failed to run %s after %d attempts: %s\nstdout: %s\nstderr: %s",
strings.Join(args, " "), runCmdMaxAttempts, lastErr, lastStdout, lastStderr)
}

// GetTestTFCacheDir returns a unique cache directory path based on the test name and template files.
Expand Down
Loading