Skip to content

chore: cache terraform providers for workspaces terraform tests #20603

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kacpersaw merged 3 commits into from
Nov 12, 2025
Merged

chore: cache terraform providers for workspaces terraform tests #20603

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2457f28
chore: cache terraform providers for workspaces terraform tests
kacpersaw Oct 31, 2025
d3ff141
Apply review suggestions
kacpersaw Nov 11, 2025
f6ccaab
Merge branch 'main' into kacpersaw/workspace-tags-flake
Nov 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 9 additions & 43 deletions enterprise/coderd/workspaces_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@ import (
"encoding/json"
"fmt"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"sync/atomic"
Expand Down Expand Up @@ -3390,51 +3388,19 @@ func workspaceTagsTerraform(t *testing.T, tc testWorkspaceTagsTerraformCase, dyn
}
}

// downloadProviders is a test helper that creates a temporary file and writes a
// terraform CLI config file with a provider_installation stanza for coder/coder
// using dev_overrides. It also fetches the latest provider release from GitHub
// and extracts the binary to the temporary dir. It is the responsibility of the
// caller to set TF_CLI_CONFIG_FILE.
// downloadProviders is a test helper that caches Terraform providers and returns
// the path to a Terraform CLI config file that uses the cached providers.
// This uses the shared testutil caching infrastructure to avoid re-downloading
// providers on every test run. It is the responsibility of the caller to set
// TF_CLI_CONFIG_FILE.
func downloadProviders(t *testing.T, providersTf string) string {
t.Helper()
// We firstly write a Terraform CLI config file to a temporary directory:
var (
tempDir = t.TempDir()
cacheDir = filepath.Join(tempDir, ".cache")
providersTfPath = filepath.Join(tempDir, "providers.tf")
cliConfigPath = filepath.Join(tempDir, "local.tfrc")
)

// Write files to disk
require.NoError(t, os.MkdirAll(cacheDir, os.ModePerm|os.ModeDir))
require.NoError(t, os.WriteFile(providersTfPath, []byte(providersTf), os.ModePerm)) // nolint:gosec
cliConfigTemplate := `
provider_installation {
filesystem_mirror {
path = %q
include = ["*/*/*"]
}
direct {
exclude = ["*/*/*"]
}
}`
err := os.WriteFile(cliConfigPath, []byte(fmt.Sprintf(cliConfigTemplate, cacheDir)), os.ModePerm) // nolint:gosec
require.NoError(t, err, "failed to write %s", cliConfigPath)

ctx := testutil.Context(t, testutil.WaitLong)

// Run terraform providers mirror to mirror required providers to cacheDir
cmd := exec.CommandContext(ctx, "terraform", "providers", "mirror", cacheDir)
cmd.Env = os.Environ() // without this terraform may complain about path
cmd.Env = append(cmd.Env, "TF_CLI_CONFIG_FILE="+cliConfigPath)
cmd.Dir = tempDir
out, err := cmd.CombinedOutput()
if !assert.NoError(t, err) {
t.Log("failed to download providers:")
t.Log(string(out))
t.FailNow()
}
cacheRootDir := filepath.Join(testutil.PersistentCacheDir(t), "terraform_workspace_tags_test")
templateFiles := map[string]string{"providers.tf": providersTf}
testName := "TestWorkspaceTagsTerraform"

cliConfigPath := testutil.CacheTFProviders(t, cacheRootDir, testName, templateFiles)
t.Logf("Set TF_CLI_CONFIG_FILE=%s", cliConfigPath)
return cliConfigPath
}
Expand Down
170 changes: 2 additions & 168 deletions provisioner/terraform/provision_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,13 @@
package terraform_test

import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
Expand Down Expand Up @@ -94,168 +90,6 @@ func configure(ctx context.Context, t *testing.T, client proto.DRPCProvisionerCl
return sess
}

func hashTemplateFilesAndTestName(t *testing.T, testName string, templateFiles map[string]string) string {
t.Helper()

sortedFileNames := make([]string, 0, len(templateFiles))
for fileName := range templateFiles {
sortedFileNames = append(sortedFileNames, fileName)
}
sort.Strings(sortedFileNames)

// Inserting a delimiter between the file name and the file content
// ensures that a file named `ab` with content `cd`
// will not hash to the same value as a file named `abc` with content `d`.
// This can still happen if the file name or content include the delimiter,
// but hopefully they won't.
delimiter := []byte("🎉 🌱 🌷")

hasher := sha256.New()
for _, fileName := range sortedFileNames {
file := templateFiles[fileName]
_, err := hasher.Write([]byte(fileName))
require.NoError(t, err)
_, err = hasher.Write(delimiter)
require.NoError(t, err)
_, err = hasher.Write([]byte(file))
require.NoError(t, err)
}
_, err := hasher.Write(delimiter)
require.NoError(t, err)
_, err = hasher.Write([]byte(testName))
require.NoError(t, err)

return hex.EncodeToString(hasher.Sum(nil))
}

const (
terraformConfigFileName = "terraform.rc"
cacheProvidersDirName = "providers"
cacheTemplateFilesDirName = "files"
)

// Writes a Terraform CLI config file (`terraform.rc`) in `dir` to enforce using the local provider mirror.
// This blocks network access for providers, forcing Terraform to use only what's cached in `dir`.
// Returns the path to the generated config file.
func writeCliConfig(t *testing.T, dir string) string {
t.Helper()

cliConfigPath := filepath.Join(dir, terraformConfigFileName)
require.NoError(t, os.MkdirAll(filepath.Dir(cliConfigPath), 0o700))

content := fmt.Sprintf(`
provider_installation {
filesystem_mirror {
path = "%s"
include = ["*/*"]
}
direct {
exclude = ["*/*"]
}
}
`, filepath.Join(dir, cacheProvidersDirName))
require.NoError(t, os.WriteFile(cliConfigPath, []byte(content), 0o600))
return cliConfigPath
}

func runCmd(t *testing.T, dir string, args ...string) {
t.Helper()

stdout, stderr := bytes.NewBuffer(nil), bytes.NewBuffer(nil)
cmd := exec.Command(args[0], args[1:]...) //#nosec
cmd.Dir = dir
cmd.Stdout = stdout
cmd.Stderr = stderr
if err := cmd.Run(); err != nil {
t.Fatalf("failed to run %s: %s\nstdout: %s\nstderr: %s", strings.Join(args, " "), err, stdout.String(), stderr.String())
}
}

// Each test gets a unique cache dir based on its name and template files.
// This ensures that tests can download providers in parallel and that they
// will redownload providers if the template files change.
func getTestCacheDir(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string {
t.Helper()

hash := hashTemplateFilesAndTestName(t, testName, templateFiles)
dir := filepath.Join(rootDir, hash[:12])
return dir
}

// Ensures Terraform providers are downloaded and cached locally in a unique directory for the test.
// Uses `terraform init` then `mirror` to populate the cache if needed.
// Returns the cache directory path.
func downloadProviders(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string {
t.Helper()

dir := getTestCacheDir(t, rootDir, testName, templateFiles)
if _, err := os.Stat(dir); err == nil {
t.Logf("%s: using cached terraform providers", testName)
return dir
}
filesDir := filepath.Join(dir, cacheTemplateFilesDirName)
defer func() {
// The files dir will contain a copy of terraform providers generated
// by the terraform init command. We don't want to persist them since
// we already have a registry mirror in the providers dir.
if err := os.RemoveAll(filesDir); err != nil {
t.Logf("failed to remove files dir %s: %s", filesDir, err)
}
if !t.Failed() {
return
}
// If `downloadProviders` function failed, clean up the cache dir.
// We don't want to leave it around because it may be incomplete or corrupted.
if err := os.RemoveAll(dir); err != nil {
t.Logf("failed to remove dir %s: %s", dir, err)
}
}()

require.NoError(t, os.MkdirAll(filesDir, 0o700))

for fileName, file := range templateFiles {
filePath := filepath.Join(filesDir, fileName)
require.NoError(t, os.MkdirAll(filepath.Dir(filePath), 0o700))
require.NoError(t, os.WriteFile(filePath, []byte(file), 0o600))
}

providersDir := filepath.Join(dir, cacheProvidersDirName)
require.NoError(t, os.MkdirAll(providersDir, 0o700))

// We need to run init because if a test uses modules in its template,
// the mirror command will fail without it.
runCmd(t, filesDir, "terraform", "init")
// Now, mirror the providers into `providersDir`. We use this explicit mirror
// instead of relying only on the standard Terraform plugin cache.
//
// Why? Because this mirror, when used with the CLI config from `writeCliConfig`,
// prevents Terraform from hitting the network registry during `plan`. This cuts
// down on network calls, making CI tests less flaky.
//
// In contrast, the standard cache *still* contacts the registry for metadata
// during `init`, even if the plugins are already cached locally - see link below.
//
// Ref: https://developer.hashicorp.com/terraform/cli/config/config-file#provider-plugin-cache
// > When a plugin cache directory is enabled, the terraform init command will
// > still use the configured or implied installation methods to obtain metadata
// > about which plugins are available
runCmd(t, filesDir, "terraform", "providers", "mirror", providersDir)

return dir
}

// Caches providers locally and generates a Terraform CLI config to use *only* that cache.
// This setup prevents network access for providers during `terraform init`, improving reliability
// in subsequent test runs.
// Returns the path to the generated CLI config file.
func cacheProviders(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string {
t.Helper()

providersParentDir := downloadProviders(t, rootDir, testName, templateFiles)
cliConfigPath := writeCliConfig(t, providersParentDir)
return cliConfigPath
}

func readProvisionLog(t *testing.T, response proto.DRPCProvisioner_SessionClient) string {
var logBuf strings.Builder
for {
Expand Down Expand Up @@ -1177,7 +1011,7 @@ func TestProvision(t *testing.T) {
cacheRootDir := filepath.Join(testutil.PersistentCacheDir(t), "terraform_provision_test")
expectedCacheDirs := make(map[string]bool)
for _, testCase := range testCases {
cacheDir := getTestCacheDir(t, cacheRootDir, testCase.Name, testCase.Files)
cacheDir := testutil.GetTestTFCacheDir(t, cacheRootDir, testCase.Name, testCase.Files)
expectedCacheDirs[cacheDir] = true
}
currentCacheDirs, err := filepath.Glob(filepath.Join(cacheRootDir, "*"))
Expand All @@ -1199,7 +1033,7 @@ func TestProvision(t *testing.T) {

cliConfigPath := ""
if !testCase.SkipCacheProviders {
cliConfigPath = cacheProviders(
cliConfigPath = testutil.CacheTFProviders(
t,
cacheRootDir,
testCase.Name,
Expand Down
Loading
Loading