Extract TF_PROVIDER_DOWNLOAD_RETRY env var from the getproviders package (#3338)

Signed-off-by: Andrei Ciobanu <andrei.ciobanu@opentofu.org>

cmd/tofu/commands.go

@@ -122,6 +122,8 @@ func initCommands(
 		UnmanagedProviders:   unmanagedProviders,
 
 		AllowExperimentalFeatures: experimentsAreAllowed(),
+
+		ProviderSourceLocationConfig: providerSourceLocationConfig(),
 	}
 
 	// The command list is included in the tofu -help
@@ -362,9 +364,9 @@ func initCommands(
 			}, nil
 		},
 
-		//-----------------------------------------------------------
+		// -----------------------------------------------------------
 		// Plumbing
-		//-----------------------------------------------------------
+		// -----------------------------------------------------------
 
 		"force-unlock": func() (cli.Command, error) {
 			return &command.UnlockCommand{

cmd/tofu/provider_source.go

@@ -12,6 +12,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strconv"
 
 	"github.com/apparentlymart/go-userdirs/userdirs"
 	"github.com/opentofu/svchost/disco"
@@ -207,7 +208,7 @@ func implicitProviderSource(
 	// local copy will take precedence.
 	searchRules = append(searchRules, getproviders.MultiSourceSelector{
 		Source: getproviders.NewMemoizeSource(
-			getproviders.NewRegistrySource(ctx, services, newRegistryHTTPClient(ctx, registryClientConfig)),
+			getproviders.NewRegistrySource(ctx, services, newRegistryHTTPClient(ctx, registryClientConfig), providerSourceLocationConfig()),
 		),
 		Exclude: directExcluded,
 	})
@@ -224,7 +225,7 @@ func providerSourceForCLIConfigLocation(
 ) (getproviders.Source, tfdiags.Diagnostics) {
 	if loc == cliconfig.ProviderInstallationDirect {
 		return getproviders.NewMemoizeSource(
-			getproviders.NewRegistrySource(ctx, services, newRegistryHTTPClient(ctx, registryClientConfig)),
+			getproviders.NewRegistrySource(ctx, services, newRegistryHTTPClient(ctx, registryClientConfig), providerSourceLocationConfig()),
 		), nil
 	}
 
@@ -258,7 +259,7 @@ func providerSourceForCLIConfigLocation(
 		// this client is not suitable for the HTTP mirror source, so we
 		// don't use this client directly.
 		httpTimeout := newRegistryHTTPClient(ctx, registryClientConfig).HTTPClient.Timeout
-		return getproviders.NewHTTPMirrorSource(ctx, url, services.CredentialsSource(), httpTimeout), nil
+		return getproviders.NewHTTPMirrorSource(ctx, url, services.CredentialsSource(), httpTimeout, providerSourceLocationConfig()), nil
 
 	case cliconfig.ProviderInstallationOCIMirror:
 		mappingFunc := loc.RepositoryMapping
@@ -298,3 +299,34 @@ func providerDevOverrides(configs []*cliconfig.ProviderInstallation) map[addrs.P
 	// ignore any additional configurations in here.
 	return configs[0].DevOverrides
 }
+
+const (
+	// providerDownloadRetryCountEnvName is the environment variable name used to customize
+	// the HTTP retry count for module downloads.
+	providerDownloadRetryCountEnvName = "TF_PROVIDER_DOWNLOAD_RETRY"
+
+	providerDownloadDefaultRetry = 2
+)
+
+// providerDownloadRetry will attempt for requests with retryable errors, like 502 status codes
+func providerDownloadRetry() int {
+	res := providerDownloadDefaultRetry
+	if v := os.Getenv(providerDownloadRetryCountEnvName); v != "" {
+		retry, err := strconv.Atoi(v)
+		if err == nil && retry > 0 {
+			res = retry
+		}
+	}
+	return res
+}
+
+// providerSourceLocationConfig is meant to build a global configuration for the
+// remote locations to download a provider from. This is built out of the
+// TF_PROVIDER_DOWNLOAD_RETRY env variable and is meant to be passed through
+// [getproviders.Source] all the way down to the [getproviders.PackageLocation]
+// to be able to tweak the configurations of the http clients used there.
+func providerSourceLocationConfig() getproviders.LocationConfig {
+	return getproviders.LocationConfig{
+		ProviderDownloadRetries: providerDownloadRetry(),
+	}
+}

cmd/tofu/provider_source_test.go

@@ -12,9 +12,11 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/opentofu/opentofu/internal/addrs"
 	"github.com/opentofu/opentofu/internal/command/cliconfig"
 	"github.com/opentofu/opentofu/internal/command/cliconfig/ociauthconfig"
+	"github.com/opentofu/opentofu/internal/getproviders"
 	"github.com/opentofu/svchost/disco"
 )
 
@@ -157,3 +159,43 @@ func TestProviderSource(t *testing.T) {
 		})
 	}
 }
+
+func TestConfigureProviderDownloadRetry(t *testing.T) {
+	tests := []struct {
+		name    string
+		envVars map[string]string
+
+		expectedConfig getproviders.LocationConfig
+	}{
+		{
+			name: "when no TF_PROVIDER_DOWNLOAD_RETRY env var, default retry attempts used for provider download",
+			expectedConfig: getproviders.LocationConfig{
+				ProviderDownloadRetries: providerDownloadDefaultRetry,
+			},
+		},
+		{
+			name: "when TF_PROVIDER_DOWNLOAD_RETRY env var configured, it is used provider download",
+			envVars: map[string]string{
+				"TF_PROVIDER_DOWNLOAD_RETRY": "7",
+			},
+			expectedConfig: getproviders.LocationConfig{
+				ProviderDownloadRetries: 7,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			for k, v := range tt.envVars {
+				t.Setenv(k, v)
+			}
+
+			// Call the function under test
+			got := providerSourceLocationConfig()
+
+			if diff := cmp.Diff(tt.expectedConfig, got); diff != "" {
+				t.Fatalf("expected no diff. got:\n%s", diff)
+			}
+		})
+	}
+}