jprdonnelly/opentf
1
0
Fork 0
mirror of https://github.com/opentffoundation/opentf.git synced 2025-12-25 01:00:16 -05:00
Code Issues Packages Projects Releases Wiki Activity

registry+getproviders: Registry client policy centralized in main

Browse Source 
The primary reason for this change is that registry.NewClient was
originally imposing its own decision about service discovery request
policy on every other user of the shared disco.Disco object by modifying
it directly.

We have been moving towards using a dependency inversion style where
package main is responsible for deciding how everything should be
configured based on global CLI arguments, environment variables, and the
CLI configuration, and so this commit moves to using that model for the
HTTP clients used by the module and provider registry client code.

This also makes explicit what was previously hidden away: that all service
discovery requests are made using the same HTTP client policy as for
requests to module registries, even if the service being discovered is not
a registry. This doesn't seem to have been the intention of the code as
previously written, but was still its ultimate effect: there is only one
disco.Disco object shared across all discovery callers and so changing its
configuration in any way changes it for everyone.

This initial rework is certainly not perfect: these components were not
originally designed to work in this way and there are lots of existing
test cases relying on them working the old way, and so this is a compromise
to get the behavior we now need (using consistent HTTP client settings
across all callers) without disrupting too much existing code.

Signed-off-by: Martin Atkins <mart@degeneration.co.uk>
This commit is contained in:
Martin Atkins
2025-05-09 11:39:48 -07:00
parent 3334ed5e1c
commit 65a0f7a656
19 changed files with 282 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
internal/registry/client.go
View File

@@ -13,9 +13,7 @@ import (
"log"
"net/http"
"net/url"
"os"
"path"
"strconv"
"strings"
"time"
@@ -26,7 +24,6 @@ import (
"go.opentelemetry.io/otel/trace"
"github.com/opentofu/opentofu/internal/httpclient"
"github.com/opentofu/opentofu/internal/logging"
"github.com/opentofu/opentofu/internal/registry/regsrc"
"github.com/opentofu/opentofu/internal/registry/response"
"github.com/opentofu/opentofu/internal/tracing"
@@ -39,35 +36,12 @@ const (
xTerraformVersion = "X-Terraform-Version"
modulesServiceID = "modules.v1"
providersServiceID = "providers.v1"
// registryDiscoveryRetryEnvName is the name of the environment variable that
// can be configured to customize number of retries for module and provider
// discovery requests with the remote registry.
registryDiscoveryRetryEnvName = "TF_REGISTRY_DISCOVERY_RETRY"
defaultRetry = 1
// registryClientTimeoutEnvName is the name of the environment variable that
// can be configured to customize the timeout duration (seconds) for module
// and provider discovery with the remote registry.
registryClientTimeoutEnvName = "TF_REGISTRY_CLIENT_TIMEOUT"
// defaultRequestTimeout is the default timeout duration for requests to the
// remote registry.
defaultRequestTimeout = 10 * time.Second
)
var (
tfVersion = version.String()
discoveryRetry int
requestTimeout time.Duration
)
func init() {
configureDiscoveryRetry()
configureRequestTimeout()
}
// Client provides methods to query OpenTofu Registries.
type Client struct {
// this is the client to be used for all requests.
@@ -79,30 +53,19 @@ type Client struct {
}
// NewClient returns a new initialized registry client.
func NewClient(ctx context.Context, services *disco.Disco, client *http.Client) *Client {
func NewClient(ctx context.Context, services *disco.Disco, client *retryablehttp.Client) *Client {
if services == nil {
services = disco.New()
}
if client == nil {
client = httpclient.New(ctx)
client.Timeout = requestTimeout
// The following is a fallback client configuration intended primarily
// for our test cases that directly call this function.
client = httpclient.NewForRegistryRequests(ctx, 1, 10*time.Second)
}
retryableClient := retryablehttp.NewClient()
retryableClient.HTTPClient = client
retryableClient.RetryMax = discoveryRetry
retryableClient.RequestLogHook = requestLogHook
retryableClient.ErrorHandler = maxRetryErrorHandler
logOutput := logging.LogOutput()
retryableClient.Logger = log.New(logOutput, "", log.Flags())
services.Transport = retryableClient.HTTPClient.Transport
services.SetUserAgent(httpclient.OpenTofuUserAgent(version.String()))
return &Client{
client: retryableClient,
client: client,
services: services,
}
}
@@ -310,60 +273,3 @@ func (c *Client) ModuleLocation(ctx context.Context, module *regsrc.Module, vers
return location, nil
}
// configureDiscoveryRetry configures the number of retries the registry client
// will attempt for requests with retryable errors, like 502 status codes
func configureDiscoveryRetry() {
discoveryRetry = defaultRetry
if v := os.Getenv(registryDiscoveryRetryEnvName); v != "" {
retry, err := strconv.Atoi(v)
if err == nil && retry > 0 {
discoveryRetry = retry
}
}
}
func requestLogHook(logger retryablehttp.Logger, req *http.Request, i int) {
if i > 0 {
logger.Printf("[INFO] Previous request to the remote registry failed, attempting retry.")
}
}
func maxRetryErrorHandler(resp *http.Response, err error, numTries int) (*http.Response, error) {
// Close the body per library instructions
if resp != nil {
resp.Body.Close()
}
// Additional error detail: if we have a response, use the status code;
// if we have an error, use that; otherwise nothing. We will never have
// both response and error.
var errMsg string
if resp != nil {
errMsg = fmt.Sprintf(": %s returned from %s", resp.Status, resp.Request.URL)
} else if err != nil {
errMsg = fmt.Sprintf(": %s", err)
}
// This function is always called with numTries=RetryMax+1. If we made any
// retry attempts, include that in the error message.
if numTries > 1 {
return resp, fmt.Errorf("the request failed after %d attempts, please try again later%s",
numTries, errMsg)
}
return resp, fmt.Errorf("the request failed, please try again later%s", errMsg)
}
// configureRequestTimeout configures the registry client request timeout from
// environment variables
func configureRequestTimeout() {
requestTimeout = defaultRequestTimeout
if v := os.Getenv(registryClientTimeoutEnvName); v != "" {
timeout, err := strconv.Atoi(v)
if err == nil && timeout > 0 {
requestTimeout = time.Duration(timeout) * time.Second
}
}
}