opentf/cmd/tofu/provider_source.go

// Copyright (c) The OpenTofu Authors
// SPDX-License-Identifier: MPL-2.0
// Copyright (c) 2023 HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0

package main

import (
	"context"
	"fmt"
	"log"
	"net/url"
	"os"
	"path/filepath"

	"github.com/apparentlymart/go-userdirs/userdirs"
	"github.com/opentofu/svchost/disco"

	"github.com/opentofu/opentofu/internal/addrs"
	"github.com/opentofu/opentofu/internal/command/cliconfig"
	"github.com/opentofu/opentofu/internal/getproviders"
	"github.com/opentofu/opentofu/internal/tfdiags"
)

// providerSource constructs a provider source based on a combination of the
// CLI configuration and some default search locations. This will be the
// provider source used for provider installation in the "tofu init"
// command, unless overridden by the special -plugin-dir option.
func providerSource(
	ctx context.Context,
	configs []*cliconfig.ProviderInstallation,
	registryClientConfig *cliconfig.RegistryProtocolsConfig,
	services *disco.Disco,
	getOCICredsPolicy ociCredsPolicyBuilder,
	originalWorkingDir string,
) (getproviders.Source, tfdiags.Diagnostics) {
	if len(configs) == 0 {
		// If there's no explicit installation configuration then we'll build
		// up an implicit one with direct registry installation along with
		// some automatically-selected local filesystem mirrors.
		return implicitProviderSource(ctx, registryClientConfig, services, originalWorkingDir), nil
	}

	// There should only be zero or one configurations, which is checked by
	// the validation logic in the cliconfig package. Therefore we'll just
	// ignore any additional configurations in here.
	config := configs[0]
	return explicitProviderSource(ctx, config, registryClientConfig, services, getOCICredsPolicy)
}

func explicitProviderSource(
	ctx context.Context,
	config *cliconfig.ProviderInstallation,
	registryClientConfig *cliconfig.RegistryProtocolsConfig,
	services *disco.Disco,
	getOCICredsPolicy ociCredsPolicyBuilder,
) (getproviders.Source, tfdiags.Diagnostics) {
	var diags tfdiags.Diagnostics
	var searchRules []getproviders.MultiSourceSelector

	log.Printf("[DEBUG] Explicit provider installation configuration is set")
	for _, methodConfig := range config.Methods {
		source, moreDiags := providerSourceForCLIConfigLocation(ctx, methodConfig.Location, methodConfig.Retries, registryClientConfig, services, getOCICredsPolicy)
		diags = diags.Append(moreDiags)
		if moreDiags.HasErrors() {
			continue
		}

		include, err := getproviders.ParseMultiSourceMatchingPatterns(methodConfig.Include)
		if err != nil {
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Invalid provider source inclusion patterns",
				fmt.Sprintf("CLI config specifies invalid provider inclusion patterns: %s.", err),
			))
			continue
		}
		exclude, err := getproviders.ParseMultiSourceMatchingPatterns(methodConfig.Exclude)
		if err != nil {
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Invalid provider source exclusion patterns",
				fmt.Sprintf("CLI config specifies invalid provider exclusion patterns: %s.", err),
			))
			continue
		}

		searchRules = append(searchRules, getproviders.MultiSourceSelector{
			Source:  source,
			Include: include,
			Exclude: exclude,
		})

		log.Printf("[TRACE] Selected provider installation method %#v with includes %s and excludes %s", methodConfig.Location, include, exclude)
	}

	return getproviders.MultiSource(searchRules), diags
}

// implicitProviderSource builds a default provider source to use if there's
// no explicit provider installation configuration in the CLI config.
//
// This implicit source looks in a number of local filesystem directories and
// directly in a provider's upstream registry. Any providers that have at least
// one version available in a local directory are implicitly excluded from
// direct installation, as if the user had listed them explicitly in the
// "exclude" argument in the direct provider source in the CLI config.
func implicitProviderSource(
	ctx context.Context,
	registryClientConfig *cliconfig.RegistryProtocolsConfig,
	services *disco.Disco,
	originalWorkingDir string,
) getproviders.Source {
	// The local search directories we use for implicit configuration are:
	// - The "terraform.d/plugins" directory in the current working directory,
	//   which we've historically documented as a place to put plugins as a
	//   way to include them in bundles uploaded to Terraform Cloud, where
	//   there has historically otherwise been no way to use custom providers.
	//   When using -chdir, this directory is checked in the launch directory
	//   (original working directory), not the directory specified with -chdir.
	// - The "plugins" subdirectory of the CLI config search directory.
	//   (that's ~/.terraform.d/plugins or $XDG_DATA_HOME/opentofu/plugins
	//   on Unix systems, equivalents elsewhere)
	// - The "plugins" subdirectory of any platform-specific search paths,
	//   following e.g. the XDG base directory specification on Unix systems,
	//   Apple's guidelines on OS X, and "known folders" on Windows.
	//
	// Any provider we find in one of those implicit directories will be
	// automatically excluded from direct installation from an upstream
	// registry. Anything not available locally will query its primary
	// upstream registry.
	var searchRules []getproviders.MultiSourceSelector

	// We'll track any providers we can find in the local search directories
	// along the way, and then exclude them from the registry source we'll
	// finally add at the end.
	foundLocally := map[addrs.Provider]struct{}{}

	addLocalDir := func(dir string) {
		// We'll make sure the directory actually exists before we add it,
		// because otherwise installation would always fail trying to look
		// in non-existent directories. (This is done here rather than in
		// the source itself because explicitly-selected directories via the
		// CLI config, once we have them, _should_ produce an error if they
		// don't exist to help users get their configurations right.)
		if info, err := os.Stat(dir); err == nil && info.IsDir() {
			log.Printf("[DEBUG] will search for provider plugins in %s", dir)
			fsSource := getproviders.NewFilesystemMirrorSource(ctx, dir)

			// We'll peep into the source to find out what providers it seems
			// to be providing, so that we can exclude those from direct
			// install. This might fail, in which case we'll just silently
			// ignore it and assume it would fail during installation later too
			// and therefore effectively doesn't provide _any_ packages.
			if available, err := fsSource.AllAvailablePackages(); err == nil {
				for found := range available {
					foundLocally[found] = struct{}{}
				}
			}

			searchRules = append(searchRules, getproviders.MultiSourceSelector{
				Source: fsSource,
			})

		} else {
			log.Printf("[DEBUG] ignoring non-existing provider search directory %s", dir)
		}
	}

	// Check and add the "terraform.d/plugins" directory in the original working directory
	addLocalDir(filepath.Join(originalWorkingDir, "terraform.d/plugins"))

	cliDataDirs, err := cliconfig.DataDirs()
	if err == nil {
		for _, cliDataDir := range cliDataDirs {
			addLocalDir(filepath.Join(cliDataDir, "plugins"))
		}
	}

	// This "userdirs" library implements an appropriate user-specific and
	// app-specific directory layout for the current platform, such as XDG Base
	// Directory on Unix, using the following name strings to construct a
	// suitable application-specific subdirectory name following the
	// conventions for each platform:
	//
	//   XDG (Unix): lowercase of the first string, "terraform"
	//   Windows:    two-level hierarchy of first two strings, "HashiCorp\Terraform"
	//   OS X:       reverse-DNS unique identifier, "io.terraform".
	sysSpecificDirs := userdirs.ForApp("Terraform", "HashiCorp", "io.terraform")
	for _, dir := range sysSpecificDirs.DataSearchPaths("plugins") {
		addLocalDir(dir)
	}

	// Anything we found in local directories above is excluded from being
	// looked up via the registry source we're about to construct.
	var directExcluded getproviders.MultiSourceMatchingPatterns
	for addr := range foundLocally {
		directExcluded = append(directExcluded, addr)
	}

	// Last but not least, the main registry source! We'll wrap a caching
	// layer around this one to help optimize the several network requests
	// we'll end up making to it while treating it as one of several sources
	// in a MultiSource (as recommended in the MultiSource docs).
	// This one is listed last so that if a particular version is available
	// both in one of the above directories _and_ in a remote registry, the
	// local copy will take precedence.
	searchRules = append(searchRules, getproviders.MultiSourceSelector{
		Source: getproviders.NewMemoizeSource(
			getproviders.NewRegistrySource(ctx, services, newRegistryHTTPClient(ctx, registryClientConfig), providerSourceLocationConfigFromEnv()),
		),
		Exclude: directExcluded,
	})

	return getproviders.MultiSource(searchRules)
}

func providerSourceForCLIConfigLocation(
	ctx context.Context,
	loc cliconfig.ProviderInstallationLocation,
	locationRetries cliconfig.ProviderInstallationMethodRetries,
	registryClientConfig *cliconfig.RegistryProtocolsConfig,
	services *disco.Disco,
	makeOCICredsPolicy ociCredsPolicyBuilder,
) (getproviders.Source, tfdiags.Diagnostics) {
	if loc == cliconfig.ProviderInstallationDirect {
		return getproviders.NewMemoizeSource(
			getproviders.NewRegistrySource(ctx, services, newRegistryHTTPClient(ctx, registryClientConfig), providerSourceLocationConfig(locationRetries)),
		), nil
	}

	switch loc := loc.(type) {

	case cliconfig.ProviderInstallationFilesystemMirror:
		return getproviders.NewFilesystemMirrorSource(ctx, string(loc)), nil

	case cliconfig.ProviderInstallationNetworkMirror:
		url, err := url.Parse(string(loc))
		if err != nil {
			var diags tfdiags.Diagnostics
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Invalid URL for provider installation source",
				fmt.Sprintf("Cannot parse %q as a URL for a network provider mirror: %s.", string(loc), err),
			))
			return nil, diags
		}
		if url.Scheme != "https" || url.Host == "" {
			var diags tfdiags.Diagnostics
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Invalid URL for provider installation source",
				fmt.Sprintf("Cannot use %q as a URL for a network provider mirror: the mirror must be at an https: URL.", string(loc)),
			))
			return nil, diags
		}
		// For historical reasons, we use the registry client timeout for this
		// even though this isn't actually a registry. The other behavior of
		// this client is not suitable for the HTTP mirror source, so we
		// don't use this client directly.
		httpTimeout := newRegistryHTTPClient(ctx, registryClientConfig).HTTPClient.Timeout
		return getproviders.NewHTTPMirrorSource(ctx, url, services.CredentialsSource(), httpTimeout, providerSourceLocationConfig(locationRetries)), nil

	case cliconfig.ProviderInstallationOCIMirror:
		mappingFunc := loc.RepositoryMapping
		return getproviders.NewOCIRegistryMirrorSource(
			ctx,
			mappingFunc,
			func(ctx context.Context, registryDomain, repositoryName string) (getproviders.OCIRepositoryStore, error) {
				// We intentionally delay the finalization of the credentials policy until
				// just before we need it because most OpenTofu commands don't install
				// providers at all, and even those that do only need to do this if
				// actually interacting with an OCI mirror, so we can avoid doing
				// this work at all most of the time.
				credsPolicy, err := makeOCICredsPolicy(ctx)
				if err != nil {
					// This deals with only a small number of errors that we can't catch during CLI config validation
					return nil, fmt.Errorf("invalid credentials configuration for OCI registries: %w", err)
				}
				return getOCIRepositoryStore(ctx, registryDomain, repositoryName, credsPolicy)
			},
		), nil

	default:
		// We should not get here because the set of cases above should
		// be comprehensive for all of the
		// cliconfig.ProviderInstallationLocation implementations.
		panic(fmt.Sprintf("unexpected provider source location type %T", loc))
	}
}

func providerDevOverrides(configs []*cliconfig.ProviderInstallation) map[addrs.Provider]getproviders.PackageLocalDir {
	if len(configs) == 0 {
		return nil
	}

	// There should only be zero or one configurations, which is checked by
	// the validation logic in the cliconfig package. Therefore we'll just
	// ignore any additional configurations in here.
	return configs[0].DevOverrides
}

// providerSourceLocationConfig is meant to build a global configuration for the
// remote locations to download a provider from. This is built out of the
// TF_PROVIDER_DOWNLOAD_RETRY env variable and is meant to be passed through
// [getproviders.Source] all the way down to the [getproviders.PackageLocation]
// to be able to tweak the configurations of the http clients used there.
func providerSourceLocationConfig(locationRetries cliconfig.ProviderInstallationMethodRetries) getproviders.LocationConfig {
	// If there is no configuration for the retries in .tofurc, get the one from env variable
	retries, configured := locationRetries()
	if !configured {
		retries = cliconfig.ProviderDownloadRetries()
	}
	return getproviders.LocationConfig{
		ProviderDownloadRetries: retries,
	}
}

// providerSourceLocationConfigFromEnv is similar to providerSourceLocationConfig but does not
// take into account the information from the configuration. This is like so because for some
// commands, there is no specific tofurc configuration for the retry, so we want to use the
// env variable if defined and if not, its default.
func providerSourceLocationConfigFromEnv() getproviders.LocationConfig {
	return getproviders.LocationConfig{
		ProviderDownloadRetries: cliconfig.ProviderDownloadRetries(),
	}
}