opentf/internal/getproviders/registry_client.go

// Copyright (c) The OpenTofu Authors
// SPDX-License-Identifier: MPL-2.0
// Copyright (c) 2023 HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0

package getproviders

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"path"

	"github.com/hashicorp/go-retryablehttp"
	"github.com/opentofu/svchost"
	"github.com/opentofu/svchost/svcauth"

	"github.com/opentofu/opentofu/internal/addrs"
	"github.com/opentofu/opentofu/internal/tracing"
	"github.com/opentofu/opentofu/internal/tracing/traceattrs"
	"github.com/opentofu/opentofu/version"
)

const (
	terraformVersionHeader = "X-Terraform-Version"
)

var SupportedPluginProtocols = MustParseVersionConstraints(">= 5, <7")

// registryClient is a client for the provider registry protocol that is
// specialized only for the needs of this package. It's not intended as a
// general registry API client.
type registryClient struct {
	baseURL *url.URL
	creds   svcauth.HostCredentials

	httpClient *retryablehttp.Client

	locationConfig LocationConfig
}

func newRegistryClient(ctx context.Context, baseURL *url.URL, creds svcauth.HostCredentials, httpClient *retryablehttp.Client, locationConfig LocationConfig) *registryClient {
	return &registryClient{
		baseURL:        baseURL,
		creds:          creds,
		httpClient:     httpClient,
		locationConfig: locationConfig,
	}
}

// ProviderVersions returns the raw version and protocol strings produced by the
// registry for the given provider.
//
// The returned error will be ErrRegistryProviderNotKnown if the registry responds with
// 404 Not Found to indicate that the namespace or provider type are not known,
// ErrUnauthorized if the registry responds with 401 or 403 status codes, or
// ErrQueryFailed for any other protocol or operational problem.
func (c *registryClient) ProviderVersions(ctx context.Context, addr addrs.Provider) (map[string][]string, []string, error) {
	ctx, span := tracing.Tracer().Start(ctx,
		"List Versions",
		tracing.SpanAttributes(
			traceattrs.OpenTofuProviderAddress(addr.String()),
		),
	)
	defer span.End()
	endpointPath, err := url.Parse(path.Join(addr.Namespace, addr.Type, "versions"))
	if err != nil {
		// Should never happen because we're constructing this from
		// already-validated components.
		return nil, nil, err
	}
	endpointURL := c.baseURL.ResolveReference(endpointPath)
	span.SetAttributes(traceattrs.URLFull(endpointURL.String()))
	req, err := retryablehttp.NewRequest("GET", endpointURL.String(), nil)
	if err != nil {
		return nil, nil, err
	}
	req = req.WithContext(ctx)
	c.addHeadersToRequest(req.Request)

	resp, err := c.httpClient.Do(req)
	if err != nil {
		errResult := c.errQueryFailed(addr, err)
		tracing.SetSpanError(span, errResult)
		return nil, nil, errResult
	}
	defer resp.Body.Close()

	switch resp.StatusCode {
	case http.StatusOK:
		// Great!
	case http.StatusNotFound:
		err := ErrRegistryProviderNotKnown{
			Provider: addr,
		}
		tracing.SetSpanError(span, err)
		return nil, nil, err
	case http.StatusUnauthorized, http.StatusForbidden:
		err := c.errUnauthorized(addr.Hostname)
		tracing.SetSpanError(span, err)
		return nil, nil, err
	default:
		err := c.errQueryFailed(addr, errors.New(resp.Status))
		tracing.SetSpanError(span, err)
		return nil, nil, err
	}

	// We ignore the platforms portion of the response body, because the
	// installer verifies the platform compatibility after pulling a provider
	// versions' metadata.
	type ResponseBody struct {
		Versions []struct {
			Version   string   `json:"version"`
			Protocols []string `json:"protocols"`
		} `json:"versions"`
		Warnings []string `json:"warnings"`
	}
	var body ResponseBody

	dec := json.NewDecoder(resp.Body)
	if err := dec.Decode(&body); err != nil {
		errResult := c.errQueryFailed(addr, err)
		tracing.SetSpanError(span, errResult)
		return nil, nil, errResult
	}

	if len(body.Versions) == 0 {
		return nil, body.Warnings, nil
	}

	ret := make(map[string][]string, len(body.Versions))
	for _, v := range body.Versions {
		ret[v.Version] = v.Protocols
	}

	return ret, body.Warnings, nil
}

// PackageMeta returns metadata about a distribution package for a provider.
//
// The returned error will be one of the following:
//
//   - ErrPlatformNotSupported if the registry responds with 404 Not Found,
//     under the assumption that the caller previously checked that the provider
//     and version are valid.
//   - ErrProtocolNotSupported if the requested provider version's protocols are not
//     supported by this version of tofu.
//   - ErrUnauthorized if the registry responds with 401 or 403 status codes
//   - ErrQueryFailed for any other operational problem.
func (c *registryClient) PackageMeta(ctx context.Context, provider addrs.Provider, version Version, target Platform) (PackageMeta, error) {
	endpointPath, err := url.Parse(path.Join(
		provider.Namespace,
		provider.Type,
		version.String(),
		"download",
		target.OS,
		target.Arch,
	))
	ctx, span := tracing.Tracer().Start(ctx,
		"Fetch metadata",
		tracing.SpanAttributes(
			traceattrs.OpenTofuProviderAddress(provider.String()),
			traceattrs.OpenTofuProviderVersion(version.String()),
		))
	defer span.End()

	if err != nil {
		// Should never happen because we're constructing this from
		// already-validated components.
		return PackageMeta{}, err
	}
	endpointURL := c.baseURL.ResolveReference(endpointPath)
	span.SetAttributes(
		traceattrs.URLFull(endpointURL.String()),
	)

	req, err := retryablehttp.NewRequest("GET", endpointURL.String(), nil)
	if err != nil {
		return PackageMeta{}, err
	}
	req = req.WithContext(ctx)
	c.addHeadersToRequest(req.Request)

	resp, err := c.httpClient.Do(req)
	if err != nil {
		tracing.SetSpanError(span, err)
		return PackageMeta{}, c.errQueryFailed(provider, err)
	}
	defer resp.Body.Close()

	switch resp.StatusCode {
	case http.StatusOK:
		// Great!
	case http.StatusNotFound:
		return PackageMeta{}, ErrPlatformNotSupported{
			Provider: provider,
			Version:  version,
			Platform: target,
		}
	case http.StatusUnauthorized, http.StatusForbidden:
		return PackageMeta{}, c.errUnauthorized(provider.Hostname)
	default:
		return PackageMeta{}, c.errQueryFailed(provider, errors.New(resp.Status))
	}

	type SigningKeyList struct {
		GPGPublicKeys []*SigningKey `json:"gpg_public_keys"`
	}
	type ResponseBody struct {
		Protocols   []string `json:"protocols"`
		OS          string   `json:"os"`
		Arch        string   `json:"arch"`
		Filename    string   `json:"filename"`
		DownloadURL string   `json:"download_url"`
		SHA256Sum   string   `json:"shasum"`

		SHA256SumsURL          string `json:"shasums_url"`
		SHA256SumsSignatureURL string `json:"shasums_signature_url"`

		SigningKeys SigningKeyList `json:"signing_keys"`
	}
	var body ResponseBody

	dec := json.NewDecoder(resp.Body)
	if err := dec.Decode(&body); err != nil {
		return PackageMeta{}, c.errQueryFailed(provider, err)
	}

	var protoVersions VersionList
	for _, versionStr := range body.Protocols {
		v, err := ParseVersion(versionStr)
		if err != nil {
			return PackageMeta{}, c.errQueryFailed(
				provider,
				fmt.Errorf("registry response includes invalid version string %q: %w", versionStr, err),
			)
		}
		protoVersions = append(protoVersions, v)
	}
	protoVersions.Sort()

	// Verify that this version of tofu supports the providers' protocol
	// version(s)
	if len(protoVersions) > 0 {
		supportedProtos := MeetingConstraints(SupportedPluginProtocols)
		protoErr := ErrProtocolNotSupported{
			Provider: provider,
			Version:  version,
		}
		match := false
		for _, version := range protoVersions {
			if supportedProtos.Has(version) {
				match = true
			}
		}
		if !match {
			// If the protocol version is not supported, try to find the closest
			// matching version.
			closest, err := c.findClosestProtocolCompatibleVersion(ctx, provider, version)
			if err != nil {
				return PackageMeta{}, err
			}
			protoErr.Suggestion = closest
			return PackageMeta{}, protoErr
		}
	}

	if body.OS != target.OS || body.Arch != target.Arch {
		return PackageMeta{}, fmt.Errorf("registry response to request for %s archive has incorrect target %s", target, Platform{body.OS, body.Arch})
	}

	downloadURL, err := url.Parse(body.DownloadURL)
	if err != nil {
		return PackageMeta{}, fmt.Errorf("registry response includes invalid download URL: %w", err)
	}
	downloadURL = resp.Request.URL.ResolveReference(downloadURL)
	if downloadURL.Scheme != "http" && downloadURL.Scheme != "https" {
		return PackageMeta{}, fmt.Errorf("registry response includes invalid download URL: must use http or https scheme")
	}

	ret := PackageMeta{
		Provider:         provider,
		Version:          version,
		ProtocolVersions: protoVersions,
		TargetPlatform: Platform{
			OS:   body.OS,
			Arch: body.Arch,
		},
		Filename: body.Filename,
		Location: PackageHTTPURL{URL: downloadURL.String(), ClientBuilder: func(ctx context.Context) *retryablehttp.Client {
			return packageHTTPUrlClientWithRetry(ctx, c.locationConfig.ProviderDownloadRetries)
		}},
		// "Authentication" is populated below
	}

	if len(body.SHA256Sum) != sha256.Size*2 { // *2 because it's hex-encoded
		return PackageMeta{}, c.errQueryFailed(
			provider,
			fmt.Errorf("registry response includes invalid SHA256 hash %q: %w", body.SHA256Sum, err),
		)
	}

	var checksum [sha256.Size]byte
	_, err = hex.Decode(checksum[:], []byte(body.SHA256Sum))
	if err != nil {
		return PackageMeta{}, c.errQueryFailed(
			provider,
			fmt.Errorf("registry response includes invalid SHA256 hash %q: %w", body.SHA256Sum, err),
		)
	}

	shasumsURL, err := url.Parse(body.SHA256SumsURL)
	if err != nil {
		return PackageMeta{}, fmt.Errorf("registry response includes invalid SHASUMS URL: %w", err)
	}
	shasumsURL = resp.Request.URL.ResolveReference(shasumsURL)
	if shasumsURL.Scheme != "http" && shasumsURL.Scheme != "https" {
		return PackageMeta{}, fmt.Errorf("registry response includes invalid SHASUMS URL: must use http or https scheme")
	}
	document, err := c.getFile(ctx, shasumsURL)
	if err != nil {
		return PackageMeta{}, c.errQueryFailed(
			provider,
			fmt.Errorf("failed to retrieve authentication checksums for provider: %w", err),
		)
	}
	signatureURL, err := url.Parse(body.SHA256SumsSignatureURL)
	if err != nil {
		return PackageMeta{}, fmt.Errorf("registry response includes invalid SHASUMS signature URL: %w", err)
	}
	signatureURL = resp.Request.URL.ResolveReference(signatureURL)
	if signatureURL.Scheme != "http" && signatureURL.Scheme != "https" {
		return PackageMeta{}, fmt.Errorf("registry response includes invalid SHASUMS signature URL: must use http or https scheme")
	}
	signature, err := c.getFile(ctx, signatureURL)
	if err != nil {
		return PackageMeta{}, c.errQueryFailed(
			provider,
			fmt.Errorf("failed to retrieve cryptographic signature for provider: %w", err),
		)
	}

	keys := make([]SigningKey, len(body.SigningKeys.GPGPublicKeys))
	for i, key := range body.SigningKeys.GPGPublicKeys {
		keys[i] = *key
	}

	ret.Authentication = PackageAuthenticationAll(
		NewMatchingChecksumAuthentication(document, body.Filename, checksum),
		NewArchiveChecksumAuthentication(ret.TargetPlatform, checksum),
		NewSignatureAuthentication(ret, document, signature, keys, provider),
	)

	return ret, nil
}

// findClosestProtocolCompatibleVersion searches for the provider version with the closest protocol match.
func (c *registryClient) findClosestProtocolCompatibleVersion(ctx context.Context, provider addrs.Provider, version Version) (Version, error) {
	var match Version
	available, _, err := c.ProviderVersions(ctx, provider)
	if err != nil {
		return UnspecifiedVersion, err
	}

	// extract the maps keys so we can make a sorted list of available versions.
	versionList := make(VersionList, 0, len(available))
	for versionStr := range available {
		v, err := ParseVersion(versionStr)
		if err != nil {
			return UnspecifiedVersion, ErrQueryFailed{
				Provider: provider,
				Wrapped:  fmt.Errorf("registry response includes invalid version string %q: %w", versionStr, err),
			}
		}
		versionList = append(versionList, v)
	}
	versionList.Sort() // lowest precedence first, preserving order when equal precedence

	protoVersions := MeetingConstraints(SupportedPluginProtocols)
FindMatch:
	// put the versions in increasing order of precedence
	for index := len(versionList) - 1; index >= 0; index-- { // walk backwards to consider newer versions first
		for _, protoStr := range available[versionList[index].String()] {
			p, err := ParseVersion(protoStr)
			if err != nil {
				return UnspecifiedVersion, ErrQueryFailed{
					Provider: provider,
					Wrapped:  fmt.Errorf("registry response includes invalid protocol string %q: %w", protoStr, err),
				}
			}
			if protoVersions.Has(p) {
				match = versionList[index]
				break FindMatch
			}
		}
	}
	return match, nil
}

func (c *registryClient) addHeadersToRequest(req *http.Request) {
	if c.creds != nil {
		c.creds.PrepareRequest(req)
	}
	req.Header.Set(terraformVersionHeader, version.String())
}

func (c *registryClient) errQueryFailed(provider addrs.Provider, err error) error {
	if err == context.Canceled {
		// This one has a special error type so that callers can
		// handle it in a different way.
		return ErrRequestCanceled{}
	}
	return ErrQueryFailed{
		Provider: provider,
		Wrapped:  err,
	}
}

func (c *registryClient) errUnauthorized(hostname svchost.Hostname) error {
	return ErrUnauthorized{
		Hostname:        hostname,
		HaveCredentials: c.creds != nil,
	}
}

func (c *registryClient) getFile(ctx context.Context, url *url.URL) ([]byte, error) {
	req, err := retryablehttp.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, err
	}
	req = req.WithContext(ctx)

	resp, err := c.httpClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("%s returned from %s", resp.Status, HostFromRequest(resp.Request))
	}

	data, err := io.ReadAll(resp.Body)
	if err != nil {
		return data, err
	}

	return data, nil
}

// HostFromRequest extracts host the same way net/http Request.Write would,
// accounting for empty Request.Host
func HostFromRequest(req *http.Request) string {
	if req.Host != "" {
		return req.Host
	}
	if req.URL != nil {
		return req.URL.Host
	}

	// this should never happen and if it does
	// it will be handled as part of Request.Write()
	// https://cs.opensource.google/go/go/+/refs/tags/go1.18.4:src/net/http/request.go;l=574
	return ""
}