workdir: Start of a new package for working directory state management

Thus far our various interactions with the bits of state we keep
associated with a working directory have all been implemented directly
inside the "command" package -- often in the huge command.Meta type -- and
not managed collectively via a single component.

There's too many little codepaths reading and writing from the working
directory and data directory to refactor it all in one step, but this is
an attempt at a first step towards a future where everything that reads
and writes from the current working directory would do so via an object
that encapsulates the implementation details and offers a high-level API
to read and write all of these session-persistent settings.

The design here continues our gradual path towards using a dependency
injection style where "package main" is solely responsible for directly
interacting with the OS command line, the OS environment, the OS working
directory, the stdio streams, and the CLI configuration, and then
communicating the resulting information to the rest of Terraform by wiring
together objects. It seems likely that eventually we'll have enough wiring
code in package main to justify a more explicit organization of that code,
but for this commit the new "workdir.Dir" object is just wired directly in
place of its predecessors, without any significant change of code
organization at that top layer.

This first commit focuses on the main files and directories we use to
find provider plugins, because a subsequent commit will lightly reorganize
the separation of concerns for plugin launching with a similar goal of
collecting all of the relevant logic together into one spot.

internal/command/workdir/dir.go

@@ -0,0 +1,149 @@
+package workdir
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// Dir represents a single Terraform working directory.
+//
+// "Working directory" is unfortunately a slight misnomer, because non-default
+// options can potentially stretch the definition such that multiple working
+// directories end up appearing to share a data directory, or other similar
+// anomolies, but we continue to use this terminology both for historical
+// reasons and because it reflects the common case without any special
+// overrides.
+//
+// The naming convention for methods on this type is that methods whose names
+// begin with "Override" affect only characteristics of the particular object
+// they're called on, changing where it looks for data, while methods whose
+// names begin with "Set" will write settings to disk such that other instances
+// referring to the same directories will also see them. Given that, the
+// "Override" methods should be used only during the initialization steps
+// for a Dir object, typically only inside "package main", so that all
+// subsequent work elsewhere will access consistent locations on disk.
+//
+// We're gradually transitioning to using this type to manage working directory
+// settings, and so not everything in the working directory "data dir" is
+// encapsulated here yet, but hopefully we'll gradually migrate all of those
+// settings here over time. The working directory state not yet managed in here
+// is typically managed directly in the "command" package, either directly
+// inside commands or in methods of the giant command.Meta type.
+type Dir struct {
+	// mainDir is the path to the directory that we present as the
+	// "working directory" in the user model, which is typically the
+	// current working directory when running Terraform CLI, or the
+	// directory explicitly chosen by the user using the -chdir=...
+	// global option.
+	mainDir string
+
+	// originalDir is the path to the working directory that was
+	// selected when creating the Terraform CLI process, regardless of
+	// -chdir=... being set. This is only for very limited purposes
+	// related to backward compatibility; most functionality should
+	// use mainDir instead.
+	originalDir string
+
+	// dataDir is the path to the directory where we will store our
+	// working directory settings and artifacts. This is typically a
+	// directory named ".terraform" within mainDir, but users may
+	// override it.
+	dataDir string
+}
+
+// NewDir constructs a new working directory, anchored at the given path.
+//
+// In normal use, mainPath should be "." to reflect the current working
+// directory, with "package main" having switched the process's current
+// working directory if necessary prior to calling this function. However,
+// unusual situations in tests may set mainPath to a temporary directory, or
+// similar.
+//
+// WARNING: Although the logic in this package is intended to work regardless
+// of whether mainPath is actually the current working directory, we're
+// currently in a transitional state where this package shares responsibility
+// for the working directory with various command.Meta methods, and those
+// often assume that the main path of the working directory will always be
+// ".". If you're writing test code that spans across both areas of
+// responsibility then you must ensure that the test temporarily changes the
+// test process's working directory to the directory returned by RootModuleDir
+// before using the result inside a command.Meta.
+func NewDir(mainPath string) *Dir {
+	mainPath = filepath.Clean(mainPath)
+	return &Dir{
+		mainDir:     mainPath,
+		originalDir: mainPath,
+		dataDir:     filepath.Join(mainPath, ".terraform"),
+	}
+}
+
+// OverrideOriginalWorkingDir records a different path as the
+// "original working directory" for the reciever.
+//
+// Use this only to record the original working directory when Terraform is run
+// with the -chdir=... global option. In that case, the directory given in
+// -chdir=... is the "main path" to pass in to NewDir, while the original
+// working directory should be sent to this method.
+func (d *Dir) OverrideOriginalWorkingDir(originalPath string) {
+	d.originalDir = filepath.Clean(originalPath)
+}
+
+// OverrideDataDir chooses a specific alternative directory to read and write
+// the persistent working directory settings.
+//
+// "package main" can call this if it detects that the user has overridden
+// the default location by setting the relevant environment variable. Don't
+// call this when that environment variable isn't set, in order to preserve
+// the default setting of a dot-prefixed directory directly inside the main
+// working directory.
+func (d *Dir) OverrideDataDir(dataDir string) {
+	d.dataDir = filepath.Clean(dataDir)
+}
+
+// RootModuleDir returns the directory where we expect to find the root module
+// configuration for this working directory.
+func (d *Dir) RootModuleDir() string {
+	// The root module configuration is just directly inside the main directory.
+	return d.mainDir
+}
+
+// OriginalWorkingDir returns the true, operating-system-originated working
+// directory that the current Terraform process was launched from.
+//
+// This is usually the same as the main working directory, but differs in the
+// special case where the user ran Terraform with the global -chdir=...
+// option. This is here only for a few backward compatibility affordances
+// from before we had the -chdir=... option, so should typically not be used
+// for anything new.
+func (d *Dir) OriginalWorkingDir() string {
+	return d.originalDir
+}
+
+// DataDir returns the base path where the reciever keeps all of the settings
+// and artifacts that must persist between consecutive commands in a single
+// session.
+//
+// This is exported only to allow the legacy behaviors in command.Meta to
+// continue accessing this directory directly. Over time we should replace
+// all of those direct accesses with methods on this type, and then remove
+// this method. Avoid using this method for new use-cases.
+func (d *Dir) DataDir() string {
+	return d.dataDir
+}
+
+// ensureDataDir creates the data directory and all of the necessary parent
+// directories that lead to it, if they don't already exist.
+//
+// For directories that already exist ensureDataDir will preserve their
+// permissions, while it'll create any new directories to be owned by the user
+// running Terraform, readable and writable by that user, and readable by
+// all other users, or some approximation of that on non-Unix platforms which
+// have a different permissions model.
+func (d *Dir) ensureDataDir() error {
+	err := os.MkdirAll(d.dataDir, 0755)
+	if err != nil {
+		return fmt.Errorf("failed to prepare working directory: %w", err)
+	}
+	return nil
+}

internal/command/workdir/doc.go

@@ -0,0 +1,16 @@
+// Package workdir models the various local artifacts and state we keep inside
+// a Terraform "working directory".
+//
+// The working directory artifacts and settings are typically initialized or
+// modified by "terraform init", after which they persist for use by other
+// commands in the same directory, but are not visible to commands run in
+// other working directories or on other computers.
+//
+// Although "terraform init" is the main command which modifies a workdir,
+// other commands do sometimes make more focused modifications for settings
+// which can typically change multiple times during a session, such as the
+// currently-selected workspace name. Any command which modifies the working
+// directory settings must discard and reload any objects which derived from
+// those settings, because otherwise the existing objects will often continue
+// to follow the settings that were present when they were created.
+package workdir

internal/command/workdir/normalize_path.go

@@ -0,0 +1,52 @@
+package workdir
+
+import (
+	"path/filepath"
+)
+
+// NormalizePath attempts to transform the given path so that it's relative
+// to the working directory, which is our preferred way to present and store
+// paths to files and directories within a configuration so that they can
+// be portable to operations in other working directories.
+//
+// It isn't always possible to produce a relative path. For example, on Windows
+// the given path might be on a different volume (e.g. drive letter or network
+// share) than the working directory.
+//
+// Note that the result will be relative to the main directory of the receiver,
+// which should always be the actual process working directory in normal code,
+// but might be some other temporary working directory when in test code.
+// If you need to access the file or directory that the result refers to with
+// functions that aren't aware of our base directory, you can use something
+// like the following, which again should be needed only in test code which
+// might need to inspect the filesystem in order to make assertions:
+//
+//     filepath.Join(d.RootModuleDir(), normalizePathResult)
+//
+// The above is suitable only for situations where the given path is known
+// to be beneath the working directory, which is the typical situation for
+// temporary working directories created for automated tests.
+func (d *Dir) NormalizePath(given string) string {
+	// We need an absolute version of d.mainDir in order for our "Rel"
+	// result to be reliable.
+	absMain, err := filepath.Abs(d.mainDir)
+	if err != nil {
+		// Weird, but okay...
+		return filepath.Clean(given)
+	}
+
+	if !filepath.IsAbs(given) {
+		given = filepath.Join(absMain, given)
+	}
+
+	ret, err := filepath.Rel(absMain, given)
+	if err != nil {
+		// It's not always possible to find a relative path. For example,
+		// the given path might be on an entirely separate volume
+		// (e.g. drive letter or network share) on a Windows system, which
+		// always requires an absolute path.
+		return filepath.Clean(given)
+	}
+
+	return ret
+}

internal/command/workdir/plugin_dirs.go

@@ -0,0 +1,83 @@
+package workdir
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+const PluginPathFilename = "plugin_path"
+
+// ProviderLocalCacheDir returns the directory we'll use as the
+// working-directory-specific local cache of providers.
+//
+// The provider installer's job is to make sure that all providers needed for
+// a particular working directory are available in this cache directory. No
+// other component may write here, and in particular a Dir object itself
+// never reads or writes into this directory, instead just delegating all of
+// that responsibility to other components.
+//
+// Typically, the caller will ultimately pass the result of this method either
+// directly or indirectly into providercache.NewDir, to get an object
+// responsible for managing the contents.
+func (d *Dir) ProviderLocalCacheDir() string {
+	return filepath.Join(d.dataDir, "providers")
+}
+
+// ForcedPluginDirs returns a list of directories to use to find plugins,
+// instead of the default locations.
+//
+// Returns an zero-length list and no error in the normal case where there
+// are no overridden search directories. If ForcedPluginDirs returns a
+// non-empty list with no errors then the result totally replaces the default
+// search directories.
+func (d *Dir) ForcedPluginDirs() ([]string, error) {
+	raw, err := ioutil.ReadFile(filepath.Join(d.dataDir, PluginPathFilename))
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	var pluginPath []string
+	if err := json.Unmarshal(raw, &pluginPath); err != nil {
+		return nil, err
+	}
+	return pluginPath, nil
+}
+
+// SetForcedPluginDirs records an overridden list of directories to search
+// to find plugins, instead of the default locations. See ForcePluginDirs
+// for more information.
+//
+// Pass a zero-length list to deactivate forced plugin directories altogether,
+// thus allowing the working directory to return to using the default
+// search directories.
+func (d *Dir) SetForcedPluginDirs(dirs []string) error {
+
+	filePath := filepath.Join(d.dataDir, PluginPathFilename)
+	switch {
+	case len(dirs) == 0:
+		err := os.Remove(filePath)
+		if !os.IsNotExist(err) {
+			return err
+		}
+		return nil
+	default:
+		// We'll ignore errors from this one, because if we fail to create
+		// the directory then we'll fail to create the file below too,
+		// and that subsequent error will more directly reflect what we
+		// are trying to do here.
+		d.ensureDataDir()
+
+		raw, err := json.MarshalIndent(dirs, "", "  ")
+		if err != nil {
+			return err
+		}
+
+		return ioutil.WriteFile(filePath, raw, 0644)
+	}
+}

internal/command/workdir/plugin_dirs_test.go

@@ -0,0 +1,60 @@
+package workdir
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestDirForcedPluginDirs(t *testing.T) {
+	tmpDir, err := os.MkdirTemp("", "terraform-workdir-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	dir := NewDir(tmpDir)
+	// We'll use the default convention of a data dir nested inside the
+	// working directory, so we don't need to override anything on "dir".
+
+	want := []string(nil)
+	got, err := dir.ForcedPluginDirs()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("wrong initial settings\n%s", diff)
+	}
+
+	fakeDir1 := filepath.Join(tmpDir, "boop1")
+	fakeDir2 := filepath.Join(tmpDir, "boop2")
+	err = dir.SetForcedPluginDirs([]string{fakeDir1, fakeDir2})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want = []string{fakeDir1, fakeDir2}
+	got, err = dir.ForcedPluginDirs()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("wrong updated settings\n%s", diff)
+	}
+
+	err = dir.SetForcedPluginDirs(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want = nil
+	got, err = dir.ForcedPluginDirs()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("wrong final settings, after reverting back to defaults\n%s", diff)
+	}
+}