backend/local: Opt-in to experimental plan/apply/refresh functions

To facilitate early development and testing of the new language runtime
we're introducing a temporary mechanism to opt in to using the new codepaths
based on an environment variable. This environment variable is effective
only for experiment-enabled builds of OpenTofu, and so it will be
completely ignored by official releases of OpenTofu.

This commit just deals with the "wiring" of this new mechanism, without
actually connecting it with the new language runtime yet. The goal here
is to disturb existing codepaths as little as possible to minimize both
the risk of making this change and the burden this causes for ongoing
maintenance unrelated to work on the new language runtime.

This strategy of switching at the local backend layer means that we will
have some duplicated logic in the experimental functions compared to the
non-experimental functions, which is an intentional tradeoff to allow us
to isolate what we're doing so we don't churn existing code while we're
still in this early exploration phase. In a later phase of the language
runtime project we may pivot to a different approach which switches at
a deeper point in the call stack, but for now we're keeping this broad
to give us flexibility.

Signed-off-by: Martin Atkins <mart@degeneration.co.uk>

cmd/tofu/experiments.go

@@ -9,7 +9,7 @@ package main
 // arguments in order to enable the use of experimental features for a
 // particular OpenTofu build:
 //
-//	go install -ldflags="-X 'main.experimentsAllowed=yes'"
+//	go install -ldflags="-X 'main.experimentsAllowed=yes'" ./cmd/tofu
 //
 // By default this variable is initialized as empty, in which case
 // experimental features are not available.

internal/backend/local/backend_apply.go

@@ -50,6 +50,14 @@ func (b *Local) opApply(
 	cancelCtx context.Context,
 	op *backend.Operation,
 	runningOp *backend.RunningOperation) {
+
+	// TEMP: Opt-in support for testing with the new experimental language
+	// runtime. Refer to backend_temp_new_runtime.go for more information.
+	if experimentalRuntimeEnabled() {
+		b.opApplyWithExperimentalRuntime(stopCtx, cancelCtx, op, runningOp)
+		return
+	}
+
 	log.Printf("[INFO] backend/local: starting Apply operation")
 
 	var diags, moreDiags tfdiags.Diagnostics

internal/backend/local/backend_plan.go

@@ -28,6 +28,13 @@ func (b *Local) opPlan(
 	op *backend.Operation,
 	runningOp *backend.RunningOperation) {
 
+	// TEMP: Opt-in support for testing with the new experimental language
+	// runtime. Refer to backend_temp_new_runtime.go for more information.
+	if experimentalRuntimeEnabled() {
+		b.opPlanWithExperimentalRuntime(stopCtx, cancelCtx, op, runningOp)
+		return
+	}
+
 	log.Printf("[INFO] backend/local: starting Plan operation")
 
 	var diags tfdiags.Diagnostics

internal/backend/local/backend_refresh.go

@@ -24,6 +24,13 @@ func (b *Local) opRefresh(
 	op *backend.Operation,
 	runningOp *backend.RunningOperation) {
 
+	// TEMP: Opt-in support for testing with the new experimental language
+	// runtime. Refer to backend_temp_new_runtime.go for more information.
+	if experimentalRuntimeEnabled() {
+		b.opRefreshWithExperimentalRuntime(stopCtx, cancelCtx, op, runningOp)
+		return
+	}
+
 	var diags tfdiags.Diagnostics
 
 	// For the moment we have a bit of a tangled mess of context.Context here, for

internal/backend/local/backend_temp_new_runtime.go

@@ -0,0 +1,99 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package local
+
+import (
+	"context"
+	"log"
+	"os"
+	"sync/atomic"
+
+	"github.com/opentofu/opentofu/internal/backend"
+	"github.com/opentofu/opentofu/internal/tfdiags"
+)
+
+/////////////////////////
+// The definitions in this file are intended as temporary shims to help support
+// the development of the new runtime engine, by allowing experiments-enabled
+// builds to be opted in to the new implementation by setting the environment
+// variable TOFU_X_EXPERIMENTAL_RUNTIME to any non-empty value.
+//
+// These shims should remain here only as long as the new runtime engine is
+// under active development and is not yet adopted as the primary engine. It's
+// also acceptable for work being done for other separate projects to ignore
+// these shims and let this code become broken, as long as the code continues
+// to compile: only those working on the implementation of the new engine are
+// responsible for updating this if the rest of the system evolves to the point
+// of that being necessary.
+//
+// Note that "tofu validate" is implemented outside of the backend abstraction
+// and so does not respond to the experiment opt-in environment variable. For
+// now, try out validation-related behaviors of the new runtime through
+// "tofu plan" instead, which should implement a superset of the validation
+// behavior.
+/////////////////////////
+
+// SetExperimentalRuntimeAllowed must be called with the argument set to true
+// at some point before calling [New] or [NewWithBackend] in order for the
+// experimental opt-in to be effective.
+//
+// In practice this is called by code in the "command" package early in the
+// backend initialization codepath and enables the experimental runtime only
+// in an experiments-enabled OpenTofu build, to make sure that it's not
+// possible to accidentally enable this experimental functionality in normal
+// release builds.
+//
+// Refer to "cmd/tofu/experiments.go" for information on how to produce an
+// experiments-enabled build.
+func SetExperimentalRuntimeAllowed(allowed bool) {
+	experimentalRuntimeAllowed.Store(allowed)
+}
+
+var experimentalRuntimeAllowed atomic.Bool
+
+func experimentalRuntimeEnabled() bool {
+	if !experimentalRuntimeAllowed.Load() {
+		// The experimental runtime is never enabled when it hasn't been
+		// explicitly allowed.
+		return false
+	}
+
+	optIn := os.Getenv("TOFU_X_EXPERIMENTAL_RUNTIME")
+	return optIn != ""
+}
+
+func (b *Local) opPlanWithExperimentalRuntime(stopCtx context.Context, cancelCtx context.Context, op *backend.Operation, runningOp *backend.RunningOperation) {
+	log.Println("[WARN] Using plan implementation from the experimental language runtime")
+	var diags tfdiags.Diagnostics
+	diags = diags.Append(tfdiags.Sourceless(
+		tfdiags.Error,
+		"Operation unsupported in experimental language runtime",
+		"The command \"tofu plan\" is not yet supported under the experimental language runtime.",
+	))
+	op.ReportResult(runningOp, diags)
+}
+
+func (b *Local) opApplyWithExperimentalRuntime(stopCtx context.Context, cancelCtx context.Context, op *backend.Operation, runningOp *backend.RunningOperation) {
+	log.Println("[WARN] Using apply implementation from the experimental language runtime")
+	var diags tfdiags.Diagnostics
+	diags = diags.Append(tfdiags.Sourceless(
+		tfdiags.Error,
+		"Operation unsupported in experimental language runtime",
+		"The command \"tofu apply\" is not yet supported under the experimental language runtime.",
+	))
+	op.ReportResult(runningOp, diags)
+}
+
+func (b *Local) opRefreshWithExperimentalRuntime(stopCtx context.Context, cancelCtx context.Context, op *backend.Operation, runningOp *backend.RunningOperation) {
+	log.Println("[WARN] Using refresh implementation from the experimental language runtime")
+	var diags tfdiags.Diagnostics
+	diags = diags.Append(tfdiags.Sourceless(
+		tfdiags.Error,
+		"Operation unsupported in experimental language runtime",
+		"The command \"tofu refresh\" is not yet supported under the experimental language runtime.",
+	))
+	op.ReportResult(runningOp, diags)
+}

internal/command/meta_backend.go

@@ -103,6 +103,16 @@ func (m *Meta) Backend(ctx context.Context, opts *BackendOpts, enc encryption.St
 		opts = &BackendOpts{}
 	}
 
+	if m.AllowExperimentalFeatures {
+		// TEMP: While we're in early development of the new language runtime
+		// we have an experimental shim to enable it using an environment
+		// variable, but that's allowed only in builds where experimental
+		// features are enabled. Refer to the file containing the following
+		// function for more information. This should be completely removed
+		// once the experiment is concluded.
+		backendLocal.SetExperimentalRuntimeAllowed(true)
+	}
+
 	// Initialize a backend from the config unless we're forcing a purely
 	// local operation.
 	var b backend.Backend