execgraph: Initial work on apply-time execution graph

So far this is mainly just the mechanism for building a graph piecemeal
from multiple callers working together as part of the planning engine.

The end goal is for it to be possible to "compile" an assembled graph into
something that can then be executed, and to be able to marshal/unmarshal an
uncompiled graph to save as part of a plan file, but those other
capabilities will follow in later commits.

Signed-off-by: Martin Atkins <mart@degeneration.co.uk>

internal/engine/internal/execgraph/builder.go

@@ -0,0 +1,355 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package execgraph
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/zclconf/go-cty/cty"
+
+	"github.com/opentofu/opentofu/internal/addrs"
+	"github.com/opentofu/opentofu/internal/lang/eval"
+	"github.com/opentofu/opentofu/internal/providers"
+	"github.com/opentofu/opentofu/internal/states"
+)
+
+// Builder is a helper for multiple codepaths to collaborate to build an
+// execution graph.
+//
+// The methods of this type each cause something to be added to the graph
+// and then return an opaque reference to what was added which can then be
+// used as an argument to another method. The opaque reference values are
+// specific to the builder that returned them; using a reference returned by
+// some other builder will at best cause a nonsense graph and at worst could
+// cause panics.
+type Builder struct {
+	// must hold mu when accessing any part of any other fields
+	mu sync.Mutex
+
+	graph *Graph
+
+	// During construction we treat certain items as singletons so that
+	// we can do the associated work only once while providing it to
+	// multiple callers, and so these maps track those singletons but
+	// we throw these away after building is complete because the graph
+	// becomes immutable at that point.
+	desiredStateRefs       addrs.Map[addrs.AbsResourceInstance, ResultRef[*eval.DesiredResourceInstance]]
+	priorStateRefs         addrs.Map[addrs.AbsResourceInstance, ResultRef[*states.ResourceInstanceObject]]
+	providerAddrRefs       map[addrs.Provider]ResultRef[addrs.Provider]
+	providerInstConfigRefs addrs.Map[addrs.AbsProviderInstanceCorrect, ResultRef[cty.Value]]
+	openProviderRefs       addrs.Map[addrs.AbsProviderInstanceCorrect, resultWithCloseBlockers[providers.Configured]]
+}
+
+func NewBuilder() *Builder {
+	return &Builder{
+		graph: &Graph{
+			resourceInstanceResults: addrs.MakeMap[addrs.AbsResourceInstance, ResultRef[*states.ResourceInstanceObject]](),
+		},
+		desiredStateRefs:       addrs.MakeMap[addrs.AbsResourceInstance, ResultRef[*eval.DesiredResourceInstance]](),
+		priorStateRefs:         addrs.MakeMap[addrs.AbsResourceInstance, ResultRef[*states.ResourceInstanceObject]](),
+		providerAddrRefs:       make(map[addrs.Provider]ResultRef[addrs.Provider]),
+		providerInstConfigRefs: addrs.MakeMap[addrs.AbsProviderInstanceCorrect, ResultRef[cty.Value]](),
+		openProviderRefs:       addrs.MakeMap[addrs.AbsProviderInstanceCorrect, resultWithCloseBlockers[providers.Configured]](),
+	}
+}
+
+// Finish returns the graph that has been built, which is then immutable.
+//
+// After calling this function the Builder is invalid and must not be used
+// anymore.
+func (b *Builder) Finish() *Graph {
+	b.mu.Lock()
+	ret := b.graph
+	b.graph = nil
+	b.mu.Unlock()
+	return ret
+}
+
+// ConstantValue adds a constant [cty.Value] as a source node. The result
+// can be used as an operand to a subsequent operation.
+//
+// Each call to this method adds a new constant value to the graph, even if
+// a previously-registered value was equal to the given value.
+func (b *Builder) ConstantValue(v cty.Value) ResultRef[cty.Value] {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	idx := appendIndex(&b.graph.constantVals, v)
+	return valueResultRef{idx}
+}
+
+// ConstantValue adds a constant [addrs.Provider] address as a source node.
+// The result can be used as an operand to a subsequent operation.
+//
+// Multiple calls with the same provider address all return the same result,
+// so in practice each distinct provider address is stored only once.
+func (b *Builder) ConstantProviderAddr(addr addrs.Provider) ResultRef[addrs.Provider] {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	if existing, ok := b.providerAddrRefs[addr]; ok {
+		return existing
+	}
+	idx := appendIndex(&b.graph.providerAddrs, addr)
+	return providerAddrResultRef{idx}
+}
+
+func (b *Builder) DesiredResourceInstance(addr addrs.AbsResourceInstance) ResultRef[*eval.DesiredResourceInstance] {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	// We only register one index for each distinct resource instance address.
+	if existing, ok := b.desiredStateRefs.GetOk(addr); ok {
+		return existing
+	}
+	idx := appendIndex(&b.graph.desiredStateRefs, addr)
+	ret := desiredResourceInstanceResultRef{idx}
+	b.desiredStateRefs.Put(addr, ret)
+	return ret
+}
+
+// ResourceInstancePriorState returns a source node whose result will be
+// the prior state resource instance object for the "current" (i.e. not deposed)
+// object associated given resource instance address, if any.
+//
+// NOTE: This is currently using states.ResourceInstanceObject from our existing
+// state model, but a real implementation of this might benefit from a slightly
+// different model tailored to be used in isolation, without the rest of the
+// state tree it came from.
+func (b *Builder) ResourceInstancePriorState(addr addrs.AbsResourceInstance) ResultRef[*states.ResourceInstanceObject] {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	// We only register one index for each distinct resource instance address.
+	if existing, ok := b.priorStateRefs.GetOk(addr); ok {
+		return existing
+	}
+	idx := appendIndex(&b.graph.priorStateRefs, resourceInstanceStateRef{
+		ResourceInstance: addr,
+		DeposedKey:       states.NotDeposed,
+	})
+	ret := resourceInstancePriorStateResultRef{idx}
+	b.priorStateRefs.Put(addr, ret)
+	return ret
+}
+
+// ResourceDeposedObjectState is like [Builder.ResourceInstancePriorState] but
+// produces the state for a deposed object currently associated with a resource
+// instance, rather than its "current" object.
+//
+// Unlike [Builder.ResourceInstancePriorState] this registers an entirely new
+// result for each call, with the expectation that there will only be one
+// codepath attempting to register the chain of nodes for any deposed object,
+// and no resource instance should depend on the result of applying changes
+// to a deposed object.
+func (b *Builder) ResourceDeposedObjectState(instAddr addrs.AbsResourceInstance, deposedKey states.DeposedKey) ResultRef[*states.ResourceInstanceObject] {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	idx := appendIndex(&b.graph.priorStateRefs, resourceInstanceStateRef{
+		ResourceInstance: instAddr,
+		DeposedKey:       deposedKey,
+	})
+	ret := resourceInstancePriorStateResultRef{idx}
+	return ret
+}
+
+// ProviderInstanceConfig registers a request to obtain the configuration for
+// a specific provider instance, returning a reference to its [cty.Value]
+// result representing the evaluated configuration.
+//
+// In most cases callers should use [Builder.ProviderInstance] to obtain a
+// preconfigured client for the provider instance, which deals with getting
+// the provider instance configuration as part of its work.
+func (b *Builder) ProviderInstanceConfig(addr addrs.AbsProviderInstanceCorrect) ResultRef[cty.Value] {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	// We only register one index for each distinct provider instance address.
+	if existing, ok := b.providerInstConfigRefs.GetOk(addr); ok {
+		return existing
+	}
+	idx := appendIndex(&b.graph.providerInstConfigRefs, addr)
+	ret := providerInstanceConfigResultRef{idx}
+	b.providerInstConfigRefs.Put(addr, ret)
+	return ret
+}
+
+// ProviderInstance encapsulates everything required to obtain a configured
+// client for a provider instance and ensure that the client stays open long
+// enough to handle one or more other operations registered afterwards.
+//
+// The returned [RegisterCloseBlockerFunc] MUST be called with a reference to
+// the result of the final operation in any linear chain of operations that
+// depends on the provider to ensure that the provider will stay open at least
+// long enough to perform those operations.
+//
+// This is a compound build action that adds a number of different items to
+// the graph at once, although each distinct provider instance address gets
+// only one set of nodes added and then subsequent calls get references to
+// the same operation results.
+func (b *Builder) ProviderInstance(addr addrs.AbsProviderInstanceCorrect, waitFor []AnyResultRef) (ResultRef[providers.Configured], RegisterCloseBlockerFunc) {
+	configResult := b.ProviderInstanceConfig(addr)
+	providerAddrResult := b.ConstantProviderAddr(addr.Config.Config.Provider)
+
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	// We only register one index for each distinct provider instance address.
+	if existing, ok := b.openProviderRefs.GetOk(addr); ok {
+		return existing.Result, existing.CloseBlockerFunc
+	}
+	waiter := b.makeWaiter(waitFor)
+	openResult := operationRef[providers.Configured](b, operationDesc{
+		opCode:   opOpenProvider,
+		operands: []AnyResultRef{providerAddrResult, configResult, waiter},
+	})
+	closeWait, registerCloseBlocker := b.makeCloseBlocker()
+	// Nothing actually depends on the result of the "close" operation, but
+	// eventual execution of the graph will still wait for it to complete
+	// because _all_ operations must complete before execution is considered
+	// to be finished.
+	_ = operationRef[struct{}](b, operationDesc{
+		opCode:   opCloseProvider,
+		operands: []AnyResultRef{openResult, closeWait},
+	})
+	return openResult, registerCloseBlocker
+}
+
+// ManagedResourceObjectFinalPlan registers an operation to decide the "final plan" for a managed
+// resource instance object, which may or may not be "desired".
+//
+// If the object is not "desired" then the desiredInst result is a
+// [NilResultRef], producing a nil pointer. The underlying provider API
+// represents that situation by setting the "configuration value" to null.
+//
+// Similarly, if the object did not previously exist but is now desired then
+// the priorState result is a [NilResultRef] producing a nil pointer, which
+// should be represented in the provider API by setting the prior state
+// value to null.
+//
+// If the planning phase learned that the provider needs to handle a change
+// as a "replace" then in the execution graph there should be two separate
+// "final plan" and "apply changes" chains, where one has a nil desiredInst
+// and the other has a nil priorState. desiredInst and priorState should only
+// both be set when handling an in-place update.
+//
+// The waitFor argument captures arbitrary additional results that the
+// operation should block on even though it doesn't directly consume their
+// results. In practice this should refer to the final results of applying
+// any resource instances that this object depends on according to the
+// resource-instance-graph calculated during the planning process, thereby
+// ensuring that a particular object cannot be final-planned until all of its
+// resource-instance-graph dependencies have had their changes applied.
+func (b *Builder) ManagedResourceObjectFinalPlan(
+	desiredInst ResultRef[*eval.DesiredResourceInstance],
+	priorState ResultRef[*states.ResourceInstanceObject],
+	plannedVal ResultRef[cty.Value],
+	providerClient ResultRef[providers.Configured],
+	waitFor []AnyResultRef,
+) ResultRef[*ManagedResourceObjectFinalPlan] {
+	// We'll aggregate all of the waitFor nodes into a waiter node so we
+	// can pass it as just a single argument to the operation.
+	waiter := b.makeWaiter(waitFor)
+	return operationRef[*ManagedResourceObjectFinalPlan](b, operationDesc{
+		opCode:   opManagedFinalPlan,
+		operands: []AnyResultRef{desiredInst, priorState, plannedVal, providerClient, waiter},
+	})
+}
+
+// ApplyManagedResourceObjectChanges registers an operation to apply a "final
+// plan" for a managed resource instance object.
+//
+// The finalPlan argument should typically be something returned by a previous
+// call to [Builder.ManagedResourceObjectFinalPlan] with the same provider
+// client.
+func (b *Builder) ApplyManagedResourceObjectChanges(
+	finalPlan ResultRef[*ManagedResourceObjectFinalPlan],
+	providerClient ResultRef[providers.Configured],
+) ResultRef[*states.ResourceInstanceObject] {
+	return operationRef[*states.ResourceInstanceObject](b, operationDesc{
+		opCode:   opManagedApplyChanges,
+		operands: []AnyResultRef{finalPlan, providerClient},
+	})
+}
+
+func (b *Builder) DataRead(
+	desiredInst ResultRef[*eval.DesiredResourceInstance],
+	providerClient ResultRef[providers.Configured],
+	waitFor []AnyResultRef,
+) ResultRef[*states.ResourceInstanceObject] {
+	waiter := b.makeWaiter(waitFor)
+	return operationRef[*states.ResourceInstanceObject](b, operationDesc{
+		opCode:   opDataRead,
+		operands: []AnyResultRef{desiredInst, providerClient, waiter},
+	})
+}
+
+// SetResourceInstanceFinalStateResult records which result should be treated
+// as the "final state" for the given resource instance, for purposes such as
+// propagating the result value back into the evaluation system to allow
+// downstream expressions to derive from it.
+//
+// Only one call is allowed per distinct [addrs.AbsResourceInstance] value. If
+// two callers try to register for the same address then the second call will
+// panic.
+func (b *Builder) SetResourceInstanceFinalStateResult(addr addrs.AbsResourceInstance, result ResultRef[*states.ResourceInstanceObject]) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	if b.graph.resourceInstanceResults.Has(addr) {
+		panic(fmt.Sprintf("duplicate registration for %s final state result", addr))
+	}
+	b.graph.resourceInstanceResults.Put(addr, result)
+}
+
+// operationRef is a helper used by all of the [Builder] methods that produce
+// "operation" nodes, dealing with the common registration part.
+//
+// Callers MUST ensure all of the following before calling this function:
+// - They already hold a lock on builder.mu and retain it throughout the call.
+// - The specified T is the correct result type for the operation being described.
+//
+// This is effectively a method on [Builder], but written as a package-level
+// function just so it can have a type parameter.
+func operationRef[T any](builder *Builder, op operationDesc) ResultRef[T] {
+	idx := appendIndex(&builder.graph.ops, op)
+	return operationResultRef[T]{idx}
+}
+
+// makeCloseBlocker is a helper used by [Builder] methods that produce
+// open/close node pairs.
+//
+// Callers MUST hold a lock on b.mu throughout any call to this method.
+func (b *Builder) makeCloseBlocker() (ResultRef[struct{}], RegisterCloseBlockerFunc) {
+	idx := appendIndex(&b.graph.waiters, []AnyResultRef{})
+	ref := waiterResultRef{idx}
+	registerFunc := RegisterCloseBlockerFunc(func(ref AnyResultRef) {
+		b.mu.Lock()
+		defer b.mu.Unlock()
+		b.graph.waiters[idx] = append(b.graph.waiters[idx], ref)
+	})
+	return ref, registerFunc
+}
+
+func (b *Builder) makeWaiter(waitFor []AnyResultRef) ResultRef[struct{}] {
+	idx := appendIndex(&b.graph.waiters, []AnyResultRef{})
+	return waiterResultRef{idx}
+}
+
+type resultWithCloseBlockers[T any] struct {
+	Result             ResultRef[T]
+	CloseBlockerFunc   RegisterCloseBlockerFunc
+	CloseBlockerResult ResultRef[struct{}]
+}
+
+// RegisterCloseBlockerFunc is the signature of a function that adds a given
+// result references as a blocker for something to be "closed".
+//
+// Exactly what means to be a "close blocker" depends on context. Refer to the
+// documentation of whatever function is returning a value of this type.
+type RegisterCloseBlockerFunc func(AnyResultRef)

internal/engine/internal/execgraph/builder_test.go

@@ -0,0 +1,69 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package execgraph
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/opentofu/opentofu/internal/addrs"
+	"github.com/zclconf/go-cty/cty"
+)
+
+func TestBuilder_basics(t *testing.T) {
+	builder := NewBuilder()
+
+	// The following approximates might appear in the planning engine's code
+	// for building the execution subgraph for a desired resource instance,
+	// arranging for its changes to be planned and applied with whatever
+	// provider instance was selected in the configuration.
+	resourceInstAddr := addrs.Resource{
+		Mode: addrs.ManagedResourceMode,
+		Type: "bar_thing",
+		Name: "example",
+	}.Absolute(addrs.RootModuleInstance).Instance(addrs.NoKey)
+	initialPlannedValue := builder.ConstantValue(cty.ObjectVal(map[string]cty.Value{
+		"name": cty.StringVal("thingy"),
+	}))
+	providerClient, addProviderUser := builder.ProviderInstance(addrs.AbsProviderInstanceCorrect{
+		Config: addrs.AbsProviderConfigCorrect{
+			Config: addrs.ProviderConfigCorrect{
+				Provider: addrs.MustParseProviderSourceString("example.com/foo/bar"),
+			},
+		},
+	}, nil)
+	desiredInst := builder.DesiredResourceInstance(resourceInstAddr)
+	priorState := builder.ResourceInstancePriorState(resourceInstAddr)
+	finalPlan := builder.ManagedResourceObjectFinalPlan(
+		desiredInst,
+		priorState,
+		initialPlannedValue,
+		providerClient,
+		nil,
+	)
+	newState := builder.ApplyManagedResourceObjectChanges(finalPlan, providerClient)
+	addProviderUser(newState)
+	builder.SetResourceInstanceFinalStateResult(resourceInstAddr, newState)
+
+	graph := builder.Finish()
+	got := graph.DebugRepr()
+	want := strings.TrimLeft(`
+v[0] = cty.ObjectVal(map[string]cty.Value{
+    "name": cty.StringVal("thingy"),
+});
+
+r[0] = OpenProvider(provider("example.com/foo/bar"), providerInstConfig(provider["example.com/foo/bar"]), await());
+r[1] = CloseProvider(r[0], await(r[3]));
+r[2] = ManagedFinalPlan(desired(bar_thing.example), priorState(bar_thing.example), v[0], r[0], await());
+r[3] = ManagedApplyChanges(r[2], r[0]);
+
+bar_thing.example = r[3];
+`, "\n")
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Error("wrong result\n" + diff)
+	}
+}

internal/engine/internal/execgraph/compiled.go

@@ -0,0 +1,113 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package execgraph
+
+import (
+	"context"
+	"sync"
+
+	"github.com/zclconf/go-cty/cty"
+
+	"github.com/opentofu/opentofu/internal/addrs"
+	"github.com/opentofu/opentofu/internal/lang/grapheval"
+	"github.com/opentofu/opentofu/internal/tfdiags"
+)
+
+type CompiledGraph struct {
+	// ops is the main essence of a compiled graph: a series of functions
+	// that we'll run all at once, one goroutine each, and then wait until
+	// they've all returned something.
+	//
+	// In practice these functions will typically depend on one another
+	// indirectly through [workgraph.Promise] values, but it's up to the
+	// compiler to arrange for the necessary data flow while it's building
+	// these compiled operations. Execution is complete once all of these
+	// functions have returned.
+	ops []anyCompiledOperation
+
+	// resourceInstanceValues provides a function for each resource instance
+	// that was registered as a "sink" during graph building which blocks
+	// until the final state for that resource instance is available and then
+	// returns the object value to represent the resource instance in downstream
+	// expression evaluation.
+	resourceInstanceValues addrs.Map[addrs.AbsResourceInstance, func(ctx context.Context) cty.Value]
+}
+
+// Execute performs all of the work described in the execution graph in a
+// suitable order, returning any diagnostics that operations might return
+// along the way.
+//
+// If there are resource instance operations in the graph (which is typical for
+// any useful execution graph) then typically the evaluation system should
+// be running concurrently and be taking resource instance results from
+// calls to [CompiledGraph.ResourceInstanceValue] so that the graph execution
+// and evaluation system can collaborate to drive the execution process forward
+// together.
+func (c *CompiledGraph) Execute(ctx context.Context) tfdiags.Diagnostics {
+	var diags tfdiags.Diagnostics
+	var diagsMu sync.Mutex
+
+	var wg sync.WaitGroup
+	wg.Add(len(c.ops))
+	for _, op := range c.ops {
+		wg.Go(func() {
+			opDiags := op(grapheval.ContextWithNewWorker(ctx))
+			diagsMu.Lock()
+			diags = diags.Append(opDiags)
+			diagsMu.Unlock()
+		})
+	}
+	wg.Wait()
+
+	return diags
+}
+
+// ResourceInstanceValue blocks until after changes have been applied for the
+// given resource instance address and then returns a [cty.Value] that should
+// represent that resource instance in downstream expression evaluation.
+//
+// Calls to this method should run concurrently with a call to
+// [CompiledGraph.Execute] because otherwise the operations that generate the
+// final state for resource instances will not run and thus this will block
+// indefinitely waiting for results that will never arrive.
+func (c *CompiledGraph) ResourceInstanceValue(ctx context.Context, addr addrs.AbsResourceInstance) cty.Value {
+	getter, ok := c.resourceInstanceValues.GetOk(addr)
+	if !ok {
+		// If we get asked for a resource instance address that wasn't involved
+		// in the plan then we'll assume it was excluded from the plan by
+		// something like the -target option or deferred actions, and so we'll
+		// just return a completely-unknown placeholder to let the rest of the
+		// evaluation proceed. This should be valid as long as the planning
+		// phase made valid and consistent decisions about what to exclude,
+		// such that if a particular resource instance is excluded then any
+		// other resource or provider instance that depends on it must also be
+		// excluded.
+		return cty.DynamicVal
+	}
+	return getter(ctx)
+}
+
+// compiledOperation is the signature of a function acting as the implementation
+// of a specific operation in a compiled graph.
+type compiledOperation[Result any] func(ctx context.Context) (Result, tfdiags.Diagnostics)
+
+// anyCompiledOperation is a type-erased version of [compiledOperation] used
+// in situations where we only care that they got executed and have completed,
+// without needing the actual results.
+//
+// The main way to create a function of this type is to pass a
+// [compiledOperation] of some other type to [typeErasedCompiledOperation].
+type anyCompiledOperation = func(ctx context.Context) tfdiags.Diagnostics
+
+// typeErasedCompiledOperation turns a [compiledOperation] of some specific
+// result type into a type-erased [anyCompiledOperation], by discarding
+// its result and just returning its diagnostics.
+func typeErasedCompiledOperation[Result any](op compiledOperation[Result]) anyCompiledOperation {
+	return func(ctx context.Context) tfdiags.Diagnostics {
+		_, diags := op(ctx)
+		return diags
+	}
+}

internal/engine/internal/execgraph/graph.go

@@ -0,0 +1,180 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package execgraph
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/zclconf/go-cty-debug/ctydebug"
+	"github.com/zclconf/go-cty/cty"
+
+	"github.com/opentofu/opentofu/internal/addrs"
+	"github.com/opentofu/opentofu/internal/states"
+)
+
+type Graph struct {
+	// Overall "graph" is modelled as a collection of tables representing
+	// different kinds of nodes, and then the actual graph relationships are
+	// modeled as [ResultRef] or [AnyResultRef] values that are all really
+	// just indices into these tables.
+
+	//////// Constants saved directly in the graph
+	// The tables in this section are values that are decided during the
+	// planning phase and need not be recalculated during the apply phase,
+	// and so we just store them directly.
+
+	// constantVals is the table of constant values that are to be saved
+	// directly inside the execution graph.
+	constantVals []cty.Value
+	// providerAddrs is the table of provider addresses that are to be saved
+	// directly inside the execution graph.
+	providerAddrs []addrs.Provider
+
+	//////// ApplyOracle queries
+	// The tables in this section represent requests for information from
+	// the configuration evaluation system via its ApplyOracle API.
+
+	// desiredStateRefs is the table of references to resource instances from
+	// the desired state.
+	desiredStateRefs []addrs.AbsResourceInstance
+	// providerInstConfigRefs is the table of references to provider instance
+	// configuration values.
+	providerInstConfigRefs []addrs.AbsProviderInstanceCorrect
+
+	//////// Prior state queries
+	// The tables in this section represent requests for information from
+	// the prior state.
+
+	// priorStateRefs is the table of references to resource instance objects
+	// from the prior state.
+	priorStateRefs []resourceInstanceStateRef
+
+	//////// The actual side-effects
+	// The tables in this section deal with the main side-effects that we're
+	// intending to perform, and modelling the interactions between them.
+	//
+	// These are the only graph nodes that can directly depend on results from
+	// other graph nodes. Everything in the other sections above is fetching
+	// data from outside of the apply engine, although those which interact
+	// with the ApplyOracle will often depend indirectly on results in this
+	// section where the configuration defines the desired state for one
+	// resource instance in terms of the final state of another resource
+	// instance.
+
+	// ops are the actual operations -- functions with side-effects --
+	// that are the main purpose of the execution graph. Operations can
+	// depend on each other and on constant values or state references.
+	ops []operationDesc
+	// waiters are nodes that just express a dependency on the work that
+	// produces some other results even though the actual value of the
+	// result isn't needed. For example, this can be used to describe
+	// what other work needs to complete before a provider instance is closed.
+	//
+	// Although it's not actually enforced by the model, it's only really useful
+	// to add _operation_ results to "waiter" nodes, because operations are
+	// how we model side-effects that we might need to wait for completion of.
+	waiters [][]AnyResultRef
+	// resourceInstanceResults are "sink" nodes that capture references to
+	// the "final state" results for desired resource instances that are
+	// subject to changes in this graph, allowing the resulting values to
+	// propagate back into the evaluation system so that downstream resource
+	// instance configurations can be derived from them.
+	//
+	// Due to the behavior of the concurrently-running expression evaluation
+	// system, there's an effective implied dependency edge between results
+	// captured in here and the entries in desiredStateRefs for any resource
+	// instances whose configuration is derived from the result of an entry
+	// in this map. However, the execution graph is not supposed to rely on
+	// those implied edges for correct execution order: the "final plan"
+	// operation for each resource instance should also directly depend on
+	// the results of any resource instances that were identified as
+	// resource-instance-graph dependencies during the planning process.
+	resourceInstanceResults addrs.Map[addrs.AbsResourceInstance, ResultRef[*states.ResourceInstanceObject]]
+}
+
+// DebugRepr returns a relatively-concise string representation of the
+// graph which includes all of the registered operations and their operands,
+// along with any constant values they rely on.
+//
+// The result is intended primarily for human consumption when testing or
+// debugging. It's not an executable or parseable representation and details
+// about how it's formatted might change over time.
+func (g *Graph) DebugRepr() string {
+	var buf strings.Builder
+	for idx, val := range g.constantVals {
+		fmt.Fprintf(&buf, "v[%d] = %s;\n", idx, strings.TrimSpace(ctydebug.ValueString(val)))
+	}
+	if len(g.constantVals) != 0 && (len(g.ops) != 0 || g.resourceInstanceResults.Len() != 0) {
+		buf.WriteByte('\n')
+	}
+	for idx, op := range g.ops {
+		fmt.Fprintf(&buf, "r[%d] = %s(", idx, strings.TrimLeft(op.opCode.String(), "op"))
+		for opIdx, result := range op.operands {
+			if opIdx != 0 {
+				buf.WriteString(", ")
+			}
+			buf.WriteString(g.resultDebugRepr(result))
+		}
+		buf.WriteString(");\n")
+	}
+	if g.resourceInstanceResults.Len() != 0 && (len(g.ops) != 0 || len(g.constantVals) != 0) {
+		buf.WriteByte('\n')
+	}
+	for _, elem := range g.resourceInstanceResults.Elems {
+		fmt.Fprintf(&buf, "%s = %s;\n", elem.Key.String(), g.resultDebugRepr(elem.Value))
+	}
+	return buf.String()
+}
+
+func (g *Graph) resultDebugRepr(result AnyResultRef) string {
+	switch result := result.(type) {
+	case valueResultRef:
+		return fmt.Sprintf("v[%d]", result.index)
+	case providerAddrResultRef:
+		providerAddr := g.providerAddrs[result.index]
+		return fmt.Sprintf("provider(%q)", providerAddr)
+	case desiredResourceInstanceResultRef:
+		instAddr := g.desiredStateRefs[result.index]
+		return fmt.Sprintf("desired(%s)", instAddr)
+	case resourceInstancePriorStateResultRef:
+		ref := g.priorStateRefs[result.index]
+		if ref.DeposedKey != states.NotDeposed {
+			return fmt.Sprintf("deposedState(%s, %s)", ref.ResourceInstance, ref.DeposedKey)
+		}
+		return fmt.Sprintf("priorState(%s)", ref.ResourceInstance)
+	case providerInstanceConfigResultRef:
+		pInstAddr := g.providerInstConfigRefs[result.index]
+		return fmt.Sprintf("providerInstConfig(%s)", pInstAddr)
+	case anyOperationResultRef:
+		return fmt.Sprintf("r[%d]", result.operationResultIndex())
+	case waiterResultRef:
+		awaiting := g.waiters[result.index]
+		var buf strings.Builder
+		buf.WriteString("await(")
+		for i, r := range awaiting {
+			if i != 0 {
+				buf.WriteString(", ")
+			}
+			buf.WriteString(g.resultDebugRepr(r))
+		}
+		buf.WriteString(")")
+		return buf.String()
+	case nil:
+		return "nil"
+	default:
+		// Should try to keep the above cases comprehensive because
+		// this default is not very readable and might even be
+		// useless if it's a reference into a table we're not otherwise
+		// including the output here.
+		return fmt.Sprintf("%#v", result)
+	}
+}
+
+type resourceInstanceStateRef struct {
+	ResourceInstance addrs.AbsResourceInstance
+	DeposedKey       states.DeposedKey
+}

internal/engine/internal/execgraph/opcode_string.go

@@ -0,0 +1,28 @@
+// Code generated by "stringer -type=opCode"; DO NOT EDIT.
+
+package execgraph
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[opManagedFinalPlan-1]
+	_ = x[opManagedApplyChanges-2]
+	_ = x[opDataRead-3]
+	_ = x[opOpenProvider-4]
+	_ = x[opCloseProvider-5]
+}
+
+const _opCode_name = "opManagedFinalPlanopManagedApplyChangesopDataReadopOpenProvideropCloseProvider"
+
+var _opCode_index = [...]uint8{0, 18, 39, 49, 63, 78}
+
+func (i opCode) String() string {
+	idx := int(i) - 1
+	if i < 1 || idx >= len(_opCode_index)-1 {
+		return "opCode(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _opCode_name[_opCode_index[idx]:_opCode_index[idx+1]]
+}

internal/engine/internal/execgraph/operation.go

@@ -0,0 +1,45 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package execgraph
+
+// operationDesc is a low-level description of an operation that can be saved in
+// serialized form for reloading later and can be "compiled" into a form
+// suitable for execution once the full graph execution graph has been built.
+type operationDesc struct {
+	opCode   opCode
+	operands []AnyResultRef
+}
+
+// opCode is an enumeration of all of the different operation types that
+// can appear in an execution graph.
+//
+// This does not represent the actual implementation of each opCode. The
+// descriptions of operations are "compiled" into executable functions as a
+// separate step after assembling the execution graph piecemeal during the
+// planning process.
+type opCode int
+
+const (
+	_ = opCode(iota) // the zero value is not a valid operation
+	// opManagedFinalPlan uses the configuration value and initial planned state
+	// for a resource instance to produce its final plan, which can then
+	// be applied by [opApplyChanges].
+	opManagedFinalPlan
+	// opManagedApplyChanges applies a plan created by [opFinalPlan].
+	opManagedApplyChanges
+	// opDataRead reads a data resource.
+	opDataRead
+	// opOpenProvider takes a provider address and a configuration value
+	// and produces a configured client for the specified provider.
+	opOpenProvider
+	// opCloseProvider takes a client previously created by [opOpenProvider],
+	// along with a "waiter" node that resolves only once all uses of the
+	// provider client are done, and closes the client once the waiter node
+	// has resolved.
+	opCloseProvider
+)
+
+//go:generate go run golang.org/x/tools/cmd/stringer -type=opCode

internal/engine/internal/execgraph/resource.go

@@ -0,0 +1,49 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package execgraph
+
+import (
+	"github.com/zclconf/go-cty/cty"
+)
+
+// ManagedResourceObjectFinalPlan represents a final plan -- ready to actually
+// be applied -- for some managed resource instance object that could be any of
+// a current object for a desired resource instance, a current object for
+// an orphan resource instance, or a deposed object for any resource
+// instance.
+//
+// Note that for execution graph purposes a "replace" action is always
+// represented as two separate "final plans", where the "delete" leg is
+// represented by the configuration being null and the "create" leg is
+// represented by the prior state being null. This struct type intentionally
+// does not carry any information about the identity of the object the
+// plan is for because that is implied by the relationships in the graph and
+// there should be no assumptions about e.g. there being exactly one final
+// plan per resource instance, etc.
+type ManagedResourceObjectFinalPlan struct {
+	// ResourceType is the resource type of the object this plan is for, as
+	// would be understood by the provider that generated this plan.
+	ResourceType string
+
+	// ConfigVal is the value representing the configuration for this
+	// object, but only if it's a "desired" object. This is always a null
+	// value for "orphan" instances and deposed objects, because they have
+	// no configuration by definition.
+	ConfigVal cty.Value
+	// PriorStateVal is the value representing this object in the prior
+	// state, or a null value if this object didn't previously exist and
+	// is therefore presumably being created.
+	PriorStateVal cty.Value
+	// PlannedVal is the value returned by the provider when it was asked
+	// to produce a plan. This is an approximation of the final result
+	// with unknown values as placeholders for anything that won't be known
+	// until after the change has been applied.
+	PlannedVal cty.Value
+	// TODO: The "Private" value that the provider returned in its planning
+	// response.
+	// TODO: Anything else we'd need to populate an "ApplyResourceChanges"
+	// request to the associated provider.
+}

internal/engine/internal/execgraph/result.go

@@ -0,0 +1,150 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package execgraph
+
+import (
+	"github.com/zclconf/go-cty/cty"
+
+	"github.com/opentofu/opentofu/internal/addrs"
+	"github.com/opentofu/opentofu/internal/lang/eval"
+	"github.com/opentofu/opentofu/internal/states"
+)
+
+// ResultRef represents a result of type T that will be produced by
+// some other operation that is opaque to the recipients of the result.
+type ResultRef[T any] interface {
+	resultPlaceholderSigil(T)
+	AnyResultRef
+}
+
+// AnyResultRef is a type-erased [ResultRef], for data
+// structures that only need to represent the relationships between results
+// and not the types of those results.
+type AnyResultRef interface {
+	anyResultPlaceholderSigil()
+}
+
+// valueResultRef is a [ResultRef] referring to an item in the a graph's
+// table of constant values.
+type valueResultRef struct {
+	index int
+}
+
+var _ ResultRef[cty.Value] = valueResultRef{}
+
+// anyResultPlaceholderSigil implements ResultPlaceholder.
+func (v valueResultRef) anyResultPlaceholderSigil() {}
+
+// resultPlaceholderSigil implements ResultPlaceholder.
+func (v valueResultRef) resultPlaceholderSigil(cty.Value) {}
+
+// providerAddrResultRef is a [ResultRef] referring to an item in the a graph's
+// table of constant provider addresses.
+type providerAddrResultRef struct {
+	index int
+}
+
+var _ ResultRef[addrs.Provider] = providerAddrResultRef{}
+
+// anyResultPlaceholderSigil implements ResultPlaceholder.
+func (v providerAddrResultRef) anyResultPlaceholderSigil() {}
+
+// resultPlaceholderSigil implements ResultPlaceholder.
+func (v providerAddrResultRef) resultPlaceholderSigil(addrs.Provider) {}
+
+// desiredResourceInstanceResultRef is a [ResultRef] referring to an item in
+// the a graph's table of desired state lookups.
+type desiredResourceInstanceResultRef struct {
+	index int
+}
+
+var _ ResultRef[*eval.DesiredResourceInstance] = desiredResourceInstanceResultRef{}
+
+// anyResultPlaceholderSigil implements ResultRef.
+func (d desiredResourceInstanceResultRef) anyResultPlaceholderSigil() {}
+
+// resultPlaceholderSigil implements ResultRef.
+func (d desiredResourceInstanceResultRef) resultPlaceholderSigil(*eval.DesiredResourceInstance) {}
+
+// resourceInstancePriorStateResultRef is a [ResultRef] referring to an item in
+// the a graph's table of prior state lookups.
+type resourceInstancePriorStateResultRef struct {
+	index int
+}
+
+var _ ResultRef[*states.ResourceInstanceObject] = resourceInstancePriorStateResultRef{}
+
+// anyResultPlaceholderSigil implements ResultRef.
+func (r resourceInstancePriorStateResultRef) anyResultPlaceholderSigil() {}
+
+// resultPlaceholderSigil implements ResultRef.
+func (r resourceInstancePriorStateResultRef) resultPlaceholderSigil(*states.ResourceInstanceObject) {}
+
+// providerInstanceConfigResultRef is a [ResultRef] referring to an item in a
+// graph's table of provider instance configuration requests.
+type providerInstanceConfigResultRef struct {
+	index int
+}
+
+var _ ResultRef[cty.Value] = providerInstanceConfigResultRef{}
+
+// anyResultPlaceholderSigil implements ResultPlaceholder.
+func (v providerInstanceConfigResultRef) anyResultPlaceholderSigil() {}
+
+// resultPlaceholderSigil implements ResultPlaceholder.
+func (v providerInstanceConfigResultRef) resultPlaceholderSigil(cty.Value) {}
+
+type operationResultRef[T any] struct {
+	index int
+}
+
+var _ ResultRef[struct{}] = operationResultRef[struct{}]{}
+var _ anyOperationResultRef = operationResultRef[struct{}]{}
+
+// anyResultPlaceholderSigil implements ResultPlaceholder.
+func (o operationResultRef[T]) anyResultPlaceholderSigil() {}
+
+// resultPlaceholderSigil implements ResultPlaceholder.
+func (o operationResultRef[T]) resultPlaceholderSigil(T) {}
+
+// operationResultIndex implements anyOperationResultRef.
+func (o operationResultRef[T]) operationResultIndex() int {
+	return o.index
+}
+
+type anyOperationResultRef interface {
+	operationResultIndex() int
+}
+
+type waiterResultRef struct {
+	index int
+}
+
+var _ ResultRef[struct{}] = waiterResultRef{}
+
+// anyResultPlaceholderSigil implements ResultRef.
+func (w waiterResultRef) anyResultPlaceholderSigil() {}
+
+// resultPlaceholderSigil implements ResultRef.
+func (w waiterResultRef) resultPlaceholderSigil(struct{}) {}
+
+// NilResultRef returns a special result ref which just always produces the
+// zero value of type T, without doing any other work or referring to any other
+// data.
+//
+// This should typically only be used for types whose zero value is considered
+// to be the "nil" value for the type, such as pointer types, since otherwise
+// the recipient cannot distinguish it from a valid result that just happens
+// to be the zero value.
+func NilResultRef[T any]() ResultRef[T] {
+	return nil
+}
+
+func appendIndex[E any](s *[]E, new E) int {
+	idx := len(*s)
+	*s = append(*s, new)
+	return idx
+}