lang/eval: Provider instance to ephemeral resource instance dependencies

This completes a previously-missing piece of the "prepareToPlan" result,
tracking which provider instances are relying on each ephemeral resource
instance.

This is important because the planning engine can "see"
resource-instance-to-provider relationships in the state that the eval
system isn't aware of, and so the planning engine must be able to keep
a provider instance open long enough to deal with both config-driven and
state-driven uses of it, which in turn means keeping open any ephemeral
resource instances that those provider instances depend on.

Signed-off-by: Martin Atkins <mart@degeneration.co.uk>

internal/lang/eval/config_prepare.go

@@ -61,6 +61,7 @@ func (c *ConfigInstance) prepareToPlan(ctx context.Context) (*ResourceRelationsh
 				if !ret.EphemeralResourceUsers.Has(dependeeAddr) {
 					ret.EphemeralResourceUsers.Put(dependeeAddr, EphemeralResourceInstanceUsers{
 						ResourceInstances: addrs.MakeSet[addrs.AbsResourceInstance](),
+						ProviderInstances: addrs.MakeSet[addrs.AbsProviderInstanceCorrect](),
 					})
 				}
 				set := ret.EphemeralResourceUsers.Get(dependeeAddr).ResourceInstances
@@ -79,6 +80,22 @@ func (c *ConfigInstance) prepareToPlan(ctx context.Context) (*ResourceRelationsh
 			set.Add(dependerAddr)
 		}
 	}
+	for depender := range evalglue.ProviderInstancesDeep(ctx, rootModuleInstance) {
+		dependerAddr := depender.Addr
+		for dependee := range depender.ResourceInstanceDependencies(ctx) {
+			dependeeAddr := dependee.Addr
+			if dependeeAddr.Resource.Resource.Mode == addrs.EphemeralResourceMode {
+				if !ret.EphemeralResourceUsers.Has(dependeeAddr) {
+					ret.EphemeralResourceUsers.Put(dependeeAddr, EphemeralResourceInstanceUsers{
+						ResourceInstances: addrs.MakeSet[addrs.AbsResourceInstance](),
+						ProviderInstances: addrs.MakeSet[addrs.AbsProviderInstanceCorrect](),
+					})
+				}
+				set := ret.EphemeralResourceUsers.Get(dependeeAddr).ProviderInstances
+				set.Add(dependerAddr)
+			}
+		}
+	}
 	return ret, diags
 }
 
@@ -106,23 +123,7 @@ type ResourceRelationships struct {
 
 type EphemeralResourceInstanceUsers struct {
 	ResourceInstances addrs.Set[addrs.AbsResourceInstance]
-
+	ProviderInstances addrs.Set[addrs.AbsProviderInstanceCorrect]
-	// TODO: ProviderInstances
-	// This should be a set of provider instance addresses, but we don't
-	// currently have a single addrs type for "absolute provider instance" --
-	// [addrs.AbsProviderInstance] is a misnomer and should really be named
-	// [addrs.ConfigProviderConfig] or similar -- and we also haven't got
-	// support for resolving provider instance references in package
-	// [configgraph] anyway.
-	//
-	// We need to model provider instances directly, rather than just
-	// using the resource instance relationships as a proxy, because
-	// the planning phase also needs to ask providers to plan "orphaned"
-	// resource instances that are only tracked in state and so this
-	// eval package cannot take them into account when figuring out
-	// which resource instances belong to a particular provider instance.
-	// (The orphan-to-provider-instance relationships are tracked in the
-	// state, rather than in the config.)
 }
 
 type ProviderInstanceUsers struct {

internal/lang/eval/config_prepare_test.go

@@ -79,12 +79,24 @@ func TestPrepare_ephemeralResourceUsers(t *testing.T) {
 						#   ephemeral.foo.b[count.index],
 						# ]
 					}
+					provider "foo" {
+						alias = "other"
+
+						name = ephemeral.foo.a[0].name
+					}
 				`),
 			}),
 			Providers: ProvidersForTesting(map[addrs.Provider]*providers.GetProviderSchemaResponse{
 				addrs.MustParseProviderSourceString("test/foo"): {
 					Provider: providers.Schema{
-						Block: &configschema.Block{},
+						Block: &configschema.Block{
+							Attributes: map[string]*configschema.Attribute{
+								"name": {
+									Type:     cty.String,
+									Optional: true,
+								},
+							},
+						},
 					},
 					EphemeralResources: map[string]providers.Schema{
 						"foo": {
@@ -153,9 +165,16 @@ func TestPrepare_ephemeralResourceUsers(t *testing.T) {
 			Provider: addrs.MustParseProviderSourceString("test/foo"),
 		},
 	}.Instance(addrs.NoKey)
+	providerOtherInstAddr := addrs.AbsProviderConfigCorrect{
+		Module: addrs.RootModuleInstance,
+		Config: addrs.ProviderConfigCorrect{
+			Provider: addrs.MustParseProviderSourceString("test/foo"),
+			Alias:    "other",
+		},
+	}.Instance(addrs.NoKey)
 
 	// The analysis should detect that:
-	// - ephemeral.foo.a[0] is used by ephemeral.foo.b[0] and foo.c[0]
+	// - ephemeral.foo.a[0] is used by ephemeral.foo.b[0] and foo.c[0], and by the foo.other provider instance
 	// - ephemeral.foo.a[1] is used by ephemeral.foo.b[1] and foo.c[1]
 	// - ephemeral.foo.b[0] is used by only foo.c[0]
 	// - ephemeral.foo.b[1] is used by only foo.c[1]
@@ -183,22 +202,28 @@ func TestPrepare_ephemeralResourceUsers(t *testing.T) {
 					fooB.Instance(inst0),
 					fooC.Instance(inst0),
 				),
+				ProviderInstances: addrs.MakeSet(
+					providerOtherInstAddr,
+				),
 			}),
 			addrs.MakeMapElem(fooA.Instance(inst1), EphemeralResourceInstanceUsers{
 				ResourceInstances: addrs.MakeSet(
 					fooB.Instance(inst1),
 					fooC.Instance(inst1),
 				),
+				ProviderInstances: addrs.MakeSet[addrs.AbsProviderInstanceCorrect](),
 			}),
 			addrs.MakeMapElem(fooB.Instance(inst0), EphemeralResourceInstanceUsers{
 				ResourceInstances: addrs.MakeSet(
 					fooC.Instance(inst0),
 				),
+				ProviderInstances: addrs.MakeSet[addrs.AbsProviderInstanceCorrect](),
 			}),
 			addrs.MakeMapElem(fooB.Instance(inst1), EphemeralResourceInstanceUsers{
 				ResourceInstances: addrs.MakeSet(
 					fooC.Instance(inst1),
 				),
+				ProviderInstances: addrs.MakeSet[addrs.AbsProviderInstanceCorrect](),
 			}),
 		),
 
@@ -343,6 +368,7 @@ func TestPrepare_crossModuleReferences(t *testing.T) {
 				ResourceInstances: addrs.MakeSet(
 					fooB.Instance(addrs.NoKey),
 				),
+				ProviderInstances: addrs.MakeSet[addrs.AbsProviderInstanceCorrect](),
 			}),
 		),
 

internal/lang/eval/internal/configgraph/provider_instance.go

@@ -7,6 +7,7 @@ package configgraph
 
 import (
 	"context"
+	"iter"
 
 	"github.com/apparentlymart/go-workgraph/workgraph"
 	"github.com/hashicorp/hcl/v2"
@@ -117,6 +118,29 @@ func (p *ProviderInstance) ConfigValue(ctx context.Context) (cty.Value, tfdiags.
 	})
 }
 
+// ResourceInstanceDependencies returns a sequence of any resource instances
+// whose results the configuration of this provider instance depends on.
+//
+// The result of this is trustworthy only if [ProviderInstance.CheckAll]
+// returns without diagnostics. If errors are present then the result is
+// best-effort but likely to be incomplete.
+func (p *ProviderInstance) ResourceInstanceDependencies(ctx context.Context) iter.Seq[*ResourceInstance] {
+	// FIXME: This should also take into account:
+	// - explicit dependencies in the depends_on argument
+	// - ....anything else?
+	//
+	// We should NOT need to take into account dependencies of the parent
+	// provider config's InstanceSelector because substitutions of
+	// count.index/each.key/each.value will transfer those in automatically by
+	// the RepetitionData values being marked.
+
+	// We ignore diagnostics here because callers should always perform a
+	// CheckAll tree walk, including a visit to this provider instance object,
+	// before trusting anything else that any configgraph nodes report.
+	resultVal := diagsHandledElsewhere(p.ConfigValue(ctx))
+	return ContributingResourceInstances(resultVal)
+}
+
 // ValueSourceRange implements exprs.Valuer.
 func (p *ProviderInstance) ValueSourceRange() *tfdiags.SourceRange {
 	// TODO: Does it make sense to return the source range of the provider

internal/lang/eval/internal/evalglue/module_instance.go

@@ -144,6 +144,18 @@ type CompiledModuleInstance interface {
 	// resource.
 	ResourceInstancesForResource(ctx context.Context, addr addrs.Resource) iter.Seq[*configgraph.ResourceInstance]
 
+	// ProviderInstances returns a sequence of all of the provider instances
+	// declared in the module.
+	//
+	// The set of provider instances can be decided dynamically based on
+	// references to other objects, and so reads from the returned sequence
+	// may block until the needed upstream objects have finished resolving.
+	//
+	// Some of the enumerated objects might be placeholders for zero or more
+	// instances where there isn't yet enough information to determine exactly
+	// which dynamic instances are declared.
+	ProviderInstances(ctx context.Context) iter.Seq[*configgraph.ProviderInstance]
+
 	// ProviderInstance returns the [configgraph.ProviderInstance]
 	// representation of the provider instance with the given address, or
 	// nil if there is no such instance declared.
@@ -251,6 +263,27 @@ func ResourceInstancesDeep(ctx context.Context, root CompiledModuleInstance) ite
 	}
 }
 
+// ProviderInstancesDeep produces all of the provider instances across the given
+// root module instance and all of its descendents.
+//
+// The decision about which instances exist can be made dynamically by arbitrary
+// expressions, so any step in the returned sequence may block until further
+// information becomes available.
+//
+// This is implemented in terms of [ModuleInstancesDeep].
+func ProviderInstancesDeep(ctx context.Context, root CompiledModuleInstance) iter.Seq[*configgraph.ProviderInstance] {
+	ctx = grapheval.ContextWithNewWorker(ctx)
+	return func(yield func(*configgraph.ProviderInstance) bool) {
+		for _, moduleInst := range ModuleInstancesDeep(ctx, root) {
+			for providerInst := range moduleInst.ProviderInstances(ctx) {
+				if !yield(providerInst) {
+					return
+				}
+			}
+		}
+	}
+}
+
 // ProviderInstance digs through the tree of module instances with the given
 // root to try to find the [configgraph.ProviderInstance] representation
 // of the provider instance with the given address.

internal/lang/eval/internal/tofu2024/module_instance.go

@@ -165,6 +165,20 @@ func (c *CompiledModuleInstance) ChildModuleInstancesForCall(ctx context.Context
 	}
 }
 
+// ProviderInstances implements evalglue.CompiledModuleInstance.
+func (c *CompiledModuleInstance) ProviderInstances(ctx context.Context) iter.Seq[*configgraph.ProviderInstance] {
+	ctx = grapheval.ContextWithNewWorker(ctx)
+	return func(yield func(*configgraph.ProviderInstance) bool) {
+		for _, node := range c.providerConfigNodes {
+			for _, compiled := range node.Instances(ctx) {
+				if !yield(compiled) {
+					return
+				}
+			}
+		}
+	}
+}
+
 // ProviderInstance implements evalglue.CompiledModuleInstance.
 func (c *CompiledModuleInstance) ProviderInstance(ctx context.Context, addr addrs.ProviderInstanceCorrect) *configgraph.ProviderInstance {
 	localName, ok := c.providerLocalNames[addr.Config.Provider]
@@ -221,24 +235,6 @@ func (c *CompiledModuleInstance) ResourceInstancesForResource(ctx context.Contex
 	}
 }
 
-// ProviderInstancesDeep implements evalglue.CompiledModuleInstance.
-func (c *CompiledModuleInstance) ProviderInstancesDeep(ctx context.Context) iter.Seq[*configgraph.ProviderInstance] {
-	return func(yield func(*configgraph.ProviderInstance) bool) {
-		for _, r := range c.providerConfigNodes {
-			// NOTE: r.Instances will block if the provider config's
-			// [InstanceSelector] depends on other parts of the configuration
-			// that aren't yet ready to produce their value.
-			for _, inst := range r.Instances(ctx) {
-				if !yield(inst) {
-					return
-				}
-			}
-		}
-
-		// TODO: Collect provider instances from child module calls too.
-	}
-}
-
 // AnnounceAllGraphevalRequests implements evalglue.CompiledModuleInstance.
 func (c *CompiledModuleInstance) AnnounceAllGraphevalRequests(announce func(workgraph.RequestID, grapheval.RequestInfo)) {
 	c.moduleInstanceNode.AnnounceAllGraphevalRequests(announce)