Ensure Ephemeral values are handled by the diff transformer (#3495)

Signed-off-by: Andrei Ciobanu <andrei.ciobanu@opentofu.org>
Signed-off-by: James Humphries <james@james-humphries.co.uk>
Co-authored-by: Andrei Ciobanu <andrei.ciobanu@opentofu.org>

internal/tofu/context_apply2_test.go

@@ -5759,6 +5759,7 @@ ephemeral "test_ephemeral_resource" "a" {
 		Result: cty.ObjectVal(map[string]cty.Value{
 			"id":     cty.StringVal("id val"),
 			"secret": cty.StringVal("val"),
+			"input":  cty.NullVal(cty.String),
 		}),
 	}
 
@@ -5795,7 +5796,7 @@ ephemeral "test_ephemeral_resource" "a" {
 		Instances: map[addrs.InstanceKey]*states.ResourceInstance{
 			addrs.NoKey: {
 				Current: &states.ResourceInstanceObjectSrc{
-					AttrsJSON:          []byte(`{"id":"id val","secret":"val"}`),
+					AttrsJSON:          []byte(`{"id":"id val","input":null,"secret":"val"}`),
 					Status:             states.ObjectReady,
 					AttrSensitivePaths: []cty.PathValueMarks{},
 					Dependencies:       []addrs.ConfigResource{},
@@ -6678,3 +6679,274 @@ resource "test_resource" "res" {
 	assertNoErrors(t, diags)
 	assertState(t, state)
 }
+
+// TestContext2Apply_ephemeralInModuleWithExpansion checks that the expansion of the
+// ephemeral resources is not pruned even when there is an ephemeral instance node to
+// be executed.
+// This test has been added when a fix for https://github.com/opentofu/opentofu/issues/3489
+// was provided.
+func TestContext2Apply_ephemeralInModuleWithExpansion(t *testing.T) {
+	cfgs := map[string]map[string]string{
+		"1 level deep with for_each on module call": {
+			`mod/main.tf`: `
+				ephemeral "test_ephemeral_resource" "mod1_secret" {
+					input = "module1"
+				}
+				
+				output "exit" {
+				  ephemeral = true
+				  value = ephemeral.test_ephemeral_resource.mod1_secret.secret
+				}
+		`,
+			`main.tf`: `
+				ephemeral "test_ephemeral_resource" "root_secret" {
+					input = "root"
+				}
+				module "level1call" {
+				    for_each = toset(["root_1", "root_2"])
+				    source = "./mod"
+				}
+				output "exit" {
+				    value = ephemeralasnull(ephemeral.test_ephemeral_resource.root_secret.secret)
+				}
+		`,
+		},
+		"1 level deep with count on module call": {
+			`mod/main.tf`: `
+				ephemeral "test_ephemeral_resource" "mod1_secret" {
+					input = "module1"
+				}
+				
+				output "exit" {
+				  ephemeral = true
+				  value = ephemeral.test_ephemeral_resource.mod1_secret.secret
+				}
+		`,
+			`main.tf`: `
+				ephemeral "test_ephemeral_resource" "root_secret" {
+					input = "root"
+				}
+				module "level1call" {
+				    count = 2
+				    source = "./mod"
+				}
+				output "exit" {
+				    value = ephemeralasnull(ephemeral.test_ephemeral_resource.root_secret.secret)
+				}
+		`,
+		},
+		"for_each 1 level deep": {
+			`mod/main.tf`: `
+				ephemeral "test_ephemeral_resource" "mod1_secret" {
+					for_each = toset(["module1_1", "module1_2"])
+					input = each.key
+				}
+				
+				output "exit" {
+				  ephemeral = true
+				  value = [for s in ephemeral.test_ephemeral_resource.mod1_secret: s.secret]
+				}
+		`,
+			`main.tf`: `
+				ephemeral "test_ephemeral_resource" "root_secret" {
+					for_each = toset(["root_1", "root_2"])
+					input = each.key
+				}
+				module "level1call" {
+				    for_each = ephemeral.test_ephemeral_resource.root_secret
+				    source = "./mod"
+				}
+				output "exit" {
+				    value = ephemeralasnull([
+                      for s in ephemeral.test_ephemeral_resource.root_secret: s.secret
+                    ])
+				}
+		`,
+		},
+		"for_each 2 levels deep": {
+			`mod/mod/main.tf`: `
+				ephemeral "test_ephemeral_resource" "mod2_secret" {
+					for_each = toset(["module2_1", "module2_2"])
+					input = each.key
+				}
+		
+				output "exit" {
+				  ephemeral = true
+				  value = [for s in ephemeral.test_ephemeral_resource.mod2_secret: s.secret]
+				}
+		`,
+			`mod/main.tf`: `
+				ephemeral "test_ephemeral_resource" "mod1_secret" {
+					for_each = toset(["module1_1", "module1_2"])
+					input = each.key
+				}
+				module "level2call" {
+				    for_each = ephemeral.test_ephemeral_resource.mod1_secret
+				    source = "./mod"
+				}
+				output "exit" {
+				  ephemeral = true
+				  value = [for s in ephemeral.test_ephemeral_resource.mod1_secret: s.secret]
+				}
+		`,
+			`main.tf`: `
+				ephemeral "test_ephemeral_resource" "root_secret" {
+					for_each = toset(["root_1", "root_2"])
+					input = each.key
+				}
+				module "level1call" {
+				    for_each = ephemeral.test_ephemeral_resource.root_secret
+				    source = "./mod"
+				}
+				output "exit" {
+				    value = ephemeralasnull([
+		              for s in ephemeral.test_ephemeral_resource.root_secret: s.secret
+		            ])
+				}
+		`,
+		},
+		"count 1 level deep": {
+			`mod/main.tf`: `
+                locals {
+                    elements = ["module1_1", "module1_2"]
+                }
+				ephemeral "test_ephemeral_resource" "mod1_secret" {
+					count = length(local.elements)
+					input = local.elements[count.index]
+				}
+				
+				output "exit" {
+				  ephemeral = true
+				  value = [for s in ephemeral.test_ephemeral_resource.mod1_secret: s.secret]
+				}
+		`,
+			`main.tf`: `
+                locals {
+                    elements = ["root_1", "root_2"]
+                }
+				ephemeral "test_ephemeral_resource" "root_secret" {
+					count = length(local.elements)
+					input = local.elements[count.index]
+				}
+				module "level1call" {
+				    count = length(ephemeral.test_ephemeral_resource.root_secret)
+				    source = "./mod"
+				}
+				output "exit" {
+				    value = ephemeralasnull([
+                      for s in ephemeral.test_ephemeral_resource.root_secret: s.secret
+                    ])
+				}
+		`,
+		},
+		"count 2 levels deep": {
+			`mod/mod/main.tf`: `
+                locals {
+                    elements = ["module2_1", "module2_2"]
+                }
+				ephemeral "test_ephemeral_resource" "mod2_secret" {
+					count = length(local.elements)
+					input = local.elements[count.index]
+				}
+		
+				output "exit" {
+				  ephemeral = true
+				  value = [for s in ephemeral.test_ephemeral_resource.mod2_secret: s.secret]
+				}
+		`,
+			`mod/main.tf`: `
+                locals {
+                    elements = ["module1_1", "module1_2"]
+                }
+				ephemeral "test_ephemeral_resource" "mod1_secret" {
+					count = length(local.elements)
+					input = local.elements[count.index]
+				}
+				module "level2call" {
+				    count = length(ephemeral.test_ephemeral_resource.mod1_secret)
+				    source = "./mod"
+				}
+				output "exit" {
+				  ephemeral = true
+				  value = [for s in ephemeral.test_ephemeral_resource.mod1_secret: s.secret]
+				}
+		`,
+			`main.tf`: `
+                locals {
+                    elements = ["module1_1", "module1_2"]
+                }
+				ephemeral "test_ephemeral_resource" "root_secret" {
+					count = length(local.elements)
+					input = local.elements[count.index]
+				}
+				module "level1call" {
+				    count = length(ephemeral.test_ephemeral_resource.root_secret)
+				    source = "./mod"
+				}
+				output "exit" {
+				    value = ephemeralasnull([
+		              for s in ephemeral.test_ephemeral_resource.root_secret: s.secret
+		            ])
+				}
+		`,
+		},
+	}
+
+	for name, cfg := range cfgs {
+		t.Run(name, func(t *testing.T) {
+			m := testModuleInline(t, cfg)
+
+			provider := testProvider("test")
+			provider.OpenEphemeralResourceFn = func(request providers.OpenEphemeralResourceRequest) providers.OpenEphemeralResourceResponse {
+				val := request.Config.GetAttr("input").AsString()
+				return providers.OpenEphemeralResourceResponse{
+					Result: cty.ObjectVal(map[string]cty.Value{
+						"id":     cty.StringVal("id val"),
+						"secret": cty.StringVal(fmt.Sprintf("%s: secret val", val)),
+						"input":  cty.StringVal(val),
+					}),
+				}
+			}
+			provider.OpenEphemeralResourceResponse = &providers.OpenEphemeralResourceResponse{
+				Result: cty.ObjectVal(map[string]cty.Value{
+					"id":     cty.StringVal("id val"),
+					"secret": cty.StringVal("secret val"),
+				}),
+			}
+
+			ps := map[addrs.Provider]providers.Factory{
+				addrs.NewDefaultProvider("test"): testProviderFuncFixed(provider),
+			}
+
+			h := &testHook{}
+			apply := func(t *testing.T, m *configs.Config, prevState *states.State) (*states.State, tfdiags.Diagnostics) {
+				ctx := testContext2(t, &ContextOpts{
+					Providers: ps,
+					Hooks:     []Hook{h},
+				})
+
+				plan, diags := ctx.Plan(context.Background(), m, prevState, &PlanOpts{
+					Mode: plans.NormalMode,
+				})
+				if diags.HasErrors() {
+					return nil, diags
+				}
+
+				return ctx.Apply(context.Background(), plan, m, nil)
+			}
+
+			_, diags := apply(t, m, states.NewState())
+			if diags.HasErrors() {
+				t.Fatal(diags.Err())
+			}
+
+			// Apply again to be sure that nothing changes and it still working
+			newState, diags := apply(t, m, states.NewState())
+			if diags.HasErrors() {
+				t.Fatal(diags.Err())
+			}
+			fmt.Println(newState)
+		})
+
+	}
+}

internal/tofu/context_plan2_test.go

@@ -8815,6 +8815,7 @@ ephemeral "test_ephemeral_resource" "a" {
 		Result: cty.ObjectVal(map[string]cty.Value{
 			"id":     cty.StringVal("id val"),
 			"secret": cty.StringVal("val"),
+			"input":  cty.NullVal(cty.String),
 		}),
 	}
 
@@ -8848,6 +8849,7 @@ ephemeral "test_ephemeral_resource" "a" {
 	afterVal, err := plans.NewDynamicValue(cty.ObjectVal(map[string]cty.Value{
 		"id":     cty.StringVal("id val"),
 		"secret": cty.StringVal("val"),
+		"input":  cty.NullVal(cty.String),
 	}), objTy)
 	if err != nil {
 		t.Fatalf("unexpected error creating after val: %s", err)

internal/tofu/context_test.go

@@ -766,6 +766,10 @@ func testProviderSchema(name string) *providers.GetProviderSchemaResponse {
 						Optional: true,
 						Computed: true,
 					},
+					"input": {
+						Type:     cty.String,
+						Optional: true,
+					},
 				},
 			},
 		},

internal/tofu/transform_diff.go

@@ -93,7 +93,7 @@ func (t *DiffTransformer) Transform(_ context.Context, g *Graph) error {
 		// Depending on the action we'll need some different combinations of
 		// nodes, because destroying uses a special node type separate from
 		// other actions.
-		var update, delete, forget, createBeforeDestroy bool
+		var update, delete, forget, open, createBeforeDestroy bool
 		switch rc.Action {
 		case plans.NoOp:
 			// For a no-op change we don't take any action but we still
@@ -109,6 +109,8 @@ func (t *DiffTransformer) Transform(_ context.Context, g *Graph) error {
 			update = true
 			delete = true
 			createBeforeDestroy = (rc.Action == plans.CreateThenDelete)
+		case plans.Open:
+			open = true
 		default:
 			update = true
 		}
@@ -162,12 +164,41 @@ func (t *DiffTransformer) Transform(_ context.Context, g *Graph) error {
 			}
 		}
 
+		if open {
+			// Ephemeral resources are always opened, even if the value is not used.
+			// This means that to ensure that the ephemeral resource instance node is created and connected
+			// to the resource node, we create it here.
+			// If we do not have this logic here, ephemeral resource instances that are not used by any other resource
+			// or output would be pruned away during the unused nodes pruning step, and thus not opened.
+			abstract := NewNodeAbstractResourceInstance(addr)
+			var node dag.Vertex = abstract
+			if t.Concrete != nil {
+				node = t.Concrete(abstract)
+			}
+
+			g.Add(node)
+
+			resourceContaining := addr.ContainingResource()
+			// strip the instance key from the module instance
+			resourceAddress := addrs.AbsResource{
+				Module:   resourceContaining.Module.Module().UnkeyedInstanceShim(),
+				Resource: resourceContaining.Resource,
+			}.String()
+
+			// Ensure that the ephemeral resource instance node connects to
+			// the resource node. This is needed to ensure that the ephemeral
+			// expansion node will not get pruned due to having no connections
+			for _, resourceNode := range resourceNodes[resourceAddress] {
+				g.Connect(dag.BasicEdge(node, resourceNode))
+			}
+		}
+
 		if update {
 			// All actions except destroying the node type chosen by t.Concrete
 			abstract := NewNodeAbstractResourceInstance(addr)
 			var node dag.Vertex = abstract
-			if f := t.Concrete; f != nil {
-				node = f(abstract)
+			if t.Concrete != nil {
+				node = t.Concrete(abstract)
 			}
 
 			if createBeforeDestroy {