jprdonnelly/opentf
1
0
Fork 0
mirror of https://github.com/opentffoundation/opentf.git synced 2025-12-19 17:59:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

Rework the way ephemeral variables are used when given on tofu apply command (#3192)

Browse Source 
Signed-off-by: Andrei Ciobanu <andrei.ciobanu@opentofu.org>
Signed-off-by: Christian Mesh <christianmesh1@gmail.com>
Co-authored-by: Christian Mesh <christianmesh1@gmail.com>
This commit is contained in:
Andrei Ciobanu
2025-09-22 09:31:19 +03:00
committed by GitHub
parent 767dd7a2fa
commit cf971eb3b6
22 changed files with 1093 additions and 754 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
internal/backend/backend.go
View File

@@ -218,6 +218,15 @@ type LocalRun struct {
//
// This is nil when we're not applying a saved plan.
Plan *plans.Plan
// ApplyOpts are options that are passed into the Apply operation.
//
// This will be nil most of the times except when the apply command is
// executed with a plan file. In that particular case, the opts will
// contain the variable values given by the user to the `tofu apply`
// command. This is later used to merge the variable values defined in
// the plan with the ones defined in the CLI.
ApplyOpts *tofu.ApplyOpts
}
// An operation represents an operation for OpenTofu to execute.

2
internal/backend/local/backend_apply.go
View File

@@ -270,7 +270,7 @@ func (b *Local) opApply(
defer panicHandler()
defer close(doneCh)
log.Printf("[INFO] backend/local: apply calling Apply")
applyState, applyDiags = lr.Core.Apply(ctx, plan, lr.Config)
applyState, applyDiags = lr.Core.Apply(ctx, plan, lr.Config, lr.ApplyOpts)
}()
if b.opWait(doneCh, stopCtx, cancelCtx, lr.Core, opState, op.View) {

89
internal/backend/local/backend_local.go
View File

@@ -12,7 +12,6 @@ import (
"sort"
"strings"
"github.com/hashicorp/hcl/v2"
"github.com/zclconf/go-cty/cty"
"github.com/opentofu/opentofu/internal/backend"
@@ -278,48 +277,12 @@ func (b *Local) localRunForPlanFile(ctx context.Context, op *backend.Operation,
}
run.Config = config
// When the configuration contains ephemeral variables in the root module, we need
// to populate the values of those inside the plan with the values given in the
// current run.
epv, epvDiags := generateEphemeralPlanValues(op.Variables, config.Module.Variables)
diags = diags.Append(epvDiags)
if diags.HasErrors() {
return nil, snap, diags
}
diags = diags.Append(plan.StoreEphemeralVariablesValues(epv))
if diags.HasErrors() {
return nil, snap, diags
}
// Check that all provided variables are in the configuration
_, undeclaredDiags := backend.ParseUndeclaredVariableValues(op.Variables, config.Module.Variables)
diags = diags.Append(undeclaredDiags)
// Check that all variables provided match
for varName, varCfg := range config.Module.Variables {
if _, ok := op.Variables[varName]; ok {
// Variable provided via cli/files/env/etc...
inputValue, inputDiags := op.RootCall.Variables()(varCfg)
// Variable provided via the plan
planValue, planDiags := subCall.Variables()(varCfg)
diags = diags.Append(inputDiags).Append(planDiags)
if inputDiags.HasErrors() || planDiags.HasErrors() {
return nil, snap, diags
}
if inputValue.Equals(planValue).False() {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Mismatch between input and plan variable value",
fmt.Sprintf("Value saved in the plan file for variable %q is different from the one given to the current command.", varName),
))
}
}
}
if diags.HasErrors() {
return nil, snap, diags
}
declaredVars, declaredDiags := backend.ParseDeclaredVariableValues(op.Variables, config.Module.Variables)
diags = diags.Append(declaredDiags)
run.ApplyOpts = &tofu.ApplyOpts{SetVariables: declaredVars}
// NOTE: We're intentionally comparing the current locks with the
// configuration snapshot, rather than the lock snapshot in the plan file,
@@ -416,52 +379,6 @@ func (b *Local) localRunForPlanFile(ctx context.Context, op *backend.Operation,
return run, snap, diags
}
// generateEphemeralPlanValues converts the user given variables into a format processable
// by plans.Plan#StoreEphemeralVariablesValues.
// This is needed because the ephemeral variables' values are not stored in the plan when saving it
// after a `tofu plan -out <planfile>` run.
// Therefore, for the ephemeral values that are required to be passed during plan creation,
// we need to process those variables during apply too.
func generateEphemeralPlanValues(vv map[string]backend.UnparsedVariableValue, vcfgs map[string]*configs.Variable) (map[string]cty.Value, tfdiags.Diagnostics) {
var diags tfdiags.Diagnostics
parsedVars, varsParsingDiags := backend.ParseDeclaredVariableValues(vv, vcfgs)
diags = diags.Append(varsParsingDiags)
if diags.HasErrors() {
return nil, diags
}
ephemeralVars, ephemeralDiags := ephemeralValuesForPlanFromVariables(parsedVars, vcfgs)
diags = diags.Append(ephemeralDiags)
return ephemeralVars, diags
}
// ephemeralValuesForPlanFromVariables is creating a map ready to be given to the plan to merge these together
// with the variables that are already in the plan.
// This function is handling only the ephemeral variables since those are the only ones that are not
// stored in the plan, so the only way to pass then into the apply phase is to provide them again
// in the -var/-var-file.
func ephemeralValuesForPlanFromVariables(parsedVars tofu.InputValues, variables map[string]*configs.Variable) (map[string]cty.Value, tfdiags.Diagnostics) {
var diags tfdiags.Diagnostics
out := make(map[string]cty.Value)
for vn, vc := range variables {
if !vc.Ephemeral {
log.Printf("[TRACE] variable %q is not ephemeral so not processing it to store in the plan", vn)
continue
}
vv, ok := parsedVars[vn]
if !ok {
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "No value for required variable",
Detail: fmt.Sprintf("Variable %q is configured as ephemeral. This type of variables need to be given a value during `tofu plan` and also during `tofu apply`.", vc.Name),
Subject: vc.DeclRange.Ptr(),
})
continue
}
out[vn] = vv.Value
}
return out, diags
}
// interactiveCollectVariables attempts to complete the given existing
// map of variables by interactively prompting for any variables that are
// declared as required but not yet present.

208
internal/backend/local/backend_local_test.go
View File

@@ -12,13 +12,7 @@ import (
"path/filepath"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/hashicorp/hcl/v2"
"github.com/opentofu/opentofu/internal/addrs"
"github.com/opentofu/opentofu/internal/configs"
"github.com/opentofu/opentofu/internal/depsfile"
"github.com/zclconf/go-cty/cty"
ctymsgpack "github.com/zclconf/go-cty/cty/msgpack"
"github.com/opentofu/opentofu/internal/backend"
"github.com/opentofu/opentofu/internal/command/arguments"
@@ -213,200 +207,6 @@ func TestLocalRun_stalePlan(t *testing.T) {
assertBackendStateUnlocked(t, b)
}
func TestLocalRun_ephemeralVariablesLoadedCorrectlyIntoThePlan(t *testing.T) {
configDir := "./testdata/apply-with-vars"
b := TestLocal(t)
_, configLoader, snap := initwd.MustLoadConfigWithSnapshot(t, configDir, "tests")
var (
backendConfigRaw plans.DynamicValue
planPath string
)
{ // create the state and backend config
sf, err := os.Create(b.StatePath)
if err != nil {
t.Fatalf("unexpected error creating state file %s: %s", b.StatePath, err)
}
if err := statefile.Write(statefile.New(states.NewState(), "boop", 3), sf, encryption.StateEncryptionDisabled()); err != nil {
t.Fatalf("unexpected error writing state file: %s", err)
}
// Refresh the state
sm, err := b.StateMgr(t.Context(), "")
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if err := sm.RefreshState(t.Context()); err != nil {
t.Fatalf("unexpected error refreshing state: %s", err)
}
backendConfig := cty.ObjectVal(map[string]cty.Value{
"path": cty.NullVal(cty.String),
"workspace_dir": cty.NullVal(cty.String),
})
backendConfigRaw, err = plans.NewDynamicValue(backendConfig, backendConfig.Type())
if err != nil {
t.Fatal(err)
}
}
{ // create the plan
plan := &plans.Plan{
UIMode: plans.NormalMode,
Changes: plans.NewChanges(),
Backend: plans.Backend{
Type: "local",
Config: backendConfigRaw,
},
PrevRunState: states.NewState(),
PriorState: states.NewState(),
VariableValues: map[string]plans.DynamicValue{
"regular_var": encodeDynamicValueWithType(t, cty.StringVal("regular_var value"), cty.DynamicPseudoType),
},
EphemeralVariables: map[string]bool{"regular_var": false, "ephemeral_var": true},
}
outDir := t.TempDir()
defer os.RemoveAll(outDir)
planPath = filepath.Join(outDir, "plan.tfplan")
planfileArgs := planfile.CreateArgs{
ConfigSnapshot: snap,
PreviousRunStateFile: statefile.New(plan.PrevRunState, "boop", 1),
StateFile: statefile.New(plan.PriorState, "boop", 3),
Plan: plan,
DependencyLocks: depsfile.NewLocks(),
}
if err := planfile.Create(planPath, planfileArgs, encryption.PlanEncryptionDisabled()); err != nil {
t.Fatalf("unexpected error writing planfile: %s", err)
}
}
streams, _ := terminal.StreamsForTesting(t)
view := views.NewView(streams)
stateLocker := clistate.NewLocker(0, views.NewStateLocker(arguments.ViewHuman, view))
planFile, err := planfile.OpenWrapped(planPath, encryption.PlanEncryptionDisabled())
if err != nil {
t.Fatalf("unexpected error reading planfile: %s", err)
}
cases := map[string]struct {
rootModuleCall configs.StaticModuleCall
givenVars map[string]backend.UnparsedVariableValue
expectedDiags tfdiags.Diagnostics
}{
"ephemeral_var given again in during apply and injected into the plan": {
rootModuleCall: configs.NewStaticModuleCall(addrs.RootModule, func(variable *configs.Variable) (cty.Value, hcl.Diagnostics) {
switch variable.Name {
case "ephemeral_var":
return cty.StringVal("ephemeral_var value"), nil
case "regular_var":
return cty.StringVal("regular_var value"), nil
}
return cty.UnknownVal(cty.DynamicPseudoType), hcl.Diagnostics{}.Append(
&hcl.Diagnostic{Summary: fmt.Sprintf("no variable value defined for %s", variable.Name)},
)
}, "", ""),
givenVars: map[string]backend.UnparsedVariableValue{
"ephemeral_var": unparsedInteractiveVariableValue{
Name: "ephemeral_var",
RawValue: "ephemeral_var value",
},
},
expectedDiags: nil,
},
"ephemeral_var not given in the apply command": {
rootModuleCall: configs.NewStaticModuleCall(addrs.RootModule, func(variable *configs.Variable) (cty.Value, hcl.Diagnostics) {
switch variable.Name {
case "ephemeral_var":
return cty.StringVal("ephemeral_var value"), nil
case "regular_var":
return cty.StringVal("regular_var value"), nil
}
return cty.UnknownVal(cty.DynamicPseudoType), hcl.Diagnostics{}.Append(
&hcl.Diagnostic{Summary: fmt.Sprintf("no variable value defined for %s", variable.Name)},
)
}, "", ""),
givenVars: map[string]backend.UnparsedVariableValue{
// This being empty indicates that the variables have not been given into the args
},
expectedDiags: tfdiags.Diagnostics{}.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "No value for required variable",
Detail: fmt.Sprintf("Variable %q is configured as ephemeral. This type of variables need to be given a value during `tofu plan` and also during `tofu apply`.", "ephemeral_var"),
}),
},
"regular_var given a different value in the apply compared with the one from plan": {
rootModuleCall: configs.NewStaticModuleCall(addrs.RootModule, func(variable *configs.Variable) (cty.Value, hcl.Diagnostics) {
switch variable.Name {
case "ephemeral_var":
return cty.StringVal("ephemeral_var value"), nil
case "regular_var":
return cty.StringVal("different value"), nil
}
return cty.UnknownVal(cty.DynamicPseudoType), hcl.Diagnostics{}.Append(
&hcl.Diagnostic{Summary: fmt.Sprintf("no variable value defined for %s", variable.Name)},
)
}, "", ""),
givenVars: map[string]backend.UnparsedVariableValue{
"ephemeral_var": unparsedInteractiveVariableValue{
Name: "ephemeral_var",
RawValue: "ephemeral_var value",
},
"regular_var": unparsedInteractiveVariableValue{
Name: "regular_var",
RawValue: "different value",
},
},
expectedDiags: tfdiags.Diagnostics{}.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Mismatch between input and plan variable value",
Detail: fmt.Sprintf(`Value saved in the plan file for variable %q is different from the one given to the current command.`, "regular_var"),
}),
},
}
for name, tt := range cases {
t.Run(name, func(t *testing.T) {
op := &backend.Operation{
ConfigDir: configDir,
ConfigLoader: configLoader,
PlanFile: planFile,
Workspace: backend.DefaultStateName,
StateLocker: stateLocker,
RootCall: tt.rootModuleCall,
DependencyLocks: depsfile.NewLocks(),
Variables: tt.givenVars,
}
// always unlock state after test done
defer func() {
_ = op.StateLocker.Unlock()
}()
_, _, diags := b.LocalRun(context.Background(), op)
if got, want := len(diags), len(tt.expectedDiags); got != want {
t.Fatalf("expected to have %d diags but got %d", want, got)
}
for i, gotDiag := range diags {
wantDiag := tt.expectedDiags[i]
if diff := cmp.Diff(wantDiag.Description(), gotDiag.Description()); diff != "" {
t.Errorf("different description of one of the diags.\ndiff:\n%s", diff)
}
if want, got := wantDiag.Severity(), gotDiag.Severity(); want != got {
t.Errorf("different severity of one of the diags. want: %s; got %s", want, got)
}
}
// LocalRun() unlocks the state on failure
if len(diags) > 0 {
assertBackendStateUnlocked(t, b)
} else {
assertBackendStateLocked(t, b)
}
})
}
}
type backendWithStateStorageThatFailsRefresh struct {
}
@@ -481,11 +281,3 @@ func (s *stateStorageThatFailsRefresh) RefreshState(_ context.Context) error {
func (s *stateStorageThatFailsRefresh) PersistState(_ context.Context, schemas *tofu.Schemas) error {
return fmt.Errorf("unimplemented")
}
func encodeDynamicValueWithType(t *testing.T, value cty.Value, ty cty.Type) []byte {
data, err := ctymsgpack.Marshal(value, ty)
if err != nil {
t.Fatalf("failed to marshal cty msgpack value: %s", err)
}
return data
}

38
internal/command/e2etest/primary_test.go
View File

@@ -355,15 +355,23 @@ Changes to Outputs:
}
expectedVal := cty.StringVal("plan_val")
if expectedVal.Equals(varVal).False() {
t.Fatalf("unexpected value saved in the plan object. expected: %s; got: %s", expectedVal.GoString(), varVal.GoString())
t.Errorf("unexpected value saved in the plan object. expected: %s; got: %s", expectedVal.GoString(), varVal.GoString())
}
// no more vars expected in the plan
if got, want := len(plan.VariableValues), 1; got != want {
t.Fatalf("expected to have only %d variables in the plan but got %d: %s", want, got, plan.VariableValues)
// ephemeral variables are to be missing from plan.VariableValues but to be found in plan.EphemeralVariables
varDynVal, ok = plan.VariableValues["ephemeral_input"]
if ok {
t.Errorf("expected variable %q to be missing from plan.VariableValues but got %s", "ephemeral_input", varDynVal)
}
// ensure that the ephemeral variable is registered as expected
if plan.EphemeralVariables != nil {
t.Fatalf("plan.EphemeralVariables is not meant to be initialised when reading the plan since there is no way to say which variable is ephemeral without having the configuration available")
// ensure that plan.EphemeralVariables is registered as expected
if plan.EphemeralVariables == nil {
t.Errorf("plan.EphemeralVariables is meant to be initialised when reading the plan since the empty value variables are marked as ephemeral=true")
}
expectedEphemeralVariables := map[string]bool{
"ephemeral_input": true,
"simple_input": false,
}
if diff := cmp.Diff(plan.EphemeralVariables, expectedEphemeralVariables); diff != "" {
t.Errorf("invalid content of plan.EphemeralVariables: %s", diff)
}
}
@@ -382,6 +390,22 @@ Changes to Outputs:
t.Errorf("expected an error message but didn't get it.\nexpected:\n%s\n\ngot:\n%s\n", expectedToContain, cleanStderr)
}
}
{ // APPLY with no ephemeral variable value
expectedToContain := "╷ Error: No value for required variable on main.tf line 15: 15: variable \"ephemeral_input\" { Variable \"ephemeral_input\" is configured as ephemeral. This type of variables need to be given a value during `tofu plan` and also during `tofu apply`.╵"
expectedErr := fmt.Errorf("exit status 1")
_, stderr, err := tf.Run("apply", `-var=simple_input=plan_val`, "tfplan")
if err == nil {
t.Fatalf("expected an error but got nothing")
}
if got, want := err.Error(), expectedErr.Error(); got != want {
t.Fatalf("expected err %q but got %q", want, got)
}
cleanStderr := SanitizeStderr(stderr)
if cleanStderr != expectedToContain {
t.Errorf("expected an error message but didn't get it.\nexpected:\n%s\n\ngot:\n%s\n", expectedToContain, cleanStderr)
}
}
{ // APPLY
stdout, stderr, err := tf.Run("apply", `-var=simple_input=plan_val`, `-var=ephemeral_input=ephemeral_val`, "tfplan")
if err != nil {

2
internal/command/test.go
View File

@@ -907,7 +907,7 @@ func (runner *TestFileRunner) apply(ctx context.Context, plan *plans.Plan, state
defer panicHandler()
defer done()
log.Printf("[DEBUG] TestFileRunner: starting apply for %s/%s", file.Name, run.Name)
updated, applyDiags = tfCtx.Apply(ctx, plan, config)
updated, applyDiags = tfCtx.Apply(ctx, plan, config, nil)
log.Printf("[DEBUG] TestFileRunner: completed apply for %s/%s", file.Name, run.Name)
}()
waitDiags, cancelled := runner.wait(tfCtx, runningCtx, run, file, created)

21
internal/plans/internal/planproto/planfile.pb.go
View File

@@ -406,9 +406,12 @@ type Plan struct {
// ResourceDrift which may have contributed to the plan changes.
RelevantAttributes []*PlanResourceAttr `protobuf:"bytes,15,rep,name=relevant_attributes,json=relevantAttributes,proto3" json:"relevant_attributes,omitempty"`
// timestamp is the record of truth for when the plan happened.
Timestamp string `protobuf:"bytes,21,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
Timestamp string `protobuf:"bytes,21,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
// EphemeralVariables records the ephemeral variables later used
// be able to validate the values for these during the apply command.
EphemeralVariables []string `protobuf:"bytes,22,rep,name=ephemeral_variables,json=ephemeralVariables,proto3" json:"ephemeral_variables,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *Plan) Reset() {
@@ -546,6 +549,13 @@ func (x *Plan) GetTimestamp() string {
return ""
}
func (x *Plan) GetEphemeralVariables() []string {
if x != nil {
return x.EphemeralVariables
}
return nil
}
// Backend is a description of backend configuration and other related settings.
type Backend struct {
state protoimpl.MessageState `protogen:"open.v1"`
@@ -1331,7 +1341,7 @@ var File_planfile_proto protoreflect.FileDescriptor
const file_planfile_proto_rawDesc = "" +
"\n" +
"\x0eplanfile.proto\x12\x06tfplan\"\x84\a\n" +
"\x0eplanfile.proto\x12\x06tfplan\"\xb5\a\n" +
"\x04Plan\x12\x18\n" +
"\aversion\x18\x01 \x01(\x04R\aversion\x12%\n" +
"\aui_mode\x18\x11 \x01(\x0e2\f.tfplan.ModeR\x06uiMode\x12\x18\n" +
@@ -1347,7 +1357,8 @@ const file_planfile_proto_rawDesc = "" +
"\x11terraform_version\x18\x0e \x01(\tR\x10terraformVersion\x12)\n" +
"\abackend\x18\r \x01(\v2\x0f.tfplan.BackendR\abackend\x12K\n" +
"\x13relevant_attributes\x18\x0f \x03(\v2\x1a.tfplan.Plan.resource_attrR\x12relevantAttributes\x12\x1c\n" +
"\ttimestamp\x18\x15 \x01(\tR\ttimestamp\x1aR\n" +
"\ttimestamp\x18\x15 \x01(\tR\ttimestamp\x12/\n" +
"\x13ephemeral_variables\x18\x16 \x03(\tR\x12ephemeralVariables\x1aR\n" +
"\x0eVariablesEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12*\n" +
"\x05value\x18\x02 \x01(\v2\x14.tfplan.DynamicValueR\x05value:\x028\x01\x1aM\n" +

4
internal/plans/internal/planproto/planfile.proto
View File

@@ -95,6 +95,10 @@ message Plan {
// timestamp is the record of truth for when the plan happened.
string timestamp = 21;
// EphemeralVariables records the ephemeral variables later used
// be able to validate the values for these during the apply command.
repeated string ephemeral_variables = 22;
}
// Mode describes the planning mode that created the plan.

53
internal/plans/plan.go
View File

@@ -6,8 +6,6 @@
package plans
import (
"fmt"
"log"
"sort"
"time"
@@ -46,13 +44,13 @@ type Plan struct {
// checked carefully against existing destroy behaviors.
UIMode Mode
// EphemeralVariables is meant is used to determine what variables should be stored
// into the plan and which shouldn't.
// Those marked as "true" should be skipped from writing into the plan.
// Later, when loading the plan from the file, this map will stay nil since there is
// no way to determine what ephemeral variables are in the configuration without
// having direct access to the config objects.
// This is later populated, before executing the apply phase.
// EphemeralVariables is used to determine what variables should
// have their values stored into the plan and which shouldn't.
// Those marked as "true" should be written with a nil value.
// Later, when loading the plan from the file, this map will be populated
// with the same variable names found in the plan file.
// The variables with a null value will be stored as ephemeral=true and
// any other with ephemeral=false.
EphemeralVariables map[string]bool
VariableValues map[string]DynamicValue
@@ -240,43 +238,6 @@ func (plan *Plan) VariableMapper() configs.StaticModuleVariables {
}
}
// StoreEphemeralVariablesValues converts the given values and store them in Plan.VariableValues for being
// able to return those from VariableMapper.
// It also stores the names of the ephemeral variables in a separate map for being able to ignore
// those later if the plan needs to be persisted again.
//
// This function is mainly used when applying a saved plan.
// Since the ephemeral variables are ignored during storing the plan, we need to
// still provide values for those when the plan is applied.
func (plan *Plan) StoreEphemeralVariablesValues(vars map[string]cty.Value) (diags hcl.Diagnostics) {
if plan.EphemeralVariables == nil {
plan.EphemeralVariables = map[string]bool{}
}
for vn, vv := range vars {
if _, ok := plan.VariableValues[vn]; ok {
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Ephemeral variable found in plan",
Detail: fmt.Sprintf("Variable value %q found in the plan. This is an error on OpenTofu's side. Please report this", vn),
})
continue
}
log.Printf("[TRACE] ephemeral variable %q value restored into the plan", vn)
vdv, err := NewDynamicValue(vv, cty.DynamicPseudoType)
if err != nil {
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to prepare variable value for plan",
Detail: fmt.Sprintf("The value for ephemeral variable %q could not be serialized to restore in the plan: %s.", vn, err),
})
continue
}
plan.VariableValues[vn] = vdv
plan.EphemeralVariables[vn] = true // true because this method should be called with values **only** for ephemeral variables
}
return diags
}
// Backend represents the backend-related configuration and other data as it
// existed when a plan was created.
type Backend struct {

15
internal/plans/planfile/planfile_test.go
View File

@@ -67,7 +67,9 @@ func TestRoundtrip(t *testing.T) {
DriftedResources: []*plans.ResourceInstanceChangeSrc{},
VariableValues: map[string]plans.DynamicValue{
"foo": plans.DynamicValue([]byte("foo placeholder")),
"bar": plans.DynamicValue([]byte("bar placeholder")),
},
EphemeralVariables: map[string]bool{"foo": false, "bar": true},
Backend: plans.Backend{
Type: "local",
Config: plans.DynamicValue([]byte("config placeholder")),
@@ -84,6 +86,17 @@ func TestRoundtrip(t *testing.T) {
PrevRunState: prevStateFileIn.State,
PriorState: stateFileIn.State,
}
// By deleting the ephemeral variable from the VariableValues, we change the given plan
// to match a plan read from the file. While loading the plan, the variables stored with
// a null value are only stored in EphemeralVariables but not in VariableValues.
deleteEphemeralVarFromVars := func(in plans.Plan) *plans.Plan {
for vn, eph := range in.EphemeralVariables {
if eph {
delete(in.VariableValues, vn)
}
}
return &in
}
locksIn := depsfile.NewLocks()
locksIn.SetProvider(
@@ -125,7 +138,7 @@ func TestRoundtrip(t *testing.T) {
if err != nil {
t.Fatalf("failed to read plan: %s", err)
}
if diff := cmp.Diff(planIn, planOut); diff != "" {
if diff := cmp.Diff(deleteEphemeralVarFromVars(*planIn), planOut); diff != "" {
t.Errorf("plan did not survive round-trip\n%s", diff)
}
})

22
internal/plans/planfile/tfplan.go
View File

@@ -58,7 +58,8 @@ func readTfplan(r io.Reader) (*plans.Plan, error) {
}
plan := &plans.Plan{
VariableValues: map[string]plans.DynamicValue{},
VariableValues: map[string]plans.DynamicValue{},
EphemeralVariables: map[string]bool{},
Changes: &plans.Changes{
Outputs: []*plans.OutputChangeSrc{},
Resources: []*plans.ResourceInstanceChangeSrc{},
@@ -245,6 +246,11 @@ func readTfplan(r io.Reader) (*plans.Plan, error) {
return nil, fmt.Errorf("invalid value for input variable %q: %w", name, err)
}
plan.VariableValues[name] = val
plan.EphemeralVariables[name] = false
}
// Record the ephemeral variables in the map used later to process these.
for _, name := range rawPlan.EphemeralVariables {
plan.EphemeralVariables[name] = true
}
if rawBackend := rawPlan.Backend; rawBackend == nil {
@@ -473,7 +479,6 @@ func valueFromTfplan(rawV *planproto.DynamicValue) (plans.DynamicValue, error) {
if len(rawV.Msgpack) == 0 { // len(0) because that's the default value for a "bytes" in protobuf
return nil, fmt.Errorf("dynamic value does not have msgpack serialization")
}
return plans.DynamicValue(rawV.Msgpack), nil
}
@@ -491,11 +496,12 @@ func writeTfplan(plan *plans.Plan, w io.Writer) error {
Version: tfplanFormatVersion,
TerraformVersion: version.String(),
Variables: map[string]*planproto.DynamicValue{},
OutputChanges: []*planproto.OutputChange{},
CheckResults: []*planproto.CheckResults{},
ResourceChanges: []*planproto.ResourceInstanceChange{},
ResourceDrift: []*planproto.ResourceInstanceChange{},
Variables: map[string]*planproto.DynamicValue{},
EphemeralVariables: []string{},
OutputChanges: []*planproto.OutputChange{},
CheckResults: []*planproto.CheckResults{},
ResourceChanges: []*planproto.ResourceInstanceChange{},
ResourceDrift: []*planproto.ResourceInstanceChange{},
}
rawPlan.Errored = plan.Errored
@@ -630,6 +636,8 @@ func writeTfplan(plan *plans.Plan, w io.Writer) error {
for name, val := range plan.VariableValues {
if is, ok := plan.EphemeralVariables[name]; ok && is {
// We want to store only the names of the ephemeral variables to be able to restore this map later.
rawPlan.EphemeralVariables = append(rawPlan.EphemeralVariables, name)
continue
}
rawPlan.Variables[name] = valueToTfplan(val)

8
internal/plans/planfile/tfplan_test.go
View File

@@ -32,9 +32,7 @@ func TestTFPlanRoundTrip(t *testing.T) {
"bar": mustNewDynamicValueStr("bar value"),
"baz": mustNewDynamicValueStr("baz value"),
},
// foo omitted on purpose to ensure that ephemeral values filtering works well and does not break
// existing functionality
EphemeralVariables: map[string]bool{"bar": false, "baz": true},
EphemeralVariables: map[string]bool{"bar": false, "baz": true, "foo": false},
Changes: &plans.Changes{
Outputs: []*plans.OutputChangeSrc{
{
@@ -336,10 +334,8 @@ func TestTFPlanRoundTrip(t *testing.T) {
})
plan.Changes.Resources[i].After = nil
plan.Changes.Resources[i].Before = nil
// remove the variables that are meant to be skipped from writing into the plan so we expect the
// read plan to not contain those
// delete the variables that are meant to be written only with the name but loaded only in the plan.EphemeralVariables
delete(plan.VariableValues, "baz")
plan.EphemeralVariables = nil
}
newPlan, err := readTfplan(&buf)

4
internal/tofu/bench_many_instances_test.go
View File

@@ -179,7 +179,7 @@ func BenchmarkManyResourceInstances(b *testing.B) {
plan, planDiags := tofuCtx.Plan(ctx, m, priorState, planOpts)
assertNoDiagnostics(b, planDiags)
_, applyDiags := tofuCtx.Apply(ctx, plan, m)
_, applyDiags := tofuCtx.Apply(ctx, plan, m, nil)
assertNoDiagnostics(b, applyDiags)
}
}
@@ -253,7 +253,7 @@ func BenchmarkManyModuleInstances(b *testing.B) {
plan, planDiags := tofuCtx.Plan(ctx, m, states.NewState(), planOpts)
assertNoDiagnostics(b, planDiags)
_, applyDiags := tofuCtx.Apply(ctx, plan, m)
_, applyDiags := tofuCtx.Apply(ctx, plan, m, nil)
assertNoDiagnostics(b, applyDiags)
}
}

187
internal/tofu/context_apply.go
View File

@@ -10,6 +10,7 @@ import (
"fmt"
"log"
"github.com/hashicorp/hcl/v2"
"github.com/zclconf/go-cty/cty"
otelAttr "go.opentelemetry.io/otel/attribute"
otelTrace "go.opentelemetry.io/otel/trace"
@@ -22,6 +23,23 @@ import (
"github.com/opentofu/opentofu/internal/tracing"
)
// ApplyOpts are the various options that affect the details of how OpenTofu
// will build a plan.
//
// This structure is created from the PlanOpts since wants some functionality
// that PlanOpts already have.
type ApplyOpts struct {
// SetVariables are the raw values for root module variables as provided
// by the user who is requesting the run, prior to any normalization or
// substitution of defaults.
// In localRunForPlanFile, where the initialization of this was initially
// introduced, there is a validation to ensure that the values from the cli
// do not differ from the ones saved in the plan. The place where this is used,
// in Context#mergePlanAndApplyVariables, the merging of this with the plan variable values
// follows the same logic and rules of the validation mentioned above.
SetVariables InputValues
}
// Apply performs the actions described by the given Plan object and returns
// the resulting updated state.
//
@@ -30,7 +48,7 @@ import (
//
// Even if the returned diagnostics contains errors, Apply always returns the
// resulting state which is likely to have been partially-updated.
func (c *Context) Apply(ctx context.Context, plan *plans.Plan, config *configs.Config) (*states.State, tfdiags.Diagnostics) {
func (c *Context) Apply(ctx context.Context, plan *plans.Plan, config *configs.Config, opts *ApplyOpts) (*states.State, tfdiags.Diagnostics) {
defer c.acquireRun("apply")()
log.Printf("[DEBUG] Building and walking apply graph for %s plan", plan.UIMode)
@@ -89,7 +107,7 @@ func (c *Context) Apply(ctx context.Context, plan *plans.Plan, config *configs.C
providerFunctionTracker := make(ProviderFunctionMapping)
graph, operation, diags := c.applyGraph(ctx, plan, config, providerFunctionTracker)
graph, operation, diags := c.applyGraph(ctx, plan, config, providerFunctionTracker, opts)
if diags.HasErrors() {
return nil, diags
}
@@ -155,45 +173,15 @@ Note that the -target and -exclude options are not suitable for routine use, and
return newState, diags
}
func (c *Context) applyGraph(ctx context.Context, plan *plans.Plan, config *configs.Config, providerFunctionTracker ProviderFunctionMapping) (*Graph, walkOperation, tfdiags.Diagnostics) {
func (c *Context) applyGraph(ctx context.Context, plan *plans.Plan, config *configs.Config, providerFunctionTracker ProviderFunctionMapping, applyOpts *ApplyOpts) (*Graph, walkOperation, tfdiags.Diagnostics) {
var diags tfdiags.Diagnostics
variables := InputValues{}
for name, dyVal := range plan.VariableValues {
val, err := dyVal.Decode(cty.DynamicPseudoType)
if err != nil {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid variable value in plan",
fmt.Sprintf("Invalid value for variable %q recorded in plan file: %s.", name, err),
))
continue
}
variables[name] = &InputValue{
Value: val,
SourceType: ValueFromPlan,
}
}
variables, vDiags := c.mergePlanAndApplyVariables(config, plan, applyOpts)
diags = diags.Append(vDiags)
if diags.HasErrors() {
return nil, walkApply, diags
}
// The plan.VariableValues field only records variables that were actually
// set by the caller in the PlanOpts, so we may need to provide
// placeholders for any other variables that the user didn't set, in
// which case OpenTofu will once again use the default value from the
// configuration when we visit these variables during the graph walk.
for name := range config.Module.Variables {
if _, ok := variables[name]; ok {
continue
}
variables[name] = &InputValue{
Value: cty.NilVal,
SourceType: ValueFromPlan,
}
}
operation := walkApply
if plan.UIMode == plans.DestroyMode {
// FIXME: Due to differences in how objects must be handled in the
@@ -242,7 +230,134 @@ func (c *Context) ApplyGraphForUI(plan *plans.Plan, config *configs.Config) (*Gr
var diags tfdiags.Diagnostics
graph, _, moreDiags := c.applyGraph(context.TODO(), plan, config, make(ProviderFunctionMapping))
graph, _, moreDiags := c.applyGraph(context.TODO(), plan, config, make(ProviderFunctionMapping), nil)
diags = diags.Append(moreDiags)
return graph, diags
}
// mergePlanAndApplyVariables is meant to prepare InputValues for the apply phase.
//
// # Context:
// As requested in opentofu/opentofu#1922, we had to add the ability to specify variable's values
// during the apply too, not only during the plan command.
// Therefore, when a plan is created via `tofu plan -out <planfile>` and then applied with `tofu apply <planfile>`
// we need to be able to specify -var/-var-file/etc to allow configuring the variables that are not kept in the
// plan (encryption configuration, ephemeral variables).
//
// # mergePlanAndApplyVariables
// This gets the plan and the *ApplyOpts and builds the InputValues. The values saved in the plan have
// priority *when defined*, but the variables marked as ephemeral in the plan and values for those are searched in the ApplyOpts.
// The implementation is an incremental check from the basic value to the most specific one:
// * First, the initial value is cty.NilVal that will force later the variable node to check for its default value
// * Second, it tries to find the value of the variable in the ApplyOpts#SetVariables, and if it does, it overrides the value from the previous step with it
// * Third, it tries to find the value of the variable in the plans.Plan#VariableValues, and if it does, it overrides the value from the previous step with it
// * Last, it executed two validations to ensure that the resulted value matches its configuration and the plan content.
func (c *Context) mergePlanAndApplyVariables(config *configs.Config, plan *plans.Plan, opts *ApplyOpts) (InputValues, tfdiags.Diagnostics) {
var diags tfdiags.Diagnostics
variables := map[string]*InputValue{}
var inputVars map[string]*InputValue
if opts != nil && opts.SetVariables != nil {
inputVars = opts.SetVariables
}
// Check for variables not in configuration (bug)
for name := range plan.VariableValues {
if _, ok := config.Module.Variables[name]; !ok {
// This should already be validated elsewhere, but we have this here just in case
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Missing variable in configuration",
fmt.Sprintf("Plan variable %q not found in the given configuration", name),
))
}
}
for name := range inputVars {
if _, ok := config.Module.Variables[name]; !ok {
// This should already be validated elsewhere, but we have this here just in case
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Missing variable in configuration",
fmt.Sprintf("Variable %q not found in the given configuration", name),
))
}
}
// no reason to process more if there are already errors
if diags.HasErrors() {
return nil, diags
}
for name, cfg := range config.Module.Variables {
// The plan.VariableValues field only records variables that were actually
// set by the caller in the PlanOpts, so we may need to provide
// placeholders for any other variables that the user didn't set, in
// which case OpenTofu will once again use the default value from the
// configuration when we visit these variables during the graph walk.
variables[name] = &InputValue{
Value: cty.NilVal,
SourceType: ValueFromPlan,
}
// Pull the var value from the input vars
var inputValue cty.Value
inputVar, inputOk := inputVars[name]
if inputOk {
inputValue = inputVar.Value
// Record the var in our return value
variables[name] = inputVar
}
// Pull the var value from the plan vars
var planValue cty.Value
planVar, planOk := plan.VariableValues[name]
if planOk {
val, err := planVar.Decode(cty.DynamicPseudoType)
if err != nil {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid variable value in plan",
fmt.Sprintf("Invalid value for variable %q recorded in plan file: %s.", name, err),
))
continue
}
planValue = val
// Record the var in our return value (potentially overriding the above set)
variables[name] = &InputValue{
Value: val,
SourceType: ValueFromPlan,
}
}
// If both are set, ensure they are identical.
// This is applicable only for non-ephemeral variables, ephemeral values can be only in one of the source at once:
// * Will be in the plan when `tofu apply` will be executed without a plan file
// * Will be in the applyOpts when `tofu apply` will be executed with a plan file
if planOk && inputOk {
if inputValue.Equals(planValue).False() {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Mismatch between input and plan variable value",
fmt.Sprintf("Value saved in the plan file for variable %q is different from the one given to the current command.", name),
))
continue
}
}
// If an ephemeral variable have no default value configured and there is no value for it in plan or input,
// then the value for this is required so ask for it.
if plan.EphemeralVariables[name] && cfg.Required() && !inputOk && !planOk {
// Ephemeral variables are not saved into the plan so these need to be passed during the apply too.
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: `No value for required variable`,
Detail: fmt.Sprintf("Variable %q is configured as ephemeral. This type of variables need to be given a value during `tofu plan` and also during `tofu apply`.", name),
Subject: cfg.DeclRange.Ptr(),
})
continue
}
}
return variables, diags
}

666
internal/tofu/context_apply2_test.go
View File

File diff suppressed because it is too large Load Diff

2
internal/tofu/context_apply_checks_test.go
View File

@@ -738,7 +738,7 @@ check "error" {
test.providerHook(test.provider)
}
state, diags := ctx.Apply(context.Background(), plan, configs)
state, diags := ctx.Apply(context.Background(), plan, configs, nil)
if validateCheckDiagnostics(t, "apply", test.applyWarning, test.applyError, diags) {
return
}

490
internal/tofu/context_apply_test.go
View File

File diff suppressed because it is too large Load Diff

8
internal/tofu/context_input_test.go
View File

@@ -124,7 +124,7 @@ func TestContext2Input_provider(t *testing.T) {
plan, diags := ctx.Plan(context.Background(), m, states.NewState(), DefaultPlanOpts)
assertNoErrors(t, diags)
if _, diags := ctx.Apply(context.Background(), plan, m); diags.HasErrors() {
if _, diags := ctx.Apply(context.Background(), plan, m, nil); diags.HasErrors() {
t.Fatalf("apply errors: %s", diags.Err())
}
@@ -212,7 +212,7 @@ func TestContext2Input_providerMulti(t *testing.T) {
return p, nil
}
if _, diags := ctx.Apply(context.Background(), plan, m); diags.HasErrors() {
if _, diags := ctx.Apply(context.Background(), plan, m, nil); diags.HasErrors() {
t.Fatalf("apply errors: %s", diags.Err())
}
@@ -301,7 +301,7 @@ func TestContext2Input_providerOnly(t *testing.T) {
})
assertNoErrors(t, diags)
state, err := ctx.Apply(context.Background(), plan, m)
state, err := ctx.Apply(context.Background(), plan, m, nil)
if err != nil {
t.Fatalf("err: %s", err)
}
@@ -352,7 +352,7 @@ func TestContext2Input_providerVars(t *testing.T) {
})
assertNoErrors(t, diags)
if _, diags := ctx.Apply(context.Background(), plan, m); diags.HasErrors() {
if _, diags := ctx.Apply(context.Background(), plan, m, nil); diags.HasErrors() {
t.Fatalf("apply errors: %s", diags.Err())
}

2
internal/tofu/context_plan.go
View File

@@ -312,7 +312,7 @@ func (c *Context) checkApplyGraph(ctx context.Context, plan *plans.Plan, config
return nil
}
log.Println("[DEBUG] building apply graph to check for errors")
_, _, diags := c.applyGraph(ctx, plan, config, make(ProviderFunctionMapping))
_, _, diags := c.applyGraph(ctx, plan, config, make(ProviderFunctionMapping), nil)
return diags
}

11
internal/tofu/context_plan2_test.go
View File

@@ -717,7 +717,7 @@ data "test_data_source" "a" {
// This is primarily a plan-time test, since the special handling of
// data resources is a plan-time concern, but we'll still try applying the
// plan here just to make sure it's valid.
newState, diags := ctx.Apply(context.Background(), plan, m)
newState, diags := ctx.Apply(context.Background(), plan, m, nil)
assertNoErrors(t, diags)
if rs := newState.ResourceInstance(dataAddr); rs != nil {
@@ -4718,7 +4718,7 @@ resource "test_object" "b" {
opts := SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables))
plan, diags := ctx.Plan(context.Background(), m, states.NewState(), opts)
assertNoErrors(t, diags)
state, diags := ctx.Apply(context.Background(), plan, m)
state, diags := ctx.Apply(context.Background(), plan, m, nil)
assertNoErrors(t, diags)
// Resource changes which have dependencies across providers which
@@ -8773,9 +8773,10 @@ ephemeral "test_ephemeral_resource" "a" {
}
}
// TestContext2Plan_ephemeralVariablesInPlan checks that the
// ephemeral variables get configured correctly in the plan
// to be used later to exclude values from being written into the plan object.
// TestContext2Plan_ephemeralVariablesInPlan checks that the variables
// are handled correctly. The additional information generated will allow
// the plan writing flow to handle the variables that are not meant to
// have their values stored in the plan (ephemeral variables)
func TestContext2Plan_ephemeralVariablesInPlan(t *testing.T) {
m := testModuleInline(t, map[string]string{
`main.tf`: `

2
internal/tofu/context_test.go
View File

@@ -306,7 +306,7 @@ func TestContext_contextValuesPropagation(t *testing.T) {
plan, diags := tofuCtx.Plan(ctx, m, states.NewState(), DefaultPlanOpts)
assertNoErrors(t, diags)
_, diags = tofuCtx.Apply(ctx, plan, m)
_, diags = tofuCtx.Apply(ctx, plan, m, nil)
assertNoErrors(t, diags)
probe.ExpectReportsFrom(t,

4
internal/tofu/node_module_variable_test.go
View File

@@ -242,7 +242,7 @@ func TestNodeModuleVariableConstraints(t *testing.T) {
}
}
state, diags := ctx.Apply(context.Background(), plan, m)
state, diags := ctx.Apply(context.Background(), plan, m, nil)
assertNoDiagnostics(t, diags)
for _, addr := range checkableObjects {
result := state.CheckResults.GetObjectResult(addr)
@@ -265,7 +265,7 @@ func TestNodeModuleVariableConstraints(t *testing.T) {
})
assertNoDiagnostics(t, diags)
state, diags = ctx.Apply(context.Background(), plan, m)
state, diags = ctx.Apply(context.Background(), plan, m, nil)
assertNoDiagnostics(t, diags)
for _, addr := range checkableObjects {
result := state.CheckResults.GetObjectResult(addr)