Support resolution of variables for transitive dependencies using parent mod 'args' property

`steampipe mod update` now updates transitive mods
It is now be possible to set values for variables in the current mod using fully qualified variable names. 
Only variables for root mod and top level dependency mods can be set by user
Closes #3533. Closes #3547. Closes #3548. Closes #3549

pkg/error_helpers/utils.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"golang.org/x/exp/maps"
 	"os"
 	"strings"
 
@@ -128,14 +129,15 @@ func CombineErrorsWithPrefix(prefix string, errors ...error) error {
 		}
 	}
 
-	combinedErrorString := []string{prefix}
+	combinedErrorString := map[string]struct{}{prefix: {}}
 	for _, e := range errors {
 		if e == nil {
 			continue
 		}
-		combinedErrorString = append(combinedErrorString, e.Error())
+		combinedErrorString[e.Error()] = struct{}{}
 	}
-	return fmt.Errorf(strings.Join(combinedErrorString, "\n\t"))
+
+	return fmt.Errorf(strings.Join(maps.Keys(combinedErrorString), "\n\t"))
 }
 
 func allErrorsNil(errors ...error) bool {

pkg/modinstaller/mod_installer.go

@@ -201,7 +201,7 @@ func (i *ModInstaller) InstallWorkspaceDependencies(ctx context.Context) (err er
 		log.Println("[TRACE] suppressing mod validation error", validationErrors)
 	}
 
-	// if mod args have been provided, add them to the the workspace mod requires
+	// if mod args have been provided, add them to the workspace mod requires
 	// (this will replace any existing dependencies of same name)
 	if len(i.mods) > 0 {
 		workspaceMod.AddModDependencies(i.mods)
@@ -309,9 +309,10 @@ func (i *ModInstaller) installMods(ctx context.Context, mods []*modconfig.ModVer
 			continue
 		}
 
-		// if the mod is not installed or needs updating, pass shouldUpdate=true into installModDependencesRecursively
+		// if the mod is not installed or needs updating, OR if this is an update command,
+		// pass shouldUpdate=true into installModDependencesRecursively
 		// this ensures that we update any dependencies which have updates available
-		shouldUpdate := modToUse == nil
+		shouldUpdate := modToUse == nil || i.updating()
 		if err := i.installModDependencesRecursively(ctx, requiredModVersion, modToUse, parent, shouldUpdate); err != nil {
 			errors = append(errors, err)
 		}

pkg/steampipeconfig/dependency_path.go

@@ -0,0 +1,28 @@
+package steampipeconfig
+
+import "strings"
+
+const pathSeparator = " -> "
+
+// DependencyPathKey is a string representation of a dependency path
+//   - a set of mod dependencyPath values separated by '->'
+//
+// e.g. local -> github.com/kaidaguerre/steampipe-mod-m1@v3.1.1 -> github.com/kaidaguerre/steampipe-mod-m2@v5.1.1
+type DependencyPathKey string
+
+func newDependencyPathKey(dependencyPath ...string) DependencyPathKey {
+	return DependencyPathKey(strings.Join(dependencyPath, pathSeparator))
+}
+
+func (k DependencyPathKey) GetParent() DependencyPathKey {
+	elements := strings.Split(string(k), pathSeparator)
+	if len(elements) == 1 {
+		return ""
+	}
+	return newDependencyPathKey(elements[:len(elements)-2]...)
+}
+
+// how long is the depdency path
+func (k DependencyPathKey) PathLength() int {
+	return len(strings.Split(string(k), pathSeparator))
+}

pkg/steampipeconfig/errors.go

@@ -0,0 +1,107 @@
+package steampipeconfig
+
+import (
+	"fmt"
+	"github.com/gertd/go-pluralize"
+	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
+	"sort"
+	"strings"
+
+	"github.com/turbot/steampipe/pkg/utils"
+)
+
+type MissingVariableError struct {
+	MissingVariables           []*modconfig.Variable
+	MissingTransitiveVariables map[DependencyPathKey][]*modconfig.Variable
+	workspaceMod               *modconfig.Mod
+}
+
+func NewMissingVarsError(workspaceMod *modconfig.Mod) MissingVariableError {
+	return MissingVariableError{
+		MissingTransitiveVariables: make(map[DependencyPathKey][]*modconfig.Variable),
+		workspaceMod:               workspaceMod,
+	}
+}
+
+func (m MissingVariableError) Error() string {
+	//allMissing := append(m.MissingVariables, m.MissingTransitiveVariables...)
+	missingCount := len(m.MissingVariables)
+	for _, missing := range m.MissingTransitiveVariables {
+		missingCount += len(missing)
+	}
+
+	return fmt.Sprintf("missing %d variable %s:\n%s%s",
+		missingCount,
+		utils.Pluralize("value", missingCount),
+		m.getVariableMissingString(),
+		m.getTransitiveVariableMissingString(),
+	)
+}
+
+func (m MissingVariableError) getVariableMissingString() string {
+	var sb strings.Builder
+
+	varNames := make([]string, len(m.MissingVariables))
+	for i, v := range m.MissingVariables {
+		varNames[i] = m.getVariableName(v)
+	}
+
+	// sort names for top level first
+	sort.Slice(varNames, func(i, j int) bool {
+		if len(strings.Split(varNames[i], ".")) < len(strings.Split(varNames[j], ".")) {
+			return true
+		} else {
+			return false
+		}
+	})
+
+	for _, v := range varNames {
+		sb.WriteString(fmt.Sprintf("\t%s not set\n", v))
+	}
+	return sb.String()
+}
+
+func (m MissingVariableError) getTransitiveVariableMissingString() string {
+	var sb strings.Builder
+	for modPath, missingVars := range m.MissingTransitiveVariables {
+		parentPath := modPath.GetParent()
+		varCount := len(missingVars)
+
+		varNames := make([]string, len(missingVars))
+		for i, v := range missingVars {
+			varNames[i] = m.getVariableName(v)
+		}
+
+		pluralizer := pluralize.NewClient()
+		pluralizer.AddIrregularRule("has", "have")
+		pluralizer.AddIrregularRule("an arg", "args")
+		varsString := strings.Join(varNames, ",")
+
+		sb.WriteString(
+			fmt.Sprintf("\tdependency mod %s cannot be loaded because %s %s %s no value.  Mod %s must pass %s for %s in the `require` block of its mod.sp\n",
+				modPath,
+				pluralizer.Pluralize("variable", varCount, false),
+				varsString,
+				pluralizer.Pluralize("has", varCount, false),
+				parentPath,
+				pluralizer.Pluralize("a value", varCount, false),
+				varsString,
+			))
+
+	}
+	return sb.String()
+}
+
+func (m MissingVariableError) getVariableName(v *modconfig.Variable) string {
+	if v.Mod.Name() == m.workspaceMod.Name() {
+		return v.ShortName
+	}
+	return fmt.Sprintf("%s.%s", v.Mod.ShortName, v.ShortName)
+}
+
+type VariableValidationFailedError struct {
+}
+
+func (m VariableValidationFailedError) Error() string {
+	return "variable validation failed"
+}

pkg/steampipeconfig/inputvars/collect_variables.go

@@ -13,9 +13,7 @@ import (
 	"github.com/spf13/viper"
 	"github.com/turbot/steampipe/pkg/constants"
 	"github.com/turbot/steampipe/pkg/filepaths"
-	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig/var_config"
-	"github.com/turbot/steampipe/pkg/steampipeconfig/parse"
 )
 
 // CollectVariableValues inspects the various places that configuration input variable
@@ -25,7 +23,7 @@ import (
 // This method returns diagnostics relating to the collection of the values,
 // but the values themselves may produce additional diagnostics when finally
 // parsed.
-func CollectVariableValues(workspacePath string, variableFileArgs []string, variablesArgs []string) (map[string]UnparsedVariableValue, error) {
+func CollectVariableValues(workspacePath string, variableFileArgs []string, variablesArgs []string, workspaceModName string) (map[string]UnparsedVariableValue, error) {
 	ret := map[string]UnparsedVariableValue{}
 
 	// First we'll deal with environment variables
@@ -136,60 +134,22 @@ func CollectVariableValues(workspacePath string, variableFileArgs []string, vari
 	}
 
 	// now map any variable names of form <modname>.<variablename> to <modname>.var.<varname>
-	ret = transformVarNames(ret)
+	// also if any var value is qualified with the workspace mod, remove the qualification
+	ret = transformVarNames(ret, workspaceModName)
 	return ret, nil
 }
 
-func CollectVariableValuesFromModRequire(mod *modconfig.Mod, parseCtx *parse.ModParseContext) (InputValues, error) {
-	res := make(InputValues)
-	if mod.Require != nil {
-		for _, depModConstraint := range mod.Require.Mods {
-			// find the loaded dep mod which satisfies this constraint
-			depMod, err := parseCtx.GetLoadedDependencyMod(depModConstraint, mod)
-			if err != nil {
-				return nil, err
-			}
-			if depMod == nil {
-				return nil, fmt.Errorf("dependency mod %s is not loaded", depMod.Name())
-			}
-
-			if args := depModConstraint.Args; args != nil {
-				for varName, varVal := range args {
-					varFullName := fmt.Sprintf("%s.var.%s", depMod.ShortName, varName)
-
-					sourceRange := tfdiags.SourceRange{
-						Filename: mod.Require.DeclRange.Filename,
-						Start: tfdiags.SourcePos{
-							Line:   mod.Require.DeclRange.Start.Line,
-							Column: mod.Require.DeclRange.Start.Column,
-							Byte:   mod.Require.DeclRange.Start.Byte,
-						},
-						End: tfdiags.SourcePos{
-							Line:   mod.Require.DeclRange.End.Line,
-							Column: mod.Require.DeclRange.End.Column,
-							Byte:   mod.Require.DeclRange.End.Byte,
-						},
-					}
-
-					res[varFullName] = &InputValue{
-						Value:       varVal,
-						SourceType:  ValueFromModFile,
-						SourceRange: sourceRange,
-					}
-
-				}
-			}
-		}
-	}
-	return res, nil
-}
-
 // map any variable names of form <modname>.<variablename> to <modname>.var.<varname>
-func transformVarNames(rawValues map[string]UnparsedVariableValue) map[string]UnparsedVariableValue {
+func transformVarNames(rawValues map[string]UnparsedVariableValue, workspaceModName string) map[string]UnparsedVariableValue {
+
 	ret := make(map[string]UnparsedVariableValue, len(rawValues))
 	for k, v := range rawValues {
 		if parts := strings.Split(k, "."); len(parts) == 2 {
-			k = fmt.Sprintf("%s.var.%s", parts[0], parts[1])
+			if parts[0] == workspaceModName {
+				k = parts[1]
+			} else {
+				k = fmt.Sprintf("%s.var.%s", parts[0], parts[1])
+			}
 		}
 		ret[k] = v
 	}

pkg/steampipeconfig/inputvars/input_values.go

@@ -118,25 +118,8 @@ func (vv InputValues) JustValues() map[string]cty.Value {
 	return ret
 }
 
-// DefaultVariableValues returns an InputValues map representing the default
-// values specified for variables in the given configuration map.
-//func DefaultVariableValues(configs map[string]*modconfig.Variable) InputValues {
-//	ret := make(InputValues)
-//	for k, c := range configs {
-//		if c.Default == cty.NilVal {
-//			continue
-//		}
-//		ret[k] = &InputValue{
-//			Value:       c.Default,
-//			SourceType:  ValueFromConfig,
-//			SourceRange: &c.DeclRange,
-//		}
-//	}
-//	return ret
-//}
-
 // SameValues returns true if the given InputValues has the same values as
-// the receiever, disregarding the source types and source ranges.
+// the receiver, disregarding the source types and source ranges.
 //
 // Values are compared using the cty "RawEquals" method, which means that
 // unknown values can be considered equal to one another if they are of the
@@ -308,3 +291,19 @@ func CheckInputVariables(vcs map[string]*modconfig.Variable, vs InputValues) tfd
 
 	return diags
 }
+
+// SetVariableValues determines whether the given variable is a public variable and if so sets its value
+func (vv InputValues) SetVariableValues(m *modconfig.ModVariableMap) {
+	for name, inputValue := range vv {
+		variable, ok := m.PublicVariables[name]
+		// if this variable does not exist in public variables, skip
+		if !ok {
+			// todo warn?
+			continue
+		}
+		variable.SetInputValue(
+			inputValue.Value,
+			inputValue.SourceTypeString(),
+			inputValue.SourceRange)
+	}
+}

pkg/steampipeconfig/inputvars/require_args.go

@@ -0,0 +1,47 @@
+package inputvars
+
+import (
+	"fmt"
+	"github.com/hashicorp/terraform/tfdiags"
+	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
+	"github.com/turbot/steampipe/pkg/steampipeconfig/versionmap"
+)
+
+func CollectVariableValuesFromModRequire(m *modconfig.Mod, lock *versionmap.WorkspaceLock) (InputValues, error) {
+	res := make(InputValues)
+	if m.Require != nil {
+		for _, depModConstraint := range m.Require.Mods {
+			if args := depModConstraint.Args; args != nil {
+				// find the loaded dep mod which satisfies this constraint
+				resolvedConstraint := lock.GetMod(depModConstraint.Name, m)
+				if resolvedConstraint == nil {
+					return nil, fmt.Errorf("dependency mod %s is not loaded", depModConstraint.Name)
+				}
+				for varName, varVal := range args {
+					varFullName := fmt.Sprintf("%s.var.%s", resolvedConstraint.Alias, varName)
+
+					sourceRange := tfdiags.SourceRange{
+						Filename: m.Require.DeclRange.Filename,
+						Start: tfdiags.SourcePos{
+							Line:   m.Require.DeclRange.Start.Line,
+							Column: m.Require.DeclRange.Start.Column,
+							Byte:   m.Require.DeclRange.Start.Byte,
+						},
+						End: tfdiags.SourcePos{
+							Line:   m.Require.DeclRange.End.Line,
+							Column: m.Require.DeclRange.End.Column,
+							Byte:   m.Require.DeclRange.End.Byte,
+						},
+					}
+
+					res[varFullName] = &InputValue{
+						Value:       varVal,
+						SourceType:  ValueFromModFile,
+						SourceRange: sourceRange,
+					}
+				}
+			}
+		}
+	}
+	return res, nil
+}

pkg/steampipeconfig/inputvars/unparsed_value.go

@@ -42,7 +42,7 @@ type UnparsedVariableValue interface {
 // InputValues may be incomplete but will include the subset of variables
 // that were successfully processed, allowing for careful analysis of the
 // partial result.
-func ParseVariableValues(inputValuesUnparsed map[string]UnparsedVariableValue, variablesMap map[string]*modconfig.Variable, depModVarValues InputValues, validate bool) (InputValues, tfdiags.Diagnostics) {
+func ParseVariableValues(inputValuesUnparsed map[string]UnparsedVariableValue, variablesMap map[string]*modconfig.Variable, validate bool) (InputValues, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
 	ret := make(InputValues, len(inputValuesUnparsed))
 
@@ -78,7 +78,7 @@ func ParseVariableValues(inputValuesUnparsed map[string]UnparsedVariableValue, v
 					diags = diags.Append(tfdiags.Sourceless(
 						tfdiags.Warning,
 						"Value for undeclared variable",
-						fmt.Sprintf("The configuration does not declare a variable named %q but a value was found. If you meant to use this value, add a \"variable\" block to the configuration.\n\n.", name), //, val.SourceRange.Filename),
+						fmt.Sprintf("The configuration does not declare a variable named %q but a value was found. If you meant to use this value, add a \"variable\" block to the configuration.\n", name), //, val.SourceRange.Filename),
 					))
 				}
 				seenUndeclaredInFile++
@@ -118,10 +118,6 @@ func ParseVariableValues(inputValuesUnparsed map[string]UnparsedVariableValue, v
 		})
 	}
 
-	// depModVarValues are values of dependency mod variables which are set in the mod file.
-	// default the inputVariables to these values (last resourt)
-	ret.DefaultTo(depModVarValues)
-
 	// By this point we should've gathered all of the required variables
 	// from one of the many possible sources.
 	// We'll now populate any we haven't gathered as their defaults and fail if any of the

pkg/steampipeconfig/load_mod.go

@@ -7,7 +7,6 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/Masterminds/semver/v3"
 	filehelpers "github.com/turbot/go-kit/files"
 	"github.com/turbot/go-kit/helpers"
 	"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
@@ -15,13 +14,14 @@ import (
 	"github.com/turbot/steampipe/pkg/error_helpers"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/parse"
+	"github.com/turbot/steampipe/pkg/steampipeconfig/versionmap"
 )
 
 // LoadMod parses all hcl files in modPath and returns a single mod
 // if CreatePseudoResources flag is set, construct hcl resources for files with specific extensions
 // NOTE: it is an error if there is more than 1 mod defined, however zero mods is acceptable
 // - a default mod will be created assuming there are any resource files
-func LoadMod(modPath string, parseCtx *parse.ModParseContext, opts ...LoadModOption) (mod *modconfig.Mod, errorsAndWarnings *modconfig.ErrorAndWarnings) {
+func LoadMod(modPath string, parseCtx *parse.ModParseContext) (mod *modconfig.Mod, errorsAndWarnings *modconfig.ErrorAndWarnings) {
 	defer func() {
 		if r := recover(); r != nil {
 			errorsAndWarnings = modconfig.NewErrorsAndWarning(helpers.ToError(r))
@@ -33,9 +33,9 @@ func LoadMod(modPath string, parseCtx *parse.ModParseContext, opts ...LoadModOpt
 		return nil, loadModResult
 	}
 
-	// apply opts to mod
-	for _, o := range opts {
-		o(mod)
+	// if this is a dependency mod, initialise the dependency config
+	if parseCtx.DependencyConfig != nil {
+		parseCtx.DependencyConfig.SetModProperties(mod)
 	}
 
 	// set the current mod on the run context
@@ -86,27 +86,25 @@ func loadModDefinition(modPath string, parseCtx *parse.ModParseContext) (mod *mo
 	return mod, errorsAndWarnings
 }
 
-func loadModDependencies(mod *modconfig.Mod, parseCtx *parse.ModParseContext) error {
+func loadModDependencies(parent *modconfig.Mod, parseCtx *parse.ModParseContext) error {
 	var errors []error
-
-	if mod.Require != nil {
+	if parent.Require != nil {
 		// now ensure there is a lock file - if we have any mod dependnecies there MUST be a lock file -
 		// otherwise 'steampipe install' must be run
-		if err := parseCtx.EnsureWorkspaceLock(mod); err != nil {
+		if err := parseCtx.EnsureWorkspaceLock(parent); err != nil {
 			return err
 		}
 
-		for _, requiredModVersion := range mod.Require.Mods {
-			// have we already loaded a mod which satisfied this
-			loadedMod, err := parseCtx.GetLoadedDependencyMod(requiredModVersion, mod)
+		for _, requiredModVersion := range parent.Require.Mods {
+			// get the locked version ofd this dependency
+			lockedVersion, err := parseCtx.WorkspaceLock.GetLockedModVersion(requiredModVersion, parent)
 			if err != nil {
 				return err
 			}
-			if loadedMod != nil {
-				continue
+			if lockedVersion == nil {
+				return fmt.Errorf("not all dependencies are installed - run 'steampipe mod install'")
 			}
-
-			if err := loadModDependency(requiredModVersion, parseCtx); err != nil {
+			if err := loadModDependency(lockedVersion, parseCtx); err != nil {
 				errors = append(errors, err)
 			}
 		}
@@ -115,18 +113,15 @@ func loadModDependencies(mod *modconfig.Mod, parseCtx *parse.ModParseContext) er
 	return error_helpers.CombineErrors(errors...)
 }
 
-func loadModDependency(modDependency *modconfig.ModVersionConstraint, parseCtx *parse.ModParseContext) error {
+func loadModDependency(modDependency *versionmap.ResolvedVersionConstraint, parseCtx *parse.ModParseContext) error {
 	// dependency mods are installed to <mod path>/<mod nam>@version
 	// for example workspace_folder/.steampipe/mods/github.com/turbot/steampipe-mod-aws-compliance@v1.0
 
 	// we need to list all mod folder in the parent folder: workspace_folder/.steampipe/mods/github.com/turbot/
 	// for each folder we parse the mod name and version and determine whether it meets the version constraint
 
-	// we need to iterate through all mods in the parent folder and find one that satisfies requirements
-	parentFolder := filepath.Dir(filepath.Join(parseCtx.WorkspaceLock.ModInstallationPath, modDependency.Name))
-
 	// search the parent folder for a mod installation which satisfied the given mod dependency
-	dependencyDir, version, err := findInstalledDependency(modDependency, parentFolder)
+	dependencyDir, err := parseCtx.WorkspaceLock.FindInstalledDependency(modDependency)
 	if err != nil {
 		return err
 	}
@@ -136,21 +131,19 @@ func loadModDependency(modDependency *modconfig.ModVersionConstraint, parseCtx *
 	parseCtx.ListOptions.Exclude = nil
 	defer func() { parseCtx.ListOptions.Exclude = prevExclusions }()
 
-	childParseCtx := parse.NewChildModParseContext(parseCtx, dependencyDir)
+	childParseCtx := parse.NewChildModParseContext(parseCtx, modDependency, dependencyDir)
 	// NOTE: pass in the version and dependency path of the mod - these must be set before it loads its dependencies
-	mod, errAndWarnings := LoadMod(dependencyDir, childParseCtx, WithDependencyConfig(modDependency.Name, version))
+	dependencyMod, errAndWarnings := LoadMod(dependencyDir, childParseCtx)
 	if errAndWarnings.GetError() != nil {
 		return errAndWarnings.GetError()
 	}
 
 	// update loaded dependency mods
-	parseCtx.AddLoadedDependencyMod(mod)
+	parseCtx.AddLoadedDependencyMod(dependencyMod)
 	if parseCtx.ParentParseCtx != nil {
-		parseCtx.ParentParseCtx.AddLoadedDependencyMod(mod)
 		// add mod resources to parent parse context
-		parseCtx.ParentParseCtx.AddModResources(mod)
+		parseCtx.ParentParseCtx.AddModResources(dependencyMod)
 	}
-
 	return nil
 
 }
@@ -186,51 +179,6 @@ func loadModResources(mod *modconfig.Mod, parseCtx *parse.ModParseContext) (*mod
 	return mod, errAndWarnings
 }
 
-// search the parent folder for a mod installation which satisfied the given mod dependency
-func findInstalledDependency(modDependency *modconfig.ModVersionConstraint, parentFolder string) (string, *semver.Version, error) {
-	shortDepName := filepath.Base(modDependency.Name)
-	entries, err := os.ReadDir(parentFolder)
-	if err != nil {
-		return "", nil, fmt.Errorf("mod satisfying '%s' is not installed", modDependency)
-	}
-
-	// results vars
-	var dependencyPath string
-	var dependencyVersion *semver.Version
-
-	for _, entry := range entries {
-		split := strings.Split(entry.Name(), "@")
-		if len(split) != 2 {
-			// invalid format - ignore
-			continue
-		}
-		modName := split[0]
-		versionString := strings.TrimPrefix(split[1], "v")
-		if modName == shortDepName {
-			v, err := semver.NewVersion(versionString)
-			if err != nil {
-				// invalid format - ignore
-				continue
-			}
-			if modDependency.Constraint.Check(v) {
-				// if there is more than 1 mod which satisfied the dependency, fail (for now)
-				if dependencyVersion != nil {
-					return "", nil, fmt.Errorf("more than one mod found which satisfies dependency %s@%s", modDependency.Name, modDependency.VersionString)
-				}
-				dependencyPath = filepath.Join(parentFolder, entry.Name())
-				dependencyVersion = v
-			}
-		}
-	}
-
-	// did we find a result?
-	if dependencyVersion != nil {
-		return dependencyPath, dependencyVersion, nil
-	}
-
-	return "", nil, fmt.Errorf("mod satisfying '%s' is not installed", modDependency)
-}
-
 // LoadModResourceNames parses all hcl files in modPath and returns the names of all resources
 func LoadModResourceNames(mod *modconfig.Mod, parseCtx *parse.ModParseContext) (resources *modconfig.WorkspaceResources, err error) {
 	defer func() {

pkg/steampipeconfig/load_mod_option.go

@@ -1,19 +0,0 @@
-package steampipeconfig
-
-import (
-	"fmt"
-	"github.com/Masterminds/semver/v3"
-	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
-)
-
-type LoadModOption = func(mod *modconfig.Mod)
-
-func WithDependencyConfig(modDependencyName string, version *semver.Version) LoadModOption {
-	return func(mod *modconfig.Mod) {
-		mod.Version = version
-		// build the ModDependencyPath from the modDependencyName and the version
-		dependencyPath := fmt.Sprintf("%s@v%s", modDependencyName, version.String())
-		mod.DependencyPath = &dependencyPath
-		mod.DependencyName = modDependencyName
-	}
-}

pkg/steampipeconfig/load_mod_variables.go

@@ -12,7 +12,7 @@ import (
 	"github.com/turbot/steampipe/pkg/steampipeconfig/inputvars"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/parse"
-	"github.com/turbot/steampipe/pkg/type_conversion"
+	"github.com/turbot/steampipe/pkg/steampipeconfig/versionmap"
 )
 
 func LoadVariableDefinitions(variablePath string, parseCtx *parse.ModParseContext) (*modconfig.ModVariableMap, error) {
@@ -23,43 +23,25 @@ func LoadVariableDefinitions(variablePath string, parseCtx *parse.ModParseContex
 		return nil, errAndWarnings.GetError()
 	}
 
-	variableMap := modconfig.NewModVariableMap(mod, parseCtx.LoadedDependencyMods)
+	variableMap := modconfig.NewModVariableMap(mod)
 
 	return variableMap, nil
 }
 
 func GetVariableValues(ctx context.Context, parseCtx *parse.ModParseContext, variableMap *modconfig.ModVariableMap, validate bool) (*modconfig.ModVariableMap, error) {
 	// now resolve all input variables
-	inputVariables, err := getInputVariables(variableMap.AllVariables, validate, parseCtx)
+	inputValues, err := getInputVariables(ctx, parseCtx, variableMap, validate)
 	if err != nil {
 		return nil, err
 	}
 
-	if validate {
-		if err := validateVariables(ctx, variableMap.AllVariables, inputVariables); err != nil {
-			return nil, err
-		}
-	}
-
 	// now update the variables map with the input values
-	for name, inputValue := range inputVariables {
-		variable := variableMap.AllVariables[name]
-		variable.SetInputValue(
-			inputValue.Value,
-			inputValue.SourceTypeString(),
-			inputValue.SourceRange)
-
-		// set variable value string in our workspace map
-		variableMap.VariableValues[name], err = type_conversion.CtyToString(inputValue.Value)
-		if err != nil {
-			return nil, err
-		}
-	}
+	inputValues.SetVariableValues(variableMap)
 
 	return variableMap, nil
 }
 
-func getInputVariables(variableMap map[string]*modconfig.Variable, validate bool, parseCtx *parse.ModParseContext) (inputvars.InputValues, error) {
+func getInputVariables(ctx context.Context, parseCtx *parse.ModParseContext, variableMap *modconfig.ModVariableMap, validate bool) (inputvars.InputValues, error) {
 	variableFileArgs := viper.GetStringSlice(constants.ArgVarFile)
 	variableArgs := viper.GetStringSlice(constants.ArgVariable)
 
@@ -67,37 +49,31 @@ func getInputVariables(variableMap map[string]*modconfig.Variable, validate bool
 	mod := parseCtx.CurrentMod
 	path := mod.ModPath
 
-	inputValuesUnparsed, err := inputvars.CollectVariableValues(path, variableFileArgs, variableArgs)
-	if err != nil {
-		return nil, err
-	}
-
-	// build map of dependency mod variable values declared in the mod 'Require' section
-	depModVarValues, err := inputvars.CollectVariableValuesFromModRequire(mod, parseCtx)
+	var inputValuesUnparsed, err = inputvars.CollectVariableValues(path, variableFileArgs, variableArgs, parseCtx.CurrentMod.ShortName)
 	if err != nil {
 		return nil, err
 	}
 
 	if validate {
-		if err := identifyMissingVariables(inputValuesUnparsed, variableMap, depModVarValues); err != nil {
+		if err := identifyAllMissingVariables(parseCtx, variableMap, inputValuesUnparsed); err != nil {
 			return nil, err
 		}
 	}
-	parsedValues, diags := inputvars.ParseVariableValues(inputValuesUnparsed, variableMap, depModVarValues, validate)
+	// only parse values for public variables
+	parsedValues, diags := inputvars.ParseVariableValues(inputValuesUnparsed, variableMap.PublicVariables, validate)
+
+	if validate {
+		moreDiags := inputvars.CheckInputVariables(variableMap.PublicVariables, parsedValues)
+		diags = append(diags, moreDiags...)
+		if diags.HasErrors() {
+			displayValidationErrors(ctx, diags)
+			return nil, VariableValidationFailedError{}
+		}
+	}
 
 	return parsedValues, diags.Err()
 }
 
-func validateVariables(ctx context.Context, variableMap map[string]*modconfig.Variable, variables inputvars.InputValues) error {
-	diags := inputvars.CheckInputVariables(variableMap, variables)
-	if diags.HasErrors() {
-		displayValidationErrors(ctx, diags)
-		// return empty error
-		return modconfig.VariableValidationFailedError{}
-	}
-	return nil
-}
-
 func displayValidationErrors(ctx context.Context, diags tfdiags.Diagnostics) {
 	fmt.Println()
 	for i, diag := range diags {
@@ -110,25 +86,100 @@ func displayValidationErrors(ctx context.Context, diags tfdiags.Diagnostics) {
 	}
 }
 
-func identifyMissingVariables(existing map[string]inputvars.UnparsedVariableValue, vcs map[string]*modconfig.Variable, depModVarValues inputvars.InputValues) error {
+func identifyAllMissingVariables(parseCtx *parse.ModParseContext, variableMap *modconfig.ModVariableMap, variableValues map[string]inputvars.UnparsedVariableValue) error {
+	// convert variableValues into a lookup
+	var variableValueLookup = make(map[string]struct{}, len(variableValues))
+	missingVarsMap, err := identifyMissingVariablesForDependencies(parseCtx.WorkspaceLock, variableMap, variableValueLookup, nil)
+
+	if err != nil {
+		return err
+	}
+	if len(missingVarsMap) == 0 {
+		// all good
+		return nil
+	}
+
+	// build a MissingVariableError
+	missingVarErr := NewMissingVarsError(parseCtx.CurrentMod)
+
+	// build a lookup with the dependency path of the root mod and all top level dependencies
+	rootName := variableMap.Mod.ShortName
+	topLevelModLookup := map[DependencyPathKey]struct{}{DependencyPathKey(rootName): {}}
+	for dep := range parseCtx.WorkspaceLock.InstallCache {
+		depPathKey := newDependencyPathKey(rootName, dep)
+		topLevelModLookup[depPathKey] = struct{}{}
+	}
+	for depPath, missingVars := range missingVarsMap {
+		if _, isTopLevel := topLevelModLookup[depPath]; isTopLevel {
+			missingVarErr.MissingVariables = append(missingVarErr.MissingVariables, missingVars...)
+		} else {
+			missingVarErr.MissingTransitiveVariables[depPath] = missingVars
+		}
+	}
+
+	return missingVarErr
+}
+
+func identifyMissingVariablesForDependencies(workspaceLock *versionmap.WorkspaceLock, variableMap *modconfig.ModVariableMap, parentVariableValuesLookup map[string]struct{}, dependencyPath []string) (map[DependencyPathKey][]*modconfig.Variable, error) {
+	// return a map of missing variables, keyed by dependency path
+	res := make(map[DependencyPathKey][]*modconfig.Variable)
+
+	// update the path to this dependency
+	dependencyPath = append(dependencyPath, variableMap.Mod.GetInstallCacheKey())
+
+	// clone variableValuesLookup so we can mutate it with depdency specific args overrides
+	var variableValueLookup = make(map[string]struct{}, len(parentVariableValuesLookup))
+	for k := range parentVariableValuesLookup {
+		variableValueLookup[k] = struct{}{}
+	}
+
+	// first get any args specified in the mod requires
+	// note the actual value of these may be unknown as we have not yet resolved
+	depModArgs, err := inputvars.CollectVariableValuesFromModRequire(variableMap.Mod, workspaceLock)
+	for varName := range depModArgs {
+		variableValueLookup[varName] = struct{}{}
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	//  handle root variables
+	missingVariables := identifyMissingVariables(variableMap.RootVariables, variableValueLookup)
+	if len(missingVariables) > 0 {
+		res[newDependencyPathKey(dependencyPath...)] = missingVariables
+	}
+
+	// now iterate through all the dependency variable maps
+	for _, dependencyVariableMap := range variableMap.DependencyVariables {
+		childMissingMap, err := identifyMissingVariablesForDependencies(workspaceLock, dependencyVariableMap, variableValueLookup, dependencyPath)
+		if err != nil {
+			return nil, err
+		}
+		// add results into map
+		for k, v := range childMissingMap {
+			res[k] = v
+		}
+	}
+	return res, nil
+}
+
+func identifyMissingVariables(variableMap map[string]*modconfig.Variable, variableValuesLookup map[string]struct{}) []*modconfig.Variable {
+
 	var needed []*modconfig.Variable
 
-	for name, vc := range vcs {
-		if !vc.Required() {
+	for name, v := range variableMap {
+		if !v.Required() {
 			continue // We only prompt for required variables
 		}
-		_, unparsedValExists := existing[name]
-		_, depModVarValueExists := depModVarValues[name]
-		if !unparsedValExists && !depModVarValueExists {
-			needed = append(needed, vc)
+		_, unparsedValExists := variableValuesLookup[name]
+
+		if !unparsedValExists {
+			needed = append(needed, v)
 		}
 	}
 	sort.SliceStable(needed, func(i, j int) bool {
 		return needed[i].Name() < needed[j].Name()
 	})
-	if len(needed) > 0 {
-		return modconfig.MissingVariableError{MissingVariables: needed}
-	}
-	return nil
+	return needed
 
 }

pkg/steampipeconfig/modconfig/mod_variable_map.go

@@ -1,95 +1,67 @@
 package modconfig
 
 import (
-	"sort"
 	"strings"
+
+	"github.com/turbot/steampipe/pkg/type_conversion"
+	"github.com/turbot/steampipe/pkg/utils"
 )
 
 // ModVariableMap is a struct containins maps of variable definitions
 type ModVariableMap struct {
-	RootVariables       map[string]*Variable
-	DependencyVariables map[string]map[string]*Variable
-	// a map of top level AND dependency variables
-	// used to set variable values from inputVariables
-	AllVariables map[string]*Variable
-	// the input variables evaluated in the parse
-	VariableValues map[string]string
+	// which mod have these variables been loaded for?
+	Mod *Mod
+	// top level variables
+	RootVariables map[string]*Variable
+	// map of dependency variable maps, keyed by dependency NAME
+	DependencyVariables map[string]*ModVariableMap
+
+	// a list of the pointers to the variables whose values can be changed
+	// NOTE: this refers to the SAME variable objects as exist in the RootVariables and DependencyVariables maps,
+	// so when we set the value of public variables, we mutate the underlying variable
+	PublicVariables map[string]*Variable
 }
 
 // NewModVariableMap builds a ModVariableMap using the variables from a mod and its dependencies
-func NewModVariableMap(mod *Mod, dependencyMods ModMap) *ModVariableMap {
+func NewModVariableMap(mod *Mod) *ModVariableMap {
 	m := &ModVariableMap{
+		Mod:                 mod,
 		RootVariables:       make(map[string]*Variable),
-		DependencyVariables: make(map[string]map[string]*Variable),
-		VariableValues:      make(map[string]string),
+		DependencyVariables: make(map[string]*ModVariableMap),
 	}
 
 	// add variables into map, modifying the key to be the variable short name
 	for k, v := range mod.ResourceMaps.Variables {
 		m.RootVariables[buildVariableMapKey(k)] = v
 	}
-	// now add variables from dependency mods
-	for dependencyPath, mod := range dependencyMods {
-		// add variables into map, modifying the key to be the variable short name
-		m.DependencyVariables[dependencyPath] = make(map[string]*Variable)
-		for k, v := range mod.ResourceMaps.Variables {
-			m.DependencyVariables[dependencyPath][buildVariableMapKey(k)] = v
+
+	// now traverse all dependency mods
+	for _, depMod := range mod.ResourceMaps.Mods {
+		// todo for some reason the mod appears in its own resource maps?
+		if depMod.Name() != mod.Name() {
+			m.DependencyVariables[depMod.DependencyName] = NewModVariableMap(depMod)
 		}
 	}
-	// build map of all variables
-	m.AllVariables = m.buildCombinedMap()
+
+	// build map of all publicy settable variables
+	m.PopulatePublicVariables()
 
 	return m
 }
 
-// build a map of top level and dependency variables
-// (dependency variables are keyed by full (qualified) name
-func (m ModVariableMap) buildCombinedMap() map[string]*Variable {
-	res := make(map[string]*Variable)
-	for k, v := range m.RootVariables {
-		// add top level vars keyed by short name
-		res[k] = v
-	}
-	for _, dep := range m.DependencyVariables {
-		for _, v := range dep {
-			// add dependency vars keyed by full name
-			res[v.FullName] = v
-		}
-	}
-	return res
-}
-
-func (m ModVariableMap) ToArray() []*Variable {
+func (m *ModVariableMap) ToArray() []*Variable {
 	var res []*Variable
 
-	if len(m.AllVariables) > 0 {
-		var keys []string
-
-		for k := range m.RootVariables {
-			keys = append(keys, k)
-		}
-		// sort keys
-		sort.Strings(keys)
-		for _, k := range keys {
-			res = append(res, m.RootVariables[k])
-		}
+	keys := utils.SortedMapKeys(m.RootVariables)
+	for _, k := range keys {
+		res = append(res, m.RootVariables[k])
 	}
 
 	for _, depVariables := range m.DependencyVariables {
-		if len(depVariables) == 0 {
-			continue
-		}
-		keys := make([]string, len(depVariables))
-		idx := 0
 
-		for k := range depVariables {
-			keys[idx] = k
-			idx++
-		}
-		// sort keys
-		sort.Strings(keys)
+		keys := utils.SortedMapKeys(depVariables.RootVariables)
 		for _, k := range keys {
-			res = append(res, depVariables[k])
+			res = append(res, depVariables.RootVariables[k])
 		}
 	}
 
@@ -102,3 +74,36 @@ func buildVariableMapKey(k string) string {
 	name := strings.TrimPrefix(k, "var.")
 	return name
 }
+
+// PopulatePublicVariables builds a map of top level and dependency variables
+// (dependency variables are keyed by full (qualified) name
+func (m *ModVariableMap) PopulatePublicVariables() {
+	res := make(map[string]*Variable)
+	for k, v := range m.RootVariables {
+		// add top level vars keyed by short name
+		res[k] = v
+	}
+	// copy ROOT variables for each top level dependency
+	for _, depVars := range m.DependencyVariables {
+		for _, v := range depVars.RootVariables {
+			// add dependency vars keyed by full name
+			res[v.FullName] = v
+		}
+	}
+	m.PublicVariables = res
+}
+
+// GetPublicVariableValues converts public variables into a map of string variable values
+func (m *ModVariableMap) GetPublicVariableValues() (map[string]string, error) {
+	res := make(map[string]string, len(m.PublicVariables))
+	for k, v := range m.PublicVariables {
+		// TODO investigate workspace usage of value string and determine whether we can simply format ValueGo
+		valueString, err := type_conversion.CtyToString(v.Value)
+		if err != nil {
+			return nil, err
+		}
+		res[k] = valueString
+	}
+	return res, nil
+}
+

pkg/steampipeconfig/modconfig/variable.go

@@ -2,7 +2,6 @@ package modconfig
 
 import (
 	"fmt"
-
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/terraform/tfdiags"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig/var_config"
@@ -47,7 +46,7 @@ func NewVariable(v *var_config.Variable, mod *Mod) *Variable {
 		defaultGo, _ = type_conversion.CtyToGo(v.Default)
 	}
 	fullName := fmt.Sprintf("%s.var.%s", mod.ShortName, v.Name)
-	return &Variable{
+	res := &Variable{
 		ModTreeItemImpl: ModTreeItemImpl{
 			HclResourceImpl: HclResourceImpl{
 				ShortName:       v.Name,
@@ -59,13 +58,22 @@ func NewVariable(v *var_config.Variable, mod *Mod) *Variable {
 			},
 			Mod: mod,
 		},
-		Default:     v.Default,
+		Default:   v.Default,
+		DefaultGo: defaultGo,
+		// initialise the value to the default - may be set later
+		Value:   v.Default,
+		ValueGo: defaultGo,
+
 		Type:        v.Type,
 		ParsingMode: v.ParsingMode,
 		ModName:     mod.ShortName,
-		DefaultGo:   defaultGo,
 		TypeString:  type_conversion.CtyTypeToHclType(v.Type, v.Default.Type()),
 	}
+	// if no type is set and a default _is_ set, use default to set the type
+	if res.Type.Equals(cty.DynamicPseudoType) && !res.Default.IsNull() {
+		res.Type = res.Default.Type()
+	}
+	return res
 }
 
 func (v *Variable) Equals(other *Variable) bool {
@@ -88,11 +96,6 @@ func (v *Variable) Required() bool {
 }
 
 func (v *Variable) SetInputValue(value cty.Value, sourceType string, sourceRange tfdiags.SourceRange) error {
-	// if no type is set and a default _is_ set, use default to set the type
-	if v.Type.Equals(cty.DynamicPseudoType) && !v.Default.IsNull() {
-		v.Type = v.Default.Type()
-	}
-
 	// if the value type is a tuple with no elem type, and we have a type, set the variable to have our type
 	if value.Type().Equals(cty.Tuple(nil)) && !v.Type.Equals(cty.DynamicPseudoType) {
 		var err error
@@ -112,6 +115,7 @@ func (v *Variable) SetInputValue(value cty.Value, sourceType string, sourceRange
 	if v.TypeString == "" {
 		v.TypeString = type_conversion.CtyTypeToHclType(value.Type())
 	}
+
 	return nil
 }
 

pkg/steampipeconfig/parse/decode_body.go

@@ -47,7 +47,7 @@ func decodeHclBodyIntoStruct(body hcl.Body, evalCtx *hcl.EvalContext, resourcePr
 	diags = append(diags, moreDiags...)
 
 	// resolve any resource references using the resource map, rather than relying on the EvalCtx
-	// (which does not work with nexted struct vals)
+	// (which does not work with nested struct vals)
 	moreDiags = resolveReferences(body, resourceProvider, resource)
 	diags = append(diags, moreDiags...)
 	return diags

pkg/steampipeconfig/parse/mod.go

@@ -124,9 +124,11 @@ func ParseMod(fileData map[string][]byte, pseudoResources []modconfig.MappableRe
 	}
 
 	// if variables were passed in parsecontext, add to the mod
-	for _, v := range parseCtx.Variables {
-		if diags = mod.AddResource(v); diags.HasErrors() {
-			return nil, modconfig.NewErrorsAndWarning(plugin.DiagsToError("Failed to add resource to mod", diags))
+	if parseCtx.Variables != nil {
+		for _, v := range parseCtx.Variables.RootVariables {
+			if diags = mod.AddResource(v); diags.HasErrors() {
+				return nil, modconfig.NewErrorsAndWarning(plugin.DiagsToError("Failed to add resource to mod", diags))
+			}
 		}
 	}
 

pkg/steampipeconfig/parse/mod_dependency_config.go

@@ -0,0 +1,26 @@
+package parse
+
+import (
+	"fmt"
+	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
+	"github.com/turbot/steampipe/pkg/steampipeconfig/versionmap"
+)
+
+type ModDependencyConfig struct {
+	ModDependency  *versionmap.ResolvedVersionConstraint
+	DependencyPath *string
+}
+
+func (c ModDependencyConfig) SetModProperties(mod *modconfig.Mod) {
+	mod.Version = c.ModDependency.Version
+	mod.DependencyPath = c.DependencyPath
+	mod.DependencyName = c.ModDependency.Name
+}
+
+func NewDependencyConfig(modDependency *versionmap.ResolvedVersionConstraint) *ModDependencyConfig {
+	d := fmt.Sprintf("%s@v%s", modDependency.Name, modDependency.Version.String())
+	return &ModDependencyConfig{
+		DependencyPath: &d,
+		ModDependency:  modDependency,
+	}
+}

pkg/steampipeconfig/parse/mod_parse_context.go

@@ -6,6 +6,7 @@ import (
 	"github.com/hashicorp/hcl/v2"
 	filehelpers "github.com/turbot/go-kit/files"
 	"github.com/turbot/go-kit/helpers"
+	"github.com/turbot/steampipe/pkg/steampipeconfig/inputvars"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/versionmap"
 	"github.com/zclconf/go-cty/cty"
@@ -41,22 +42,14 @@ type ModParseContext struct {
 
 	Flags       ParseModFlag
 	ListOptions *filehelpers.ListOptions
-	// map of loaded dependency mods, keyed by DependencyPath (including version)
-	// there may be multiple versions of same mod in this map
-	LoadedDependencyMods modconfig.ModMap
 
 	// Variables are populated in an initial parse pass top we store them on the run context
 	// so we can set them on the mod when we do the main parse
 
-	// Variables is a map of the variables in the current mod
-	// it is used to populate the variables property on the mod
-	Variables map[string]*modconfig.Variable
+	// Variables is a tree of maps of the variables in the current mod and child dependency mods
+	Variables *modconfig.ModVariableMap
 
-	// DependencyVariables is a map of the variables in the dependency mods of the current mod
-	// it is used to populate the variables values on child parseContexts when parsing dependencies
-	// (keyed by mod DependencyPath)
-	DependencyVariables map[string]map[string]*modconfig.Variable
-	ParentParseCtx      *ModParseContext
+	ParentParseCtx *ModParseContext
 
 	// stack of parent resources for the currently parsed block
 	// (unqualified name)
@@ -73,23 +66,24 @@ type ModParseContext struct {
 	// NOTE: all values from root mod are keyed with "local"
 	referenceValues map[string]ReferenceTypeValueMap
 
-	// a map of just the top level dependencies of the CurrentMod, keyed my full mod DepdencyName (with no version)
+	// a map of just the top level dependencies of the CurrentMod, keyed my full mod DependencyName (with no version)
 	topLevelDependencyMods modconfig.ModMap
+	// if we are loading dependency mod, this contains the details
+	DependencyConfig *ModDependencyConfig
 }
 
 func NewModParseContext(workspaceLock *versionmap.WorkspaceLock, rootEvalPath string, flags ParseModFlag, listOptions *filehelpers.ListOptions) *ModParseContext {
 	parseContext := NewParseContext(rootEvalPath)
 	c := &ModParseContext{
-		ParseContext:         parseContext,
-		Flags:                flags,
-		WorkspaceLock:        workspaceLock,
-		ListOptions:          listOptions,
-		LoadedDependencyMods: make(modconfig.ModMap),
+		ParseContext:  parseContext,
+		Flags:         flags,
+		WorkspaceLock: workspaceLock,
+		ListOptions:   listOptions,
 
-		blockChildMap: make(map[string][]string),
-		blockNameMap:  make(map[string]string),
+		topLevelDependencyMods: make(modconfig.ModMap),
+		blockChildMap:          make(map[string][]string),
+		blockNameMap:           make(map[string]string),
 		// initialise reference maps - even though we later overwrite them
-		Variables: make(map[string]*modconfig.Variable),
 		referenceValues: map[string]ReferenceTypeValueMap{
 			"local": make(ReferenceTypeValueMap),
 		},
@@ -101,7 +95,7 @@ func NewModParseContext(workspaceLock *versionmap.WorkspaceLock, rootEvalPath st
 	return c
 }
 
-func NewChildModParseContext(parent *ModParseContext, rootEvalPath string) *ModParseContext {
+func NewChildModParseContext(parent *ModParseContext, modVersion *versionmap.ResolvedVersionConstraint, rootEvalPath string) *ModParseContext {
 	// create a child run context
 	child := NewModParseContext(
 		parent.WorkspaceLock,
@@ -112,8 +106,17 @@ func NewChildModParseContext(parent *ModParseContext, rootEvalPath string) *ModP
 	child.BlockTypes = parent.BlockTypes
 	// set the child's parent
 	child.ParentParseCtx = parent
-	// copy DependencyVariables
-	child.DependencyVariables = parent.DependencyVariables
+	// set the dependency config
+	child.DependencyConfig = NewDependencyConfig(modVersion)
+	// set variables if parent has any
+	if parent.Variables != nil {
+		childVars, ok := parent.Variables.DependencyVariables[modVersion.Name]
+		if ok {
+			child.Variables = childVars
+			child.Variables.PopulatePublicVariables()
+			child.AddVariablesToEvalContext()
+		}
+	}
 
 	return child
 }
@@ -155,53 +158,71 @@ func VariableValueCtyMap(variables map[string]*modconfig.Variable) map[string]ct
 	return ret
 }
 
-// AddInputVariables adds variables to the run context.
+// AddInputVariableValues adds evaluated variables to the run context.
 // This function is called for the root run context after loading all input variables
-func (m *ModParseContext) AddInputVariables(inputVariables *modconfig.ModVariableMap) {
+func (m *ModParseContext) AddInputVariableValues(inputVariables *modconfig.ModVariableMap) {
 	// store the variables
-	m.Variables = inputVariables.RootVariables
-	// store the depdency variables sop we can pass them down to our children
-	m.DependencyVariables = inputVariables.DependencyVariables
+	m.Variables = inputVariables
+
+	// now add variables into eval context
+	m.AddVariablesToEvalContext()
 }
 
 func (m *ModParseContext) AddVariablesToEvalContext() {
-	m.addRootVariablesToReferenceMap(m.Variables)
+	m.addRootVariablesToReferenceMap()
 	m.addDependencyVariablesToReferenceMap()
 	m.buildEvalContext()
 }
 
 // addRootVariablesToReferenceMap sets the Variables property
 // and adds the variables to the referenceValues map (used to build the eval context)
-func (m *ModParseContext) addRootVariablesToReferenceMap(variables map[string]*modconfig.Variable) {
+func (m *ModParseContext) addRootVariablesToReferenceMap() {
 
+	variables := m.Variables.RootVariables
 	// write local variables directly into referenceValues map
 	// NOTE: we add with the name "var" not "variable" as that is how variables are referenced
 	m.referenceValues["local"]["var"] = VariableValueCtyMap(variables)
 }
 
-// addDependencyVariablesToReferenceMap sets the DependencyVariables property
-// and adds the dependency variables to the referenceValues map (used to build the eval context)
+// addDependencyVariablesToReferenceMap adds the dependency variables to the referenceValues map
+// (used to build the eval context)
 func (m *ModParseContext) addDependencyVariablesToReferenceMap() {
-	currentModKey := m.CurrentMod.GetInstallCacheKey()
-	topLevelDependencies := m.WorkspaceLock.InstallCache[currentModKey]
+	// retrieve the resolved dependency versions for the parent mod
+	resolvedVersions := m.WorkspaceLock.InstallCache[m.Variables.Mod.GetInstallCacheKey()]
 
-	// convert topLevelDependencies into as map keyed by depdency path
-	topLevelDependencyPathMap := topLevelDependencies.ToDependencyPathMap()
-	// NOTE: we add with the name "var" not "variable" as that is how variables are referenced
-	// add dependency mod variables to dependencyVariableValues, scoped by DependencyPath
-	for depModName, depVars := range m.DependencyVariables {
-		// only add variables from top level dependencies
-		if _, ok := topLevelDependencyPathMap[depModName]; ok {
-			// create map for this dependency if needed
-			alias := topLevelDependencyPathMap[depModName]
-			if m.referenceValues[alias] == nil {
-				m.referenceValues[alias] = make(ReferenceTypeValueMap)
-			}
-			m.referenceValues[alias]["var"] = VariableValueCtyMap(depVars)
+	for depModName, depVars := range m.Variables.DependencyVariables {
+		alias := resolvedVersions[depModName].Alias
+		if m.referenceValues[alias] == nil {
+			m.referenceValues[alias] = make(ReferenceTypeValueMap)
 		}
+		m.referenceValues[alias]["var"] = VariableValueCtyMap(depVars.RootVariables)
 	}
 }
 
+// when reloading a mod dependency tree to resolve require args values, this function is called after each mod is loaded
+// to load the require arg values and update the variable values
+func (m *ModParseContext) loadModRequireArgs() error {
+	//if we have not loaded variable definitions yet, do not load require args
+	if m.Variables == nil {
+		return nil
+	}
+
+	depModVarValues, err := inputvars.CollectVariableValuesFromModRequire(m.CurrentMod, m.WorkspaceLock)
+	if err != nil {
+		return err
+	}
+	if len(depModVarValues) == 0 {
+		return nil
+	}
+	// now update the variables map with the input values
+	depModVarValues.SetVariableValues(m.Variables)
+
+	// now add  overridden variables into eval context - in case the root mod references any dependency variable values
+	m.AddVariablesToEvalContext()
+
+	return nil
+}
+
 // AddModResources is used to add mod resources to the eval context
 func (m *ModParseContext) AddModResources(mod *modconfig.Mod) hcl.Diagnostics {
 	if len(m.UnresolvedBlocks) > 0 {
@@ -296,7 +317,7 @@ func (m *ModParseContext) GetMod(modShortName string) *modconfig.Mod {
 	key := m.CurrentMod.GetInstallCacheKey()
 	deps := m.WorkspaceLock.InstallCache[key]
 	for _, dep := range deps {
-		depMod, ok := m.LoadedDependencyMods[dep.DependencyPath()]
+		depMod, ok := m.topLevelDependencyMods[dep.Name]
 		if ok && depMod.ShortName == modShortName {
 			return depMod
 		}
@@ -509,60 +530,17 @@ func (m *ModParseContext) IsTopLevelBlock(block *hcl.Block) bool {
 	return isTopLevel
 }
 
-func (m *ModParseContext) GetLoadedDependencyMod(requiredModVersion *modconfig.ModVersionConstraint, mod *modconfig.Mod) (*modconfig.Mod, error) {
-	// if we have a locked version, update the required version to reflect this
-	lockedVersion, err := m.WorkspaceLock.GetLockedModVersionConstraint(requiredModVersion, mod)
-	if err != nil {
-		return nil, err
-	}
-	if lockedVersion == nil {
-		return nil, fmt.Errorf("not all dependencies are installed - run 'steampipe mod install'")
-	}
-	// use the full name of the locked version as key
-	d, _ := m.LoadedDependencyMods[lockedVersion.DependencyPath()]
-	return d, nil
-}
-
 func (m *ModParseContext) AddLoadedDependencyMod(mod *modconfig.Mod) {
-	// should never happen
-	if mod.DependencyPath == nil {
-		return
-	}
-	m.LoadedDependencyMods[*mod.DependencyPath] = mod
+	m.topLevelDependencyMods[mod.DependencyName] = mod
 }
 
 // GetTopLevelDependencyMods build a mod map of top level loaded dependencies, keyed by mod name
 func (m *ModParseContext) GetTopLevelDependencyMods() modconfig.ModMap {
-	// lazy load m.topLevelDependencyMods
-	if m.topLevelDependencyMods != nil {
-		return m.topLevelDependencyMods
-	}
-	// get install cache key fpor this mod (short name for top level mod or ModDependencyPath for dep mods)
-	installCacheKey := m.CurrentMod.GetInstallCacheKey()
-	deps := m.WorkspaceLock.InstallCache[installCacheKey]
-	m.topLevelDependencyMods = make(modconfig.ModMap, len(deps))
-
-	// merge in the dependency mods
-	for _, dep := range deps {
-		key := dep.DependencyPath()
-		loadedDepMod := m.LoadedDependencyMods[key]
-		if loadedDepMod != nil {
-			// as key use the ModDependencyPath _without_ the version
-			m.topLevelDependencyMods[loadedDepMod.DependencyName] = loadedDepMod
-		}
-	}
 	return m.topLevelDependencyMods
 }
 
 func (m *ModParseContext) SetCurrentMod(mod *modconfig.Mod) {
 	m.CurrentMod = mod
-
-	// if the current mod is a dependency mod (i.e. has a DependencyPath property set), update the Variables property
-	if dependencyVariables, ok := m.DependencyVariables[mod.GetInstallCacheKey()]; ok {
-		m.Variables = dependencyVariables
-	}
-	// set the root variables from the parent
-	// now the mod is set we can add variables to the eval context
-	// ( we cannot do this until mod as set as we need to identify which variables to use if we are a dependency
-	m.AddVariablesToEvalContext()
+	// now we have the mod, load any arg values from the mod require - these will be passed to dependency mods
+	m.loadModRequireArgs()
 }

pkg/steampipeconfig/versionmap/resolved_version_map.go

@@ -19,12 +19,3 @@ func (m ResolvedVersionMap) ToVersionListMap() ResolvedVersionListMap {
 	}
 	return res
 }
-
-// ToDependencyPathMap converts to a map of mod aliases, keyed by mod dependency path
-func (m ResolvedVersionMap) ToDependencyPathMap() map[string]string {
-	res := make(map[string]string, len(m))
-	for _, c := range m {
-		res[c.DependencyPath()] = c.Alias
-	}
-	return res
-}

pkg/steampipeconfig/versionmap/workspace_lock.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -327,3 +328,13 @@ func (l *WorkspaceLock) StructVersion() int {
 	return WorkspaceLockStructVersion
 
 }
+
+func (l *WorkspaceLock) FindInstalledDependency(modDependency *ResolvedVersionConstraint) (string, error) {
+	dependencyFilepath := path.Join(l.ModInstallationPath, modDependency.DependencyPath())
+
+	if filehelpers.DirectoryExists(dependencyFilepath) {
+		return dependencyFilepath, nil
+	}
+
+	return "", fmt.Errorf("dependency mod '%s' is not installed - run 'steampipe mod install'", modDependency.DependencyPath())
+}

pkg/workspace/load_workspace.go

@@ -10,6 +10,7 @@ import (
 	"github.com/spf13/viper"
 	"github.com/turbot/steampipe/pkg/constants"
 	"github.com/turbot/steampipe/pkg/statushooks"
+	"github.com/turbot/steampipe/pkg/steampipeconfig"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/inputvars"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
 )
@@ -24,11 +25,15 @@ func LoadWorkspacePromptingForVariables(ctx context.Context) (*Workspace, *modco
 	if errAndWarnings.GetError() == nil {
 		return w, errAndWarnings
 	}
-	missingVariablesError, ok := errAndWarnings.GetError().(modconfig.MissingVariableError)
+	missingVariablesError, ok := errAndWarnings.GetError().(*steampipeconfig.MissingVariableError)
 	// if there was an error which is NOT a MissingVariableError, return it
 	if !ok {
 		return nil, errAndWarnings
 	}
+	// if there are missing transitive dependency variables, fail as we do not prompt for these
+	if len(missingVariablesError.MissingTransitiveVariables) > 0 {
+		return nil, errAndWarnings
+	}
 	// if interactive input is disabled, return the missing variables error
 	if !viper.GetBool(constants.ArgInput) {
 		return nil, modconfig.NewErrorsAndWarning(missingVariablesError)

pkg/workspace/workspace.go

@@ -252,15 +252,19 @@ func (w *Workspace) loadWorkspaceMod(ctx context.Context) *modconfig.ErrorAndWar
 		return modconfig.NewErrorsAndWarning(err)
 	}
 	// populate the parsed variable values
-	w.VariableValues = inputVariables.VariableValues
+	w.VariableValues, err = inputVariables.GetPublicVariableValues()
+	if err != nil {
+		return modconfig.NewErrorsAndWarning(err)
+	}
 
 	// build run context which we use to load the workspace
 	parseCtx, err := w.getParseContext(ctx)
 	if err != nil {
 		return modconfig.NewErrorsAndWarning(err)
 	}
-	// add variables
-	parseCtx.AddInputVariables(inputVariables)
+
+	// add evaluated variables to the context
+	parseCtx.AddInputVariableValues(inputVariables)
 	// do not reload variables as we already have them
 	parseCtx.BlockTypeExclusions = []string{modconfig.BlockTypeVariable}
 
@@ -293,28 +297,12 @@ func (w *Workspace) getInputVariables(ctx context.Context, validateMissing bool)
 		return nil, err
 	}
 
-	inputVariables, err := w.getVariableValues(ctx, variablesParseCtx, validateMissing)
+	inputVariableValues, err := w.getVariableValues(ctx, variablesParseCtx, validateMissing)
 	if err != nil {
 		return nil, err
 	}
 
-	// if needed, reload
-	// if a mod require has args which use a variable, this will not have been resolved in the first pass
-	// - we need to parse again
-	if variablesParseCtx.CurrentMod.RequireHasUnresolvedArgs() {
-		// add the variables into the parse context and rebuild the eval context
-		variablesParseCtx.AddInputVariables(inputVariables)
-		variablesParseCtx.AddVariablesToEvalContext()
-
-		// now try to parse the mod again
-		inputVariables, err = w.getVariableValues(ctx, variablesParseCtx, validateMissing)
-		if err != nil {
-			return nil, err
-		}
-
-	}
-	return inputVariables, nil
-
+	return inputVariableValues, nil
 }
 
 func (w *Workspace) getVariableValues(ctx context.Context, variablesParseCtx *parse.ModParseContext, validateMissing bool) (*modconfig.ModVariableMap, error) {