For query command, load mod variables syncronously but the rest of mod async. Closes #4109

cmd/check.go

@@ -317,6 +317,7 @@ func validateCheckArgs(ctx context.Context, cmd *cobra.Command, args []string) b
 		fmt.Println()
 		error_helpers.ShowError(ctx, fmt.Errorf("you must provide at least one argument"))
 		fmt.Println()
+		//nolint:errcheck // cmd.Help always returns a nil error
 		cmd.Help()
 		fmt.Println()
 		return false

cmd/list.go

@@ -6,9 +6,7 @@ import (
 	"sort"
 
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 	"github.com/turbot/steampipe/pkg/cmdconfig"
-	"github.com/turbot/steampipe/pkg/constants"
 	"github.com/turbot/steampipe/pkg/display"
 	"github.com/turbot/steampipe/pkg/error_helpers"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
@@ -44,9 +42,10 @@ func getRunListSubCmd(opts listSubCmdOptions) func(cmd *cobra.Command, args []st
 
 	return func(cmd *cobra.Command, _ []string) {
 		ctx := cmd.Context()
-		workspacePath := viper.GetString(constants.ArgModLocation)
 
-		w, errAndWarnings := workspace.Load(ctx, workspacePath)
+		w, errAndWarnings := workspace.LoadWorkspaceVars(ctx)
+		error_helpers.FailOnError(errAndWarnings.GetError())
+		errAndWarnings = w.LoadWorkspaceMod(ctx)
 		error_helpers.FailOnError(errAndWarnings.GetError())
 
 		modResources, depResources, err := listResourcesInMod(ctx, w.Mod, cmd)

go.mod

@@ -146,7 +146,6 @@ require (
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
-	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect

pkg/dashboard/dashboardexecute/check_run.go

@@ -2,6 +2,7 @@ package dashboardexecute
 
 import (
 	"context"
+
 	"github.com/turbot/steampipe/pkg/control/controlexecute"
 	"github.com/turbot/steampipe/pkg/control/controlstatus"
 	"github.com/turbot/steampipe/pkg/dashboard/dashboardtypes"
@@ -65,7 +66,10 @@ func (r *CheckRun) Execute(ctx context.Context) {
 
 	// create a context with a DashboardEventControlHooks to report control execution progress
 	ctx = controlstatus.AddControlHooksToContext(ctx, NewDashboardEventControlHooks(r))
-	r.controlExecutionTree.Execute(ctx)
+	if err := r.controlExecutionTree.Execute(ctx); err != nil {
+		r.SetError(ctx, err)
+		return
+	}
 
 	// set the summary on the CheckRun
 	r.Summary = r.controlExecutionTree.Root.Summary

pkg/dashboard/dashboardexecute/container_run.go

@@ -3,9 +3,10 @@ package dashboardexecute
 import (
 	"context"
 	"fmt"
+	"log"
+
 	"github.com/turbot/steampipe/pkg/dashboard/dashboardtypes"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
-	"log"
 )
 
 // DashboardContainerRun is a struct representing a container run
@@ -45,6 +46,7 @@ func NewDashboardContainerRun(container *modconfig.DashboardContainer, parent da
 	r.childCompleteChan = make(chan dashboardtypes.DashboardTreeRun, len(children))
 	for _, child := range children {
 		var childRun dashboardtypes.DashboardTreeRun
+		//nolint:golint // using a global var here to maintain parity with definition of childRun
 		var err error
 		switch i := child.(type) {
 		case *modconfig.DashboardContainer:

pkg/dashboard/dashboardexecute/referenced_variables.go

@@ -2,10 +2,11 @@ package dashboardexecute
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/turbot/steampipe/pkg/dashboard/dashboardtypes"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
 	"github.com/turbot/steampipe/pkg/workspace"
-	"strings"
 )
 
 // GetReferencedVariables builds map of variables values containing only those mod variables which are referenced
@@ -35,6 +36,7 @@ func GetReferencedVariables(root dashboardtypes.DashboardTreeRun, w *workspace.W
 
 	switch r := root.(type) {
 	case *DashboardRun:
+		//nolint:errcheck // we don't care about errors here, since the callback does not return an error
 		r.dashboard.WalkResources(
 			func(resource modconfig.HclResource) (bool, error) {
 				if resourceWithMetadata, ok := resource.(modconfig.ResourceWithMetadata); ok {
@@ -46,6 +48,7 @@ func GetReferencedVariables(root dashboardtypes.DashboardTreeRun, w *workspace.W
 	case *CheckRun:
 		switch n := r.resource.(type) {
 		case *modconfig.Benchmark:
+			//nolint:errcheck // we don't care about errors here, since the callback does not return an error
 			n.WalkResources(
 				func(resource modconfig.ModTreeItem) (bool, error) {
 					if resourceWithMetadata, ok := resource.(modconfig.ResourceWithMetadata); ok {

pkg/dashboard/dashboardexecute/runtime_dependency_publisher_impl.go

@@ -4,12 +4,13 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/turbot/steampipe/pkg/dashboard/dashboardtypes"
-	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
-	"github.com/turbot/steampipe/pkg/utils"
 	"log"
 	"strconv"
 	"sync"
+
+	"github.com/turbot/steampipe/pkg/dashboard/dashboardtypes"
+	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
+	"github.com/turbot/steampipe/pkg/utils"
 )
 
 type runtimeDependencyPublisherImpl struct {
@@ -234,7 +235,6 @@ func (p *runtimeDependencyPublisherImpl) setWithValue(w *LeafRun) {
 		populateData(w.Data, result)
 	}
 	p.PublishRuntimeDependencyValue(name, result)
-	return
 }
 
 func populateData(withData *dashboardtypes.LeafData, result *dashboardtypes.ResolvedRuntimeDependencyValue) {

pkg/dashboard/dashboardexecute/runtime_dependency_subscriber_impl.go

@@ -361,6 +361,7 @@ func (s *RuntimeDependencySubscriberImpl) populateParamDefaults(provider modconf
 		if dep := s.findRuntimeDependencyForParentProperty(paramDef.UnqualifiedName); dep != nil {
 			// assuming the default property is the target, set the default
 			if typehelpers.SafeString(dep.Dependency.TargetPropertyName) == "default" {
+				//nolint:errcheck // the only reason where SetDefault could fail is if `dep.Value` cannot be marshalled as a JSON string
 				paramDef.SetDefault(dep.Value)
 			}
 		}

pkg/dashboard/dashboardexecute/snapshot.go

@@ -31,7 +31,9 @@ func GenerateSnapshot(ctx context.Context, target string, initData *initialisati
 
 	// all runtime dependencies must be resolved before execution (i.e. inputs must be passed in)
 	Executor.interactive = false
-	Executor.ExecuteDashboard(ctx, sessionId, target, inputs, w, initData.Client)
+	if err := Executor.ExecuteDashboard(ctx, sessionId, target, inputs, w, initData.Client); err != nil {
+		return nil, err
+	}
 
 	select {
 	case err = <-errorChannel:

pkg/query/init_data.go

@@ -51,8 +51,8 @@ func NewInitData(ctx context.Context, args []string) *InitData {
 
 	statushooks.SetStatus(ctx, "Loading workspace")
 
-	// load workspace syncronously
-	w, errAndWarnings := workspace.LoadWorkspacePromptingForVariables(ctx)
+	// load workspace variables syncronously
+	w, errAndWarnings := workspace.LoadWorkspaceVars(ctx)
 	if errAndWarnings.GetError() != nil {
 		i.Result.Error = fmt.Errorf("failed to load workspace: %s", error_helpers.HandleCancelError(errAndWarnings.GetError()).Error())
 		return i
@@ -115,6 +115,14 @@ func (i *InitData) init(ctx context.Context, args []string) {
 		}
 	}
 
+	// load the workspace mod (this load is asynchronous as it is within the async init function)
+	errAndWarnings := i.Workspace.LoadWorkspaceMod(ctx)
+	i.Result.AddWarnings(errAndWarnings.Warnings...)
+	if errAndWarnings.GetError() != nil {
+		i.Result.Error = fmt.Errorf("failed to load workspace mod: %s", error_helpers.HandleCancelError(errAndWarnings.GetError()).Error())
+		return
+	}
+
 	// set max DB connections to 1
 	viper.Set(constants.ArgMaxParallel, 1)
 

pkg/steampipeconfig/modconfig/mod.go

@@ -410,3 +410,12 @@ func (m *Mod) RequireHasUnresolvedArgs() bool {
 	}
 	return false
 }
+
+func (m *Mod) hasChild(item ModTreeItem) bool {
+	for _, c := range m.children {
+		if c == item {
+			return true
+		}
+	}
+	return false
+}

pkg/steampipeconfig/modconfig/mod_resource_tree.go

@@ -47,6 +47,13 @@ func (m *Mod) addResourcesIntoTree(sourceMod *Mod) error {
 
 	resourceFunc := func(item HclResource) (bool, error) {
 		if treeItem, ok := item.(ModTreeItem); ok {
+			// TACTICAL: addResourcesIntoTree getc called severasl times, as we parse the mod in stages
+			// - first the variables then the rest of the resources
+			// if we have already added this into the tree, skip
+			if m.hasChild(treeItem) {
+				return true, nil
+			}
+
 			// NOTE: add resource into _our_ resource tree, i.e. mod 'm'
 			if err = m.addItemIntoResourceTree(treeItem); err != nil {
 				// stop walking

pkg/workspace/load_workspace.go

@@ -2,7 +2,6 @@ package workspace
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"log"
 	"time"
@@ -10,47 +9,24 @@ import (
 	"github.com/spf13/viper"
 	"github.com/turbot/steampipe/pkg/constants"
 	"github.com/turbot/steampipe/pkg/error_helpers"
-	"github.com/turbot/steampipe/pkg/statushooks"
-	"github.com/turbot/steampipe/pkg/steampipeconfig"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/inputvars"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
 	"github.com/turbot/terraform-components/terraform"
 )
 
 func LoadWorkspacePromptingForVariables(ctx context.Context) (*Workspace, *error_helpers.ErrorAndWarnings) {
-	workspacePath := viper.GetString(constants.ArgModLocation)
 	t := time.Now()
 	defer func() {
 		log.Printf("[TRACE] Workspace load took %dms\n", time.Since(t).Milliseconds())
 	}()
-	w, errAndWarnings := Load(ctx, workspacePath)
-	if errAndWarnings.GetError() == nil {
+	w, errAndWarnings := LoadWorkspaceVars(ctx)
+	if errAndWarnings.GetError() != nil {
 		return w, errAndWarnings
 	}
 
-	var missingVariablesError steampipeconfig.MissingVariableError
-	ok := errors.As(errAndWarnings.GetError(), &missingVariablesError)
-	// if there was an error which is NOT a MissingVariableError, return it
-	if !ok {
-		return nil, errAndWarnings
-	}
-	// if there are missing transitive dependency variables, fail as we do not prompt for these
-	if len(missingVariablesError.MissingTransitiveVariables) > 0 {
-		return nil, errAndWarnings
-	}
-	// if interactive input is disabled, return the missing variables error
-	if !viper.GetBool(constants.ArgInput) {
-		return nil, error_helpers.NewErrorsAndWarning(missingVariablesError)
-	}
-	// so we have missing variables - prompt for them
-	// first hide spinner if it is there
-	statushooks.Done(ctx)
-	if err := promptForMissingVariables(ctx, missingVariablesError.MissingVariables, workspacePath); err != nil {
-		log.Printf("[TRACE] Interactive variables prompting returned error %v", err)
-		return nil, error_helpers.NewErrorsAndWarning(err)
-	}
-	// ok we should have all variables now - reload workspace
-	return Load(ctx, workspacePath)
+	// load the workspace mod
+	errAndWarnings = w.LoadWorkspaceMod(ctx)
+	return w, errAndWarnings
 }
 
 func promptForMissingVariables(ctx context.Context, missingVariables []*modconfig.Variable, workspacePath string) error {

pkg/workspace/workspace.go

@@ -3,6 +3,7 @@ package workspace
 import (
 	"bufio"
 	"context"
+	"errors"
 	"fmt"
 	"log"
 	"os"
@@ -24,6 +25,7 @@ import (
 	"github.com/turbot/steampipe/pkg/error_helpers"
 	"github.com/turbot/steampipe/pkg/filepaths"
 	"github.com/turbot/steampipe/pkg/modinstaller"
+	"github.com/turbot/steampipe/pkg/statushooks"
 	"github.com/turbot/steampipe/pkg/steampipeconfig"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/parse"
@@ -62,10 +64,13 @@ type Workspace struct {
 	loadPseudoResources        bool
 	// channel used to send dashboard events to the handleDashboardEvent goroutine
 	dashboardEventChan chan dashboardevents.DashboardEvent
+	allVariables       *modconfig.ModVariableMap
 }
 
-// Load creates a Workspace and loads the workspace mod
-func Load(ctx context.Context, workspacePath string) (*Workspace, *error_helpers.ErrorAndWarnings) {
+// LoadWorkspaceVars creates a Workspace and loads the variables
+func LoadWorkspaceVars(ctx context.Context) (*Workspace, *error_helpers.ErrorAndWarnings) {
+	workspacePath := viper.GetString(constants.ArgModLocation)
+
 	utils.LogTime("workspace.Load start")
 	defer utils.LogTime("workspace.Load end")
 
@@ -74,9 +79,15 @@ func Load(ctx context.Context, workspacePath string) (*Workspace, *error_helpers
 		return nil, error_helpers.NewErrorsAndWarning(err)
 	}
 
-	// load the workspace mod
-	errAndWarnings := workspace.loadWorkspaceMod(ctx)
-	return workspace, errAndWarnings
+	// check if your workspace path is home dir and if modfile exists - if yes then warn and ask user to continue or not
+	if err := HomeDirectoryModfileCheck(ctx, workspacePath); err != nil {
+		return nil, error_helpers.NewErrorsAndWarning(err)
+	}
+	errorsAndWarnings := workspace.populateVariables(ctx)
+	if errorsAndWarnings.Error != nil {
+		return nil, errorsAndWarnings
+	}
+	return workspace, errorsAndWarnings
 }
 
 // LoadVariables creates a Workspace and uses it to load all variables, ignoring any value resolution errors
@@ -289,24 +300,8 @@ func HomeDirectoryModfileCheck(ctx context.Context, workspacePath string) error
 	return nil
 }
 
-func (w *Workspace) loadWorkspaceMod(ctx context.Context) *error_helpers.ErrorAndWarnings {
-	// check if your workspace path is home dir and if modfile exists - if yes then warn and ask user to continue or not
-	if err := HomeDirectoryModfileCheck(ctx, w.Path); err != nil {
-		return error_helpers.NewErrorsAndWarning(err)
-	}
-
-	// resolve values of all input variables
-	// we WILL validate missing variables when loading
-	validateMissing := true
-	inputVariables, errorsAndWarnings := w.getInputVariables(ctx, validateMissing)
-	if errorsAndWarnings.Error != nil {
-		return errorsAndWarnings
-	}
-	// populate the parsed variable values
-	w.VariableValues, errorsAndWarnings.Error = inputVariables.GetPublicVariableValues()
-	if errorsAndWarnings.Error != nil {
-		return errorsAndWarnings
-	}
+func (w *Workspace) LoadWorkspaceMod(ctx context.Context) *error_helpers.ErrorAndWarnings {
+	var errorsAndWarnings = &error_helpers.ErrorAndWarnings{}
 
 	// build run context which we use to load the workspace
 	parseCtx, err := w.getParseContext(ctx)
@@ -315,8 +310,6 @@ func (w *Workspace) loadWorkspaceMod(ctx context.Context) *error_helpers.ErrorAn
 		return errorsAndWarnings
 	}
 
-	// add evaluated variables to the context
-	parseCtx.AddInputVariableValues(inputVariables)
 	// do not reload variables as we already have them
 	parseCtx.BlockTypeExclusions = []string{modconfig.BlockTypeVariable}
 
@@ -343,6 +336,54 @@ func (w *Workspace) loadWorkspaceMod(ctx context.Context) *error_helpers.ErrorAn
 	return errorsAndWarnings
 }
 
+func (w *Workspace) populateVariables(ctx context.Context) *error_helpers.ErrorAndWarnings {
+	// resolve values of all input variables
+	// we WILL validate missing variables when loading
+	validateMissing := true
+	inputVariables, errorsAndWarnings := w.getInputVariables(ctx, validateMissing)
+	if errorsAndWarnings.Error != nil {
+
+		// so there was an error - was it missing variables error
+		var missingVariablesError steampipeconfig.MissingVariableError
+		ok := errors.As(errorsAndWarnings.GetError(), &missingVariablesError)
+		// if there was an error which is NOT a MissingVariableError, return it
+		if !ok {
+			return errorsAndWarnings
+		}
+		// if there are missing transitive dependency variables, fail as we do not prompt for these
+		if len(missingVariablesError.MissingTransitiveVariables) > 0 {
+			return errorsAndWarnings
+		}
+		// if interactive input is disabled, return the missing variables error
+		if !viper.GetBool(constants.ArgInput) {
+			return error_helpers.NewErrorsAndWarning(missingVariablesError)
+		}
+		// so we have missing variables - prompt for them
+		// first hide spinner if it is there
+		statushooks.Done(ctx)
+		if err := promptForMissingVariables(ctx, missingVariablesError.MissingVariables, w.Path); err != nil {
+			log.Printf("[TRACE] Interactive variables prompting returned error %v", err)
+			return error_helpers.NewErrorsAndWarning(err)
+		}
+
+		// now try to load vars again
+		inputVariables, errorsAndWarnings = w.getInputVariables(ctx, validateMissing)
+		if errorsAndWarnings.Error != nil {
+			return errorsAndWarnings
+		}
+
+	}
+	// store the full variable map
+	w.allVariables = inputVariables
+	// populate the parsed variable values
+	w.VariableValues, errorsAndWarnings.Error = inputVariables.GetPublicVariableValues()
+	if errorsAndWarnings.Error == nil {
+		return errorsAndWarnings
+	}
+
+	return errorsAndWarnings
+}
+
 func (w *Workspace) getInputVariables(ctx context.Context, validateMissing bool) (*modconfig.ModVariableMap, *error_helpers.ErrorAndWarnings) {
 	// build a run context just to use to load variable definitions
 	variablesParseCtx, err := w.getParseContext(ctx)
@@ -386,6 +427,11 @@ func (w *Workspace) getParseContext(ctx context.Context) (*parse.ModParseContext
 			Include: filehelpers.InclusionsFromExtensions(constants.ModDataExtensions),
 		})
 
+	// add any evaluated variables to the context
+	if w.allVariables != nil {
+		parseCtx.AddInputVariableValues(w.allVariables)
+	}
+
 	return parseCtx, nil
 }
 

pkg/workspace/workspace_events.go

@@ -128,7 +128,7 @@ func (w *Workspace) reloadResourceMaps(ctx context.Context) (*modconfig.Resource
 	}
 
 	// now reload the workspace
-	errAndWarnings := w.loadWorkspaceMod(ctx)
+	errAndWarnings := w.LoadWorkspaceMod(ctx)
 	if errAndWarnings.GetError() != nil {
 		// check the existing watcher error - if we are already in an error state, do not show error
 		if w.watcherError == nil {

tests/acceptance/test_data/mods/dependent_mod_with_variables/.steampipe/mods/github.com/pskrbasu/steampipe-mod-m1@v4.0/.gitignore

@@ -1,33 +0,0 @@
-# Binaries for programs and plugins
-*.exe
-*.exe~
-*.dll
-*.so
-*.dylib
-
-# Test binary, built with `go test -c`
-*.test
-
-# Output of the go coverage tool, specifically wh.en used with LiteIDE
-*.out
-
-# Dependency directories (remove the comment below to include it)
-# vendor/
-
-# Ignore JetBrains files
-.idea
-
-# Ignore swap files
-*.swo
-*.swp
-
-# Parliament IAM permissions outputs
-scripts/generate_parliament_iam_permissions/__pycache__
-scripts/generate_parliament_iam_permissions/docs
-scripts/generate_parliament_iam_permissions/generate_go_file.
-
-# Ignore node dependencies
-node_modules
-
-# Ignore file types
-.vscode/launch.json

tests/acceptance/test_data/mods/dependent_mod_with_variables/.steampipe/mods/github.com/pskrbasu/steampipe-mod-m1@v4.0/mod.sp

@@ -1,3 +0,0 @@
-mod "m1" {
-  title = "m1"
-}

tests/acceptance/test_data/mods/dependent_mod_with_variables/.steampipe/mods/github.com/pskrbasu/steampipe-mod-m1@v4.0/query.sp

@@ -1,17 +0,0 @@
-variable dep_mod_var1 {
-    type = string
-}
-
-
-variable dep_mod_var2 {
-    type = string
-}
-
-variable dupe_name_var {
-    type = string
-}
-
-
-query "m1_q1"{
-    sql = "select 1 as query"
-}

tests/acceptance/test_data/mods/dependent_mod_with_variables/steampipe.spvars

@@ -6,3 +6,5 @@ m1.dep_mod_var1 = "select 'm1.dep_mod_var_set_in_file' as a"
 m1.dupe_name_var = "select 'm1.dupe_name_var_set_in_file' as a"
 
 dupe_name_var = "select 'dupe_name_var_set_in_file' as a"
+
+m1.dep_mod_var2 = "bar"