merge v0.21.x

Fix modfile being left invalid after mod uninstall. Fix variables not being reloaded after file watch event. Closes #4123. Closes #4124

CHANGELOG.md

@@ -1,3 +1,8 @@
+## v0.21.7 [2024-02-09]
+_Bug fixes_
+* Fix variables not being reloaded after file watch event. ([#4123](https://github.com/turbot/steampipe/issues/4123))
+* Fix modfile being left invalid after mod uninstall. Fix variables not being reloaded after file watch event. ([#4124](https://github.com/turbot/steampipe/issues/4124))
+
 ## v0.21.6 [2024-02-06]
 _Bug fixes_
 * Fix `HomeDirectoryModfileCheck` returning false positive, causing errors when executing steampipe out of the home directory. ([#4118](https://github.com/turbot/steampipe/issues/4118))

pkg/cmdconfig/cmd_hooks.go

@@ -366,12 +366,11 @@ func createLogger(logBuffer *bytes.Buffer, cmd *cobra.Command) {
 		// this will also write out the Execution ID - enabling easy filtering of logs for a single execution
 		// we need to do this since all instances will log to a single file and logs will be interleaved
 		log.Printf("[INFO] ********************************************************\n")
-		log.Printf("[INFO] **%16s%20s%16s**\n", " ", fmt.Sprintf("Steampipe [%s]", runtime.ExecutionID), " ")
-		log.Printf("[INFO] ********************************************************\n")
+		log.Printf("[INFO] steampipe %s [%s]", cmd.Name(), runtime.ExecutionID)
 		log.Printf("[INFO] Version:   v%s\n", version.VersionString)
 		log.Printf("[INFO] Log level: %s\n", sdklogging.LogLevel())
-		log.Printf("[INFO] Log date: %s\n", time.Now().Format("2006-01-02"))
-		//
+		log.Printf("[INFO] Log date:  %s\n", time.Now().Format("2006-01-02"))
+		log.Printf("[INFO] ********************************************************\n")
 	}
 }
 

pkg/constants/db.go

@@ -28,7 +28,7 @@ const (
 // constants for installing db and fdw images
 const (
 	DatabaseVersion = "14.2.0"
-	FdwVersion      = "1.9.1"
+	FdwVersion      = "1.9.3"
 
 	// PostgresImageRef is the OCI Image ref for the database binaries
 	PostgresImageRef    = "us-docker.pkg.dev/steampipe/steampipe/db:14.2.0"

pkg/modinstaller/mod_installer_require.go

@@ -75,8 +75,8 @@ func (i *ModInstaller) shouldCreateRequireBlock(oldRequire *modconfig.Require, n
 func (i *ModInstaller) buildChangeSetForRequireDelete(oldRequire *modconfig.Require, newRequire *modconfig.Require) ChangeSet {
 	return NewChangeSet(&Change{
 		Operation:   Delete,
-		OffsetStart: oldRequire.DeclRange.Start.Byte,
-		OffsetEnd:   oldRequire.BodyRange.End.Byte,
+		OffsetStart: oldRequire.TypeRange.Start.Byte,
+		OffsetEnd:   oldRequire.DeclRange.End.Byte,
 	})
 }
 
@@ -89,11 +89,11 @@ func (i *ModInstaller) buildChangeSetForRequireCreate(oldRequire *modconfig.Requ
 	var body *hclwrite.Body
 	var insertOffset int
 
-	if oldRequire.BodyRange.Start.Byte != 0 {
+	if oldRequire.TypeRange.Start.Byte != 0 {
 		// this means that there is a require block
 		// but is probably empty
 		body = f.Body()
-		insertOffset = oldRequire.BodyRange.End.Byte - 1
+		insertOffset = oldRequire.TypeRange.End.Byte - 1
 	} else {
 		// we don't have a require block at all
 		// let's create one to append to
@@ -186,7 +186,7 @@ func (i *ModInstaller) calcChangesForInstall(oldRequire *modconfig.Require, newR
 	return ChangeSet{
 		&Change{
 			Operation:   Insert,
-			OffsetStart: oldRequire.BodyRange.End.Byte - 1,
+			OffsetStart: oldRequire.DeclRange.End.Byte - 1,
 			Content:     f.Bytes(),
 		},
 	}

pkg/steampipeconfig/inputvars/collect_variables.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/turbot/steampipe/pkg/error_helpers"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
+	"log"
 	"os"
 	"regexp"
 	"strings"
@@ -58,6 +59,7 @@ func CollectVariableValues(workspacePath string, variableFileArgs []string, vari
 				name:       name,
 				sourceType: ValueFromEnvVar,
 			}
+			log.Printf("[INFO] adding value for variable '%s' from environment", name)
 		}
 	}
 
@@ -67,6 +69,7 @@ func CollectVariableValues(workspacePath string, variableFileArgs []string, vari
 	// ending in .auto.spvars.
 	defaultVarsPath := filepaths.DefaultVarsFilePath(workspacePath)
 	if _, err := os.Stat(defaultVarsPath); err == nil {
+		log.Printf("[INFO] adding values from %s", defaultVarsPath)
 		diags := addVarsFromFile(defaultVarsPath, ValueFromAutoFile, ret)
 		if diags.HasErrors() {
 			return nil, error_helpers.DiagsToError(fmt.Sprintf("failed to load variables from '%s'", defaultVarsPath), diags)
@@ -81,6 +84,7 @@ func CollectVariableValues(workspacePath string, variableFileArgs []string, vari
 			if !isAutoVarFile(name) {
 				continue
 			}
+			log.Printf("[INFO] adding values from %s", name)
 			diags := addVarsFromFile(name, ValueFromAutoFile, ret)
 			if diags.HasErrors() {
 				return nil, error_helpers.DiagsToError(fmt.Sprintf("failed to load variables from '%s'", name), diags)
@@ -92,6 +96,7 @@ func CollectVariableValues(workspacePath string, variableFileArgs []string, vari
 	// Finally we process values given explicitly on the command line, either
 	// as individual literal settings or as additional files to read.
 	for _, fileArg := range variableFileArgs {
+		log.Printf("[INFO] adding values from %s", fileArg)
 		diags := addVarsFromFile(fileArg, ValueFromNamedFile, ret)
 		if diags.HasErrors() {
 			return nil, error_helpers.DiagsToError(fmt.Sprintf("failed to load variables from '%s'", fileArg), diags)
@@ -121,10 +126,11 @@ func CollectVariableValues(workspacePath string, variableFileArgs []string, vari
 			name:       name,
 			sourceType: ValueFromCLIArg,
 		}
+		log.Printf("[INFO] adding value for variable '%s' from command line arg", name)
 	}
 
 	if diags.HasErrors() {
-		return nil, error_helpers.DiagsToError(fmt.Sprintf("failed to evaluate var args:"), diags)
+		return nil, error_helpers.DiagsToError("failed to evaluate var args:", diags)
 	}
 
 	// check viper for any interactively added variables
@@ -137,6 +143,7 @@ func CollectVariableValues(workspacePath string, variableFileArgs []string, vari
 				Name:     name,
 				RawValue: rawVal.(string),
 			}
+			log.Printf("[INFO] adding value for variable '%s' specified on interactive prompt", name)
 		}
 	}
 
@@ -248,6 +255,7 @@ func addVarsFromFile(filename string, sourceType ValueSourceType, to map[string]
 			expr:       attr.Expr,
 			sourceType: sourceType,
 		}
+		log.Printf("[INFO] adding value for variable '%s' from var file", name)
 	}
 	return diags
 }

pkg/steampipeconfig/inputvars/unparsed_value.go

@@ -3,6 +3,7 @@ package inputvars
 import (
 	"fmt"
 	"github.com/zclconf/go-cty/cty"
+	"log"
 	"strings"
 
 	"github.com/hashicorp/hcl/v2"
@@ -159,6 +160,7 @@ func ParseVariableValues(inputValuesUnparsed map[string]UnparsedVariableValue, v
 				SourceType:  ValueFromConfig,
 				SourceRange: tfdiags.SourceRangeFromHCL(vc.DeclRange),
 			}
+			log.Printf("[INFO] adding value for variable '%s' from var default", name)
 		}
 	}
 

pkg/steampipeconfig/load_mod_variables.go

@@ -3,7 +3,9 @@ package steampipeconfig
 import (
 	"context"
 	"golang.org/x/exp/maps"
+	"log"
 	"sort"
+	"strings"
 
 	"github.com/spf13/viper"
 	"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
@@ -31,6 +33,7 @@ func LoadVariableDefinitions(ctx context.Context, variablePath string, parseCtx
 }
 
 func GetVariableValues(parseCtx *parse.ModParseContext, variableMap *modconfig.ModVariableMap, validate bool) (*modconfig.ModVariableMap, *error_helpers.ErrorAndWarnings) {
+	log.Printf("[INFO] GetVariableValues")
 	// now resolve all input variables
 	inputValues, errorsAndWarnings := getInputVariables(parseCtx, variableMap, validate)
 	if errorsAndWarnings.Error == nil {
@@ -49,18 +52,32 @@ func getInputVariables(parseCtx *parse.ModParseContext, variableMap *modconfig.M
 	mod := parseCtx.CurrentMod
 	path := mod.ModPath
 
+	log.Printf("[INFO] getInputVariables, variableFileArgs: %s, variableArgs: %s", variableFileArgs, variableArgs)
+
 	var inputValuesUnparsed, err = inputvars.CollectVariableValues(path, variableFileArgs, variableArgs, parseCtx.CurrentMod)
 	if err != nil {
+		log.Printf("[WARN] CollectVariableValues failed: %s", err.Error())
+
 		return nil, error_helpers.NewErrorsAndWarning(err)
 	}
 
+	log.Printf("[INFO] collected unparsed input values for vars: %s", strings.Join(maps.Keys(inputValuesUnparsed), ","))
+
 	if validate {
 		if err := identifyAllMissingVariables(parseCtx, variableMap, inputValuesUnparsed); err != nil {
+			log.Printf("[INFO] identifyAllMissingVariables returned a validation error: %s", err.Error())
+
 			return nil, error_helpers.NewErrorsAndWarning(err)
 		}
 	}
+
 	// only parse values for public variables
 	parsedValues, diags := inputvars.ParseVariableValues(inputValuesUnparsed, variableMap, validate)
+	if diags.HasErrors() {
+		log.Printf("[INFO] ParseVariableValues returned error: %s", diags.Err())
+	} else {
+		log.Printf("[INFO] parsed values for public variables: %s", strings.Join(maps.Keys(parsedValues), ","))
+	}
 
 	if validate {
 		moreDiags := inputvars.CheckInputVariables(variableMap.PublicVariables, parsedValues)

pkg/steampipeconfig/modconfig/mod_variable_map.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/turbot/pipe-fittings/hclhelpers"
 	"github.com/turbot/steampipe/pkg/utils"
+	"golang.org/x/exp/maps"
 )
 
 // ModVariableMap is a struct containins maps of variable definitions
@@ -106,3 +107,23 @@ func (m *ModVariableMap) GetPublicVariableValues() (map[string]string, error) {
 	}
 	return res, nil
 }
+
+func (m *ModVariableMap) String() string {
+	var str strings.Builder
+	if len(m.RootVariables) > 0 {
+		str.WriteString("Root Variables: ")
+		str.WriteString(strings.Join(maps.Keys(m.RootVariables), ","))
+		str.WriteString(" ")
+	}
+	if len(m.DependencyVariables) > 0 {
+		str.WriteString("Dependency Variables: ")
+		str.WriteString(strings.Join(maps.Keys(m.DependencyVariables), ","))
+		str.WriteString(" ")
+	}
+	if len(m.PublicVariables) > 0 {
+		str.WriteString("Public Variables: ")
+		str.WriteString(strings.Join(maps.Keys(m.PublicVariables), ","))
+		str.WriteString(" ")
+	}
+	return str.String()
+}

pkg/steampipeconfig/modconfig/query_args_helpers.go

@@ -37,6 +37,8 @@ func ResolveArgs(qp QueryProvider, runtimeArgs *QueryArgs) ([]any, error) {
 		runtimeArgs = &QueryArgs{}
 	}
 
+	log.Printf("[TRACE] ResolveArgs: resolving args for %s", qp.Name())
+
 	// merge the query provider args (if any) with the runtime args
 	sourceArgs := qp.GetArgs()
 	if sourceArgs == nil {
@@ -44,27 +46,34 @@ func ResolveArgs(qp QueryProvider, runtimeArgs *QueryArgs) ([]any, error) {
 	}
 	mergedArgs, err := sourceArgs.Merge(runtimeArgs, qp)
 	if err != nil {
+		log.Printf("[WARN] ResolveArgs failed to merge args for %s: %s", qp.Name(), err.Error())
 		return nil, err
 	}
+
 	if namedArgCount := len(mergedArgs.ArgMap); namedArgCount > 0 {
+		log.Printf("[TRACE] %s defines %d named %s", qp.Name(), namedArgCount, utils.Pluralize("arg", namedArgCount))
 		// if named args are provided and the query does not define params, we cannot resolve the args
 		if len(qp.GetParams()) == 0 {
 			log.Printf("[TRACE] %s defines %d named %s but has no parameters definitions", qp.Name(), namedArgCount, utils.Pluralize("arg", namedArgCount))
 		} else {
 			// do params contain named params?
 			argVals, missingParams, err = mergedArgs.resolveNamedParameters(qp)
+			log.Printf("[TRACE] resolved %d named %s for %s", len(argVals), utils.Pluralize("params", len(argVals)), qp.Name())
 		}
 	} else {
 		// resolve as positional parameters
 		// (or fall back to defaults if no positional params are present)
 		argVals, missingParams, err = mergedArgs.resolvePositionalParameters(qp)
+		log.Printf("[TRACE] resolved %d positional %s for %s", len(argVals), utils.Pluralize("params", len(argVals)), qp.Name())
 	}
 	if err != nil {
+		log.Printf("[WARN] ResolveArgs failed to resolve args for %s: %s", qp.Name(), err.Error())
 		return nil, err
 	}
 
 	// did we resolve them all?
 	if len(missingParams) > 0 {
+		log.Printf("[WARN] ResolveArgs: args missing for %s: %s", qp.Name(), strings.Join(missingParams, ","))
 		// a better error will be constructed by the calling code
 		return nil, fmt.Errorf("%s", strings.Join(missingParams, ","))
 	}

pkg/steampipeconfig/modconfig/require.go

@@ -23,8 +23,8 @@ type Require struct {
 	modMap map[string]*ModVersionConstraint
 	// range of the definition of the require block
 	DeclRange hcl.Range
-	// range of the body of the require block
-	BodyRange hcl.Range
+	// range of the require block type
+	TypeRange hcl.Range
 }
 
 func NewRequire() *Require {
@@ -38,7 +38,7 @@ func (r *Require) Clone() *Require {
 	require.Plugins = r.Plugins
 	require.Mods = r.Mods
 	require.DeclRange = r.DeclRange
-	require.BodyRange = r.BodyRange
+	require.TypeRange = r.TypeRange
 
 	// we need to shallow copy the map
 	// if we don't, when the other one gets
@@ -75,7 +75,7 @@ func (r *Require) initialise(modBlock *hcl.Block) hcl.Diagnostics {
 
 	// set our Ranges
 	r.DeclRange = hclhelpers.BlockRange(requireBlock)
-	r.BodyRange = requireBlock.Body.(*hclsyntax.Body).SrcRange
+	r.TypeRange = requireBlock.TypeRange
 
 	// build maps of plugin and mod blocks
 	pluginBlockMap := hclhelpers.BlocksToMap(hclhelpers.FindChildBlocks(requireBlock, BlockTypePlugin))

pkg/version/version.go

@@ -18,7 +18,7 @@ Also https://www.digitalocean.com/community/tutorials/using-ldflags-to-set-versi
 **/
 
 // The main version number that is being run at the moment.
-var steampipeVersion = "0.21.6"
+var steampipeVersion = "0.21.7"
 
 // A pre-release marker for the version. If this is "" (empty string)
 // then it means that it is a final release. Otherwise, this is a pre-release

pkg/workspace/workspace.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"golang.org/x/exp/maps"
 	"log"
 	"os"
 	"path/filepath"
@@ -69,6 +70,7 @@ type Workspace struct {
 
 // LoadWorkspaceVars creates a Workspace and loads the variables
 func LoadWorkspaceVars(ctx context.Context) (*Workspace, *error_helpers.ErrorAndWarnings) {
+	log.Printf("[INFO] LoadWorkspaceVars: creating workspace, loading variable and resolving variable values")
 	workspacePath := viper.GetString(constants.ArgModLocation)
 
 	utils.LogTime("workspace.Load start")
@@ -76,17 +78,23 @@ func LoadWorkspaceVars(ctx context.Context) (*Workspace, *error_helpers.ErrorAnd
 
 	workspace, err := createShellWorkspace(workspacePath)
 	if err != nil {
+		log.Printf("[INFO] createShellWorkspace failed %s", err.Error())
 		return nil, error_helpers.NewErrorsAndWarning(err)
 	}
 
 	// check if your workspace path is home dir and if modfile exists - if yes then warn and ask user to continue or not
 	if err := HomeDirectoryModfileCheck(ctx, workspacePath); err != nil {
+		log.Printf("[INFO] HomeDirectoryModfileCheck failed %s", err.Error())
 		return nil, error_helpers.NewErrorsAndWarning(err)
 	}
-	errorsAndWarnings := workspace.populateVariables(ctx)
+	errorsAndWarnings := workspace.PopulateVariables(ctx)
 	if errorsAndWarnings.Error != nil {
+		log.Printf("[WARN] PopulateVariables failed %s", errorsAndWarnings.Error.Error())
 		return nil, errorsAndWarnings
 	}
+
+	log.Printf("[INFO] LoadWorkspaceVars succededed - got values for vars: %s", strings.Join(maps.Keys(workspace.VariableValues), ", "))
+
 	return workspace, errorsAndWarnings
 }
 
@@ -340,13 +348,13 @@ func (w *Workspace) LoadWorkspaceMod(ctx context.Context) *error_helpers.ErrorAn
 	return errorsAndWarnings
 }
 
-func (w *Workspace) populateVariables(ctx context.Context) *error_helpers.ErrorAndWarnings {
+func (w *Workspace) PopulateVariables(ctx context.Context) *error_helpers.ErrorAndWarnings {
+	log.Printf("[TRACE] Workspace.PopulateVariables")
 	// resolve values of all input variables
 	// we WILL validate missing variables when loading
 	validateMissing := true
 	inputVariables, errorsAndWarnings := w.getInputVariables(ctx, validateMissing)
 	if errorsAndWarnings.Error != nil {
-
 		// so there was an error - was it missing variables error
 		var missingVariablesError steampipeconfig.MissingVariableError
 		ok := errors.As(errorsAndWarnings.GetError(), &missingVariablesError)
@@ -389,21 +397,21 @@ func (w *Workspace) populateVariables(ctx context.Context) *error_helpers.ErrorA
 }
 
 func (w *Workspace) getInputVariables(ctx context.Context, validateMissing bool) (*modconfig.ModVariableMap, *error_helpers.ErrorAndWarnings) {
+	log.Printf("[TRACE] Workspace.getInputVariables")
 	// build a run context just to use to load variable definitions
 	variablesParseCtx, err := w.getParseContext(ctx)
 	if err != nil {
 		return nil, error_helpers.NewErrorsAndWarning(err)
 	}
 
-	return w.getVariableValues(ctx, variablesParseCtx, validateMissing)
-}
-
-func (w *Workspace) getVariableValues(ctx context.Context, variablesParseCtx *parse.ModParseContext, validateMissing bool) (*modconfig.ModVariableMap, *error_helpers.ErrorAndWarnings) {
 	// load variable definitions
 	variableMap, err := steampipeconfig.LoadVariableDefinitions(ctx, w.Path, variablesParseCtx)
 	if err != nil {
 		return nil, error_helpers.NewErrorsAndWarning(err)
 	}
+
+	log.Printf("[INFO] loaded variable definitions: %s", variableMap)
+
 	// get the values
 	return steampipeconfig.GetVariableValues(variablesParseCtx, variableMap, validateMissing)
 }

pkg/workspace/workspace_events.go

@@ -115,10 +115,21 @@ func (w *Workspace) onNewIntrospectionData(ctx context.Context, client db_common
 	}
 }
 
-func (w *Workspace) reloadResourceMaps(ctx context.Context) (*modconfig.ResourceMaps, *modconfig.ResourceMaps, *error_helpers.ErrorAndWarnings) {
+func (w *Workspace) reloadResourceMaps(ctx context.Context) (_ *modconfig.ResourceMaps, _ *modconfig.ResourceMaps, errAndWarnings *error_helpers.ErrorAndWarnings) {
 	w.loadLock.Lock()
 	defer w.loadLock.Unlock()
 
+	defer func() {
+		if errAndWarnings.GetError() != nil {
+			// check the existing watcher error - if we are already in an error state, do not show error
+			if w.watcherError == nil {
+				w.fileWatcherErrorHandler(ctx, error_helpers.PrefixError(errAndWarnings.GetError(), "failed to reload workspace"))
+			}
+			// now set watcher error to new error
+			w.watcherError = errAndWarnings.GetError()
+		}
+	}()
+
 	// get the pre-load resource maps
 	// NOTE: do not call GetResourceMaps - we DO NOT want to lock loadLock
 	prevResourceMaps := w.Mod.ResourceMaps
@@ -128,14 +139,12 @@ func (w *Workspace) reloadResourceMaps(ctx context.Context) (*modconfig.Resource
 	}
 
 	// now reload the workspace
-	errAndWarnings := w.LoadWorkspaceMod(ctx)
+	errAndWarnings = w.PopulateVariables(ctx)
+	if errAndWarnings.GetError() != nil {
+		return nil, nil, errAndWarnings
+	}
+	errAndWarnings = w.LoadWorkspaceMod(ctx)
 	if errAndWarnings.GetError() != nil {
-		// check the existing watcher error - if we are already in an error state, do not show error
-		if w.watcherError == nil {
-			w.fileWatcherErrorHandler(ctx, error_helpers.PrefixError(errAndWarnings.GetError(), "failed to reload workspace"))
-		}
-		// now set watcher error to new error
-		w.watcherError = errAndWarnings.GetError()
 		return nil, nil, errAndWarnings
 	}
 	// clear watcher error

tests/acceptance/test_data/mods/functionality_test_mod/.mod.cache.json

@@ -0,0 +1,11 @@
+{
+  "functionality_test_mod": {
+    "github.com/turbot/steampipe-mod-aws-compliance": {
+      "name": "github.com/turbot/steampipe-mod-aws-compliance",
+      "alias": "aws_compliance",
+      "version": "0.85.0",
+      "constraint": "*",
+      "struct_version": 20220411
+    }
+  }
+}

tests/acceptance/test_data/mods/functionality_test_mod_pp/.mod.cache.json

@@ -0,0 +1,11 @@
+{
+  "functionality_test_mod_pp": {
+    "github.com/turbot/steampipe-mod-aws-compliance": {
+      "name": "github.com/turbot/steampipe-mod-aws-compliance",
+      "alias": "aws_compliance",
+      "version": "0.85.0",
+      "constraint": "*",
+      "struct_version": 20220411
+    }
+  }
+}