Update workspace profile parsing to load options blocks. Include these options values in viper with correct precedence. Closes #2579

cmd/dashboard.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	filehelpers "github.com/turbot/go-kit/files"
 	"github.com/turbot/go-kit/helpers"
 	"github.com/turbot/steampipe-plugin-sdk/v4/logging"
 	"github.com/turbot/steampipe/pkg/cloud"
@@ -65,7 +64,6 @@ The current mod is the working directory, or the directory specified by the --mo
 		// Cobra will interpret values passed to a StringSliceFlag as CSV, where args passed to StringArrayFlag are not parsed and used raw
 		AddStringArrayFlag(constants.ArgDashboardInput, "", nil, "Specify the value of a dashboard input").
 		AddStringSliceFlag(constants.ArgSnapshotTag, "", nil, "Specify tags to set on the snapshot").
-		AddStringArrayFlag(constants.ArgSourceSnapshot, "", nil, "Specify one or more snapshots to display").
 		AddStringSliceFlag(constants.ArgExport, "", nil, "Export output to a snapshot file").
 		// hidden flags that are used internally
 		AddBoolFlag(constants.ArgServiceMode, "", false, "Hidden flag to specify whether this is starting as a service", cmdconfig.FlagOptions.Hidden())
@@ -204,19 +202,6 @@ func displaySnapshot(snapshot *dashboardtypes.SteampipeSnapshot) {
 }
 
 func initDashboard(ctx context.Context) *initialisation.InitData {
-	sourceSnapshots := viper.GetStringSlice(constants.ArgSourceSnapshot)
-	if len(sourceSnapshots) > 0 {
-		for _, s := range sourceSnapshots {
-			if !filehelpers.FileExists(s) {
-				return initialisation.NewErrorInitData(fmt.Errorf("source snapshot' %s' does not exist", s))
-			}
-		}
-		dashboardserver.OutputWait(ctx, "Loading Source Snapshots")
-		w := workspace.NewSourceSnapshotWorkspace(sourceSnapshots)
-		// return init data containing only this workspace - do not initialise it
-		return initialisation.NewInitData(w)
-	}
-
 	dashboardserver.OutputWait(ctx, "Loading Workspace")
 	w, err := interactive.LoadWorkspacePromptingForVariables(ctx)
 	if err != nil {

cmd/root.go

@@ -169,7 +169,7 @@ func initGlobalConfig() {
 
 	// 4) if an explicit workspace profile was set, add to viper as highest precedence default
 	if viper.IsSet(constants.ArgWorkspaceProfile) {
-		cmdconfig.SetDefaultsFromWorkspaceProfile(workspaceProfile)
+		cmdconfig.SetDefaultsFromConfig(workspaceProfile.ConfigMap())
 		// tildefy all paths in viper
 		// (this has already been done in BootstrapViper but we may have added a path from the workspace profile)
 		err = cmdconfig.TildefyPaths()

pkg/cmdconfig/viper.go

@@ -22,7 +22,7 @@ func BootstrapViper(defaultWorkspaceProfile *modconfig.WorkspaceProfile) error {
 	setBaseDefaults()
 
 	// set defaults from defaultWorkspaceProfile
-	SetDefaultsFromWorkspaceProfile(defaultWorkspaceProfile)
+	SetDefaultsFromConfig(defaultWorkspaceProfile.ConfigMap())
 
 	// set defaults from env vars
 	setDefaultsFromEnv()
@@ -49,27 +49,6 @@ func TildefyPaths() error {
 	return nil
 }
 
-func SetDefaultsFromWorkspaceProfile(profile *modconfig.WorkspaceProfile) {
-	if profile.CloudHost != "" {
-		viper.SetDefault(constants.ArgCloudHost, profile.CloudHost)
-	}
-	if profile.CloudToken != "" {
-		viper.SetDefault(constants.ArgCloudToken, profile.CloudToken)
-	}
-	if profile.InstallDir != "" {
-		viper.SetDefault(constants.ArgInstallDir, profile.InstallDir)
-	}
-	if profile.ModLocation != "" {
-		viper.SetDefault(constants.ArgModLocation, profile.ModLocation)
-	}
-	if profile.SnapshotLocation != "" {
-		viper.SetDefault(constants.ArgSnapshotLocation, profile.SnapshotLocation)
-	}
-	if profile.WorkspaceDatabase != "" {
-		viper.SetDefault(constants.ArgWorkspaceDatabase, profile.WorkspaceDatabase)
-	}
-}
-
 // SetDefaultsFromConfig overrides viper default values from hcl config values
 func SetDefaultsFromConfig(configMap map[string]interface{}) {
 	for k, v := range configMap {

pkg/constants/args.go

@@ -52,7 +52,6 @@ const (
 	ArgShare               = "share"
 	ArgSnapshot            = "snapshot"
 	ArgSnapshotTag         = "snapshot-tag"
-	ArgSourceSnapshot      = "source-snapshot"
 	ArgWorkspaceProfile    = "workspace"
 	ArgModLocation         = "mod-location"
 	ArgSnapshotLocation    = "snapshot-location"

pkg/control/controlexecute/control_run.go

@@ -182,8 +182,8 @@ func (r *ControlRun) setSearchPath(ctx context.Context, session *db_common.Datab
 	utils.LogTime("ControlRun.setSearchPath start")
 	defer utils.LogTime("ControlRun.setSearchPath end")
 
-	searchPath := []string{}
-	searchPathPrefix := []string{}
+	var searchPath []string
+	var searchPathPrefix []string
 
 	if r.Control.SearchPath == nil && r.Control.SearchPathPrefix == nil {
 		return nil

pkg/control/controlexecute/result_group.go

@@ -154,7 +154,7 @@ func (r *ResultGroup) AllTagKeys() []string {
 			tags = append(tags, k)
 		}
 	}
-	tags = utils.StringSliceDistinct(tags)
+	tags = helpers.StringSliceDistinct(tags)
 	sort.Strings(tags)
 	return tags
 }
@@ -244,7 +244,7 @@ func (r *ResultGroup) addDimensionKeys(keys ...string) {
 	if r.Parent != nil {
 		r.Parent.addDimensionKeys(keys...)
 	}
-	r.DimensionKeys = utils.StringSliceDistinct(r.DimensionKeys)
+	r.DimensionKeys = helpers.StringSliceDistinct(r.DimensionKeys)
 	sort.Strings(r.DimensionKeys)
 }
 

pkg/db/db_client/db_client_search_path.go

@@ -86,9 +86,12 @@ func (c *DbClient) GetRequiredSessionSearchPath() []string {
 }
 
 func (c *DbClient) ContructSearchPath(ctx context.Context, customSearchPath, searchPathPrefix []string) ([]string, error) {
+	// strip empty elements from search path and prefix
+	customSearchPath = helpers.RemoveFromStringSlice(customSearchPath, "")
+	searchPathPrefix = helpers.RemoveFromStringSlice(searchPathPrefix, "")
+
 	// store custom search path and search path prefix
 	c.searchPathPrefix = searchPathPrefix
-
 	var requiredSearchPath []string
 	// if a search path was passed, add 'internal' to the end
 	if len(customSearchPath) > 0 {

pkg/steampipeconfig/modconfig/config_map.go

@@ -0,0 +1,35 @@
+package modconfig
+
+import (
+	"fmt"
+	"github.com/turbot/go-kit/helpers"
+	"github.com/turbot/steampipe/pkg/steampipeconfig/options"
+	"reflect"
+	"strings"
+)
+
+type ConfigMap map[string]interface{}
+
+// SetStringItem checks is string pointer is non-nul and if so, add to map with given key
+func (m ConfigMap) SetStringItem(argValue *string, argName string) {
+	if argValue != nil {
+		m[argName] = *argValue
+	}
+}
+
+// PopulateConfigMapForOptions populates the config map for a given options object
+// NOTE: this mutates configMap
+func (m ConfigMap) PopulateConfigMapForOptions(o options.Options) {
+	for k, v := range o.ConfigMap() {
+		m[k] = v
+		// also store a scoped version of the config property
+		m[getScopedKey(o, k)] = v
+	}
+}
+
+// generated a scoped key for the config property. For example if o is a database options object and k is 'search-path'
+// the scoped key will be 'database.search-path'
+func getScopedKey(o options.Options, k string) string {
+	t := reflect.TypeOf(helpers.DereferencePointer(o)).Name()
+	return fmt.Sprintf("%s.%s", strings.ToLower(t), k)
+}

pkg/steampipeconfig/modconfig/workspace_profile.go

@@ -3,6 +3,7 @@ package modconfig
 import (
 	"fmt"
 	"github.com/hashicorp/hcl/v2"
+	"github.com/turbot/steampipe/pkg/constants"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/options"
 	"github.com/zclconf/go-cty/cty"
 	"reflect"
@@ -10,18 +11,18 @@ import (
 
 type WorkspaceProfile struct {
 	ProfileName       string            `hcl:"name,label" cty:"name"`
-	CloudHost         string            `hcl:"cloud_host,optional" cty:"cloud_host"`
-	CloudToken        string            `hcl:"cloud_token,optional" cty:"cloud_token"`
-	InstallDir        string            `hcl:"install_dir,optional" cty:"install_dir"`
-	ModLocation       string            `hcl:"mod_location,optional" cty:"mod_location"`
-	SnapshotLocation  string            `hcl:"snapshot_location,optional" cty:"snapshot_location"`
-	WorkspaceDatabase string            `hcl:"workspace_database,optional" cty:"workspace_database"`
+	CloudHost         *string           `hcl:"cloud_host,optional" cty:"cloud_host"`
+	CloudToken        *string           `hcl:"cloud_token,optional" cty:"cloud_token"`
+	InstallDir        *string           `hcl:"install_dir,optional" cty:"install_dir"`
+	ModLocation       *string           `hcl:"mod_location,optional" cty:"mod_location"`
+	SnapshotLocation  *string           `hcl:"snapshot_location,optional" cty:"snapshot_location"`
+	WorkspaceDatabase *string           `hcl:"workspace_database,optional" cty:"workspace_database"`
 	Base              *WorkspaceProfile `hcl:"base"`
 
 	// options
-	ConnectionOptions *options.Connection
-	TerminalOptions   *options.Terminal
 	GeneralOptions    *options.General
+	TerminalOptions   *options.Terminal
+	ConnectionOptions *options.Connection
 	DeclRange         hcl.Range
 }
 
@@ -38,10 +39,19 @@ func (p *WorkspaceProfile) SetOptions(opts options.Options, block *hcl.Block) hc
 	var diags hcl.Diagnostics
 	switch o := opts.(type) {
 	case *options.Connection:
+		if p.ConnectionOptions != nil {
+			diags = append(diags, duplicateOptionsBlockDiag(block))
+		}
 		p.ConnectionOptions = o
 	case *options.Terminal:
+		if p.TerminalOptions != nil {
+			diags = append(diags, duplicateOptionsBlockDiag(block))
+		}
 		p.TerminalOptions = o
 	case *options.General:
+		if p.GeneralOptions != nil {
+			diags = append(diags, duplicateOptionsBlockDiag(block))
+		}
 		p.GeneralOptions = o
 	default:
 		diags = append(diags, &hcl.Diagnostic{
@@ -53,6 +63,14 @@ func (p *WorkspaceProfile) SetOptions(opts options.Options, block *hcl.Block) hc
 	return diags
 }
 
+func duplicateOptionsBlockDiag(block *hcl.Block) *hcl.Diagnostic {
+	return &hcl.Diagnostic{
+		Severity: hcl.DiagError,
+		Summary:  fmt.Sprintf("duplicate %s options block", block.Type),
+		Subject:  &block.DefRange,
+	}
+}
+
 func (p *WorkspaceProfile) Name() string {
 	return fmt.Sprintf("workspace.%s", p.ProfileName)
 }
@@ -66,33 +84,58 @@ func (p *WorkspaceProfile) OnDecoded() hcl.Diagnostics {
 	return nil
 }
 
-//func (c *WorkspaceProfile) AddReference(*ResourceReference)     {}
-//func (c *WorkspaceProfile) GetReferences() []*ResourceReference { return nil }
-//func (c *WorkspaceProfile) GetDeclRange() *hcl.Range {
-//	return &c.DeclRange
-//}
-
 func (p *WorkspaceProfile) setBaseProperties() {
 	if p.Base == nil {
 		return
 	}
 
-	if p.CloudHost == "" {
+	if p.CloudHost == nil {
 		p.CloudHost = p.Base.CloudHost
 	}
-	if p.CloudToken == "" {
+	if p.CloudToken == nil {
 		p.CloudToken = p.Base.CloudToken
 	}
-	if p.InstallDir == "" {
+	if p.InstallDir == nil {
 		p.InstallDir = p.Base.InstallDir
 	}
-	if p.ModLocation == "" {
+	if p.ModLocation == nil {
 		p.ModLocation = p.Base.ModLocation
 	}
-	if p.SnapshotLocation == "" {
+	if p.SnapshotLocation == nil {
 		p.SnapshotLocation = p.Base.SnapshotLocation
 	}
-	if p.WorkspaceDatabase == "" {
+	if p.WorkspaceDatabase == nil {
 		p.WorkspaceDatabase = p.Base.WorkspaceDatabase
 	}
 }
+
+// ConfigMap creates a config map containing all options to pass to viper
+func (p *WorkspaceProfile) ConfigMap() map[string]interface{} {
+	res := ConfigMap{}
+	// add non-empty properties to config map
+
+	res.SetStringItem(p.CloudHost, constants.ArgCloudHost)
+	res.SetStringItem(p.CloudToken, constants.ArgCloudToken)
+	res.SetStringItem(p.InstallDir, constants.ArgInstallDir)
+	res.SetStringItem(p.ModLocation, constants.ArgModLocation)
+	res.SetStringItem(p.SnapshotLocation, constants.ArgSnapshotLocation)
+	res.SetStringItem(p.WorkspaceDatabase, constants.ArgWorkspaceDatabase)
+
+	// now add options
+	// build flat config map with order or precedence (low to high): general, terminal, connection
+	// this means if (for example) 'search-path' is set in both terminal and connection options,
+	// the value from connection options will have precedence
+	// however, we also store all values scoped by their options type, so we will store:
+	// 'database.search-path', 'terminal.search-path' AND 'search-path' (which will be equal to 'terminal.search-path')
+	if p.GeneralOptions != nil {
+		res.PopulateConfigMapForOptions(p.GeneralOptions)
+	}
+	if p.TerminalOptions != nil {
+		res.PopulateConfigMapForOptions(p.TerminalOptions)
+	}
+	if p.ConnectionOptions != nil {
+		res.PopulateConfigMapForOptions(p.ConnectionOptions)
+	}
+
+	return res
+}

pkg/steampipeconfig/parse/schema.go

@@ -32,11 +32,7 @@ var WorkspaceProfileListBlockSchema = &hcl.BodySchema{
 }
 
 var WorkspaceProfileBlockSchema = &hcl.BodySchema{
-	Attributes: []hcl.AttributeSchema{
-		{
-			Name: "base",
-		},
-	},
+
 	Blocks: []hcl.BlockHeaderSchema{
 		{
 			Type:       "options",

pkg/steampipeconfig/parse/workspace_profile.go

@@ -63,7 +63,6 @@ func LoadWorkspaceProfiles(workspaceProfilePath string) (profileMap map[string]*
 
 }
 func parseWorkspaceProfiles(parseCtx *WorkspaceProfileParseContext) (map[string]*modconfig.WorkspaceProfile, error) {
-
 	// we may need to decode more than once as we gather dependencies as we go
 	// continue decoding as long as the number of unresolved blocks decreases
 	prevUnresolvedBlocks := 0
@@ -82,7 +81,7 @@ func parseWorkspaceProfiles(parseCtx *WorkspaceProfileParseContext) (map[string]
 		// if the number of unresolved blocks has NOT reduced, fail
 		if prevUnresolvedBlocks != 0 && unresolvedBlocks >= prevUnresolvedBlocks {
 			str := parseCtx.FormatDependencies()
-			return nil, fmt.Errorf("failed to resolve mod dependencies after %d attempts\nDependencies:\n%s", attempts+1, str)
+			return nil, fmt.Errorf("failed to resolve workspace profile dependencies after %d attempts\nDependencies:\n%s", attempts+1, str)
 		}
 		// update prevUnresolvedBlocks
 		prevUnresolvedBlocks = unresolvedBlocks
@@ -120,18 +119,47 @@ func decodeWorkspaceProfiles(parseCtx *WorkspaceProfileParseContext) (map[string
 	return profileMap, diags
 }
 
-// func DecodeWorkspaceProfiles(content *hcl.BodyContent, workspaceProfilePath string) map[string]*modconfig.WorkspaceProfile {
-// }
 func decodeWorkspaceProfile(block *hcl.Block, parseCtx *WorkspaceProfileParseContext) (*modconfig.WorkspaceProfile, *decodeResult) {
 	res := newDecodeResult()
 	// get shell resource
 	resource := modconfig.NewWorkspaceProfile(block)
 
-	diags := gohcl.DecodeBody(block.Body, parseCtx.EvalCtx, resource)
+	// do a partial decode to get options blocks into workspaceProfileOptions, with all other attributes in rest
+	workspaceProfileOptions, rest, diags := block.Body.PartialContent(WorkspaceProfileBlockSchema)
+	if diags.HasErrors() {
+		res.handleDecodeDiags(diags)
+		return nil, res
+	}
+
+	diags = gohcl.DecodeBody(rest, parseCtx.EvalCtx, resource)
 	if len(diags) > 0 {
 		res.handleDecodeDiags(diags)
 	}
 
+	for _, block := range workspaceProfileOptions.Blocks {
+		switch block.Type {
+		case "options":
+			// if we already found settings, fail
+			opts, moreDiags := DecodeOptions(block)
+			if moreDiags.HasErrors() {
+				diags = append(diags, moreDiags...)
+				break
+			}
+			moreDiags = resource.SetOptions(opts, block)
+			if moreDiags.HasErrors() {
+				diags = append(diags, moreDiags...)
+			}
+
+		default:
+			// this should never happen
+			diags = append(diags, &hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  fmt.Sprintf("invalid block type '%s' - only 'options' blocks are supported for Connections", block.Type),
+				Subject:  &block.DefRange,
+			})
+		}
+	}
+
 	handleWorkspaceProfileDecodeResult(resource, res, block, parseCtx)
 	return resource, res
 }

pkg/steampipeconfig/steampipeconfig.go

@@ -4,11 +4,9 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"reflect"
 	"strings"
 
 	"github.com/hashicorp/go-version"
-	"github.com/turbot/go-kit/helpers"
 	"github.com/turbot/go-kit/types"
 	"github.com/turbot/steampipe/pkg/constants"
 	"github.com/turbot/steampipe/pkg/ociinstaller"
@@ -59,7 +57,7 @@ func (c *SteampipeConfig) Validate() error {
 
 // ConfigMap creates a config map to pass to viper
 func (c *SteampipeConfig) ConfigMap() map[string]interface{} {
-	res := map[string]interface{}{}
+	res := modconfig.ConfigMap{}
 
 	// build flat config map with order or precedence (low to high): general, database, terminal
 	// this means if (for example) 'search-path' is set in both database and terminal options,
@@ -67,35 +65,18 @@ func (c *SteampipeConfig) ConfigMap() map[string]interface{} {
 	// however, we also store all values scoped by their options type, so we will store:
 	// 'database.search-path', 'terminal.search-path' AND 'search-path' (which will be equal to 'terminal.search-path')
 	if c.GeneralOptions != nil {
-		c.populateConfigMapForOptions(c.GeneralOptions, res)
+		res.PopulateConfigMapForOptions(c.GeneralOptions)
 	}
 	if c.DatabaseOptions != nil {
-		c.populateConfigMapForOptions(c.DatabaseOptions, res)
+		res.PopulateConfigMapForOptions(c.DatabaseOptions)
 	}
 	if c.TerminalOptions != nil {
-		c.populateConfigMapForOptions(c.TerminalOptions, res)
+		res.PopulateConfigMapForOptions(c.TerminalOptions)
 	}
 
 	return res
 }
 
-// populate the config map for a given options object
-// NOTE: this mutates configMap
-func (c *SteampipeConfig) populateConfigMapForOptions(o options.Options, configMap map[string]interface{}) {
-	for k, v := range o.ConfigMap() {
-		configMap[k] = v
-		// also store a scoped version of the config property
-		configMap[getScopedKey(o, k)] = v
-	}
-}
-
-// generated a scoped key for the config property. For example if o is a database options object and k is 'search-path'
-// the scoped key will be 'database.search-path'
-func getScopedKey(o options.Options, k string) string {
-	t := reflect.TypeOf(helpers.DereferencePointer(o)).Name()
-	return fmt.Sprintf("%s.%s", strings.ToLower(t), k)
-}
-
 func (c *SteampipeConfig) SetOptions(opts options.Options) {
 	switch o := opts.(type) {
 	case *options.Connection:

pkg/steampipeconfig/workspace_profile_loader.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/parse"
+	"github.com/turbot/steampipe/pkg/utils"
 	"log"
 )
 
@@ -64,8 +65,8 @@ func (l *WorkspaceProfileLoader) getImplicitWorkspace(name string) *modconfig.Wo
 	if IsCloudWorkspaceIdentifier(name) {
 		log.Printf("[TRACE] getImplicitWorkspace - %s is implicit workspace: SnapshotLocation=%s, WorkspaceDatabase=%s", name, name, name)
 		return &modconfig.WorkspaceProfile{
-			SnapshotLocation:  name,
-			WorkspaceDatabase: name,
+			SnapshotLocation:  utils.ToStringPointer(name),
+			WorkspaceDatabase: utils.ToStringPointer(name),
 		}
 	}
 	return nil

pkg/utils/string_slice.go

@@ -2,19 +2,6 @@ package utils
 
 import "strings"
 
-// TODO: investigate turbot/go-kit/helpers
-func StringSliceDistinct(slice []string) []string {
-	var res []string
-	occurenceMap := make(map[string]struct{})
-	for _, item := range slice {
-		occurenceMap[item] = struct{}{}
-	}
-	for item := range occurenceMap {
-		res = append(res, item)
-	}
-	return res
-}
-
 // UnquoteStringArray removes quote marks from elements of string array
 func UnquoteStringArray(stringArray []string) []string {
 	res := make([]string, len(stringArray))

pkg/workspace/workspace.go

@@ -83,15 +83,6 @@ func Load(ctx context.Context, workspacePath string) (*Workspace, error) {
 	return workspace, nil
 }
 
-// NewSourceSnapshotWorkspace creates a Workspace which contains ONLY source snapshoyt paths
-func NewSourceSnapshotWorkspace(sourceSnapshots []string) *Workspace {
-	return &Workspace{
-		SourceSnapshots: sourceSnapshots,
-		// empty mod to avoid referencing crashes
-		Mod: &modconfig.Mod{},
-	}
-}
-
 // LoadVariables creates a Workspace and uses it to load all variables, ignoring any value resolution errors
 // this is use for the variable list command
 func LoadVariables(ctx context.Context, workspacePath string) ([]*modconfig.Variable, error) {