Update CLI to upload snapshots to the cloud using --share and --snapshot options. Update Dashboard command to support passing a dashboard name as an argument. Closes #2365. Closes #2367

cmd/check.go

@@ -5,14 +5,13 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/turbot/steampipe/pkg/initialisation"
 	"io"
-	"log"
 	"os"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/mattn/go-isatty"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/turbot/go-kit/helpers"
@@ -24,6 +23,7 @@ import (
 	"github.com/turbot/steampipe/pkg/control/controlexecute"
 	"github.com/turbot/steampipe/pkg/control/controlstatus"
 	"github.com/turbot/steampipe/pkg/display"
+	"github.com/turbot/steampipe/pkg/interactive"
 	"github.com/turbot/steampipe/pkg/statushooks"
 	"github.com/turbot/steampipe/pkg/utils"
 	"github.com/turbot/steampipe/pkg/workspace"
@@ -62,7 +62,7 @@ You may specify one or more benchmarks or controls to run (separated by a space)
 		AddBoolFlag(constants.ArgHeader, "", true, "Include column headers for csv and table output").
 		AddBoolFlag(constants.ArgHelp, "h", false, "Help for check").
 		AddStringFlag(constants.ArgSeparator, "", ",", "Separator string for csv output").
-		AddStringFlag(constants.ArgOutput, "", constants.CheckOutputFormatText, "Select a console output format: brief, csv, html, json, md, text or none").
+		AddStringFlag(constants.ArgOutput, "", constants.OutputFormatText, "Select a console output format: brief, csv, html, json, md, text or none").
 		AddBoolFlag(constants.ArgTiming, "", false, "Turn on the timer which reports check time").
 		AddStringSliceFlag(constants.ArgSearchPath, "", nil, "Set a custom search_path for the steampipe user for a check session (comma-separated)").
 		AddStringSliceFlag(constants.ArgSearchPathPrefix, "", nil, "Set a prefix to the current search path for a check session (comma-separated)").
@@ -79,7 +79,9 @@ You may specify one or more benchmarks or controls to run (separated by a space)
 		AddStringFlag(constants.ArgWhere, "", "", "SQL 'where' clause, or named query, used to filter controls (cannot be used with '--tag')").
 		AddIntFlag(constants.ArgMaxParallel, "", constants.DefaultMaxConnections, "The maximum number of parallel executions", cmdconfig.FlagOptions.Hidden()).
 		AddBoolFlag(constants.ArgModInstall, "", true, "Specify whether to install mod dependencies before running the check").
-		AddBoolFlag(constants.ArgInput, "", true, "Enable interactive prompts")
+		AddBoolFlag(constants.ArgInput, "", true, "Enable interactive prompts").
+		AddStringFlag(constants.ArgSnapshot, "", "", "Create snapshot in Steampipe Cloud with the default (workspace) visibility.", cmdconfig.FlagOptions.NoOptDefVal(constants.ArgShareNoOptDefault)).
+		AddStringFlag(constants.ArgShare, "", "", "Create snapshot in Steampipe Cloud with 'anyone_with_link' visibility.", cmdconfig.FlagOptions.NoOptDefVal(constants.ArgShareNoOptDefault))
 
 	return cmd
 }
@@ -89,7 +91,7 @@ You may specify one or more benchmarks or controls to run (separated by a space)
 
 func runCheckCmd(cmd *cobra.Command, args []string) {
 	utils.LogTime("runCheckCmd start")
-	initData := &control.InitData{}
+	initData := &initialisation.InitData{}
 
 	// setup a cancel context and start cancel handler
 	ctx, cancel := context.WithCancel(cmd.Context())
@@ -104,17 +106,11 @@ func runCheckCmd(cmd *cobra.Command, args []string) {
 			exitCode = constants.ExitCodeUnknownErrorPanic
 		}
 
-		if initData.Client != nil {
-			log.Printf("[TRACE] close client")
-			initData.Client.Close(ctx)
-		}
-		if initData.Workspace != nil {
-			initData.Workspace.Close()
-		}
+		initData.Cleanup(ctx)
 	}()
 
 	// verify we have an argument
-	if !validateArgs(ctx, cmd, args) {
+	if !validateCheckArgs(ctx, cmd, args) {
 		return
 	}
 
@@ -125,13 +121,9 @@ func runCheckCmd(cmd *cobra.Command, args []string) {
 
 	// initialise
 	initData = initialiseCheck(ctx)
-
-	// check the init result - should we quit?
-	if err := handleCheckInitResult(ctx, initData); err != nil {
-		initData.Cleanup(ctx)
-		// if there was an error, display it
-		utils.FailOnError(err)
-	}
+	utils.FailOnError(initData.Result.Error)
+	// if there is a usage warning we display it
+	initData.Result.DisplayMessages()
 
 	// pull out useful properties
 	workspace := initData.Workspace
@@ -157,7 +149,7 @@ func runCheckCmd(cmd *cobra.Command, args []string) {
 
 		// create the execution tree
 		executionTree, err := controlexecute.NewExecutionTree(ctx, workspace, client, arg)
-		utils.FailOnErrorWithMessage(err, "failed to resolve controls from argument")
+		utils.FailOnError(err)
 
 		// execute controls synchronously (execute returns the number of failures)
 		failures += executionTree.Execute(ctx)
@@ -195,16 +187,10 @@ func runCheckCmd(cmd *cobra.Command, args []string) {
 
 // create the context for the check run - add a control status renderer
 func createCheckContext(ctx context.Context) context.Context {
-	var controlHooks controlstatus.ControlHooks = controlstatus.NullHooks
-	// if the client is a TTY, inject a status spinner
-	if isatty.IsTerminal(os.Stdout.Fd()) {
-		controlHooks = controlstatus.NewControlStatusHooks()
-	}
-
-	return controlstatus.AddControlHooksToContext(ctx, controlHooks)
+	return controlstatus.AddControlHooksToContext(ctx, controlstatus.NewStatusControlHooks())
 }
 
-func validateArgs(ctx context.Context, cmd *cobra.Command, args []string) bool {
+func validateCheckArgs(ctx context.Context, cmd *cobra.Command, args []string) bool {
 	if len(args) == 0 {
 		fmt.Println()
 		utils.ShowError(ctx, fmt.Errorf("you must provide at least one argument"))
@@ -214,39 +200,70 @@ func validateArgs(ctx context.Context, cmd *cobra.Command, args []string) bool {
 		exitCode = constants.ExitCodeInsufficientOrWrongArguments
 		return false
 	}
+	// only 1 of 'share' and 'snapshot' may be set
+	if len(viper.GetString(constants.ArgShare)) > 0 && len(viper.GetString(constants.ArgShare)) > 0 {
+		utils.ShowError(ctx, fmt.Errorf("only 1 of 'share' and 'dashboard' may be set"))
+		return false
+	}
 	return true
 }
 
-func initialiseCheck(ctx context.Context) *control.InitData {
+func initialiseCheck(ctx context.Context) *initialisation.InitData {
 	statushooks.SetStatus(ctx, "Initializing...")
 	defer statushooks.Done(ctx)
 
 	// load the workspace
-	w, err := loadWorkspacePromptingForVariables(ctx)
+	w, err := interactive.LoadWorkspacePromptingForVariables(ctx)
 	utils.FailOnErrorWithMessage(err, "failed to load workspace")
 
-	initData := control.NewInitData(ctx, w)
+	initData := initialisation.NewInitData(ctx, w)
+	if initData.Result.Error != nil {
+		return initData
+	}
 
+	// control specific init
 	if !w.ModfileExists() {
 		initData.Result.Error = workspace.ErrorNoModDefinition
 	}
 
+	if viper.GetString(constants.ArgOutput) == constants.OutputFormatNone {
+		// set progress to false
+		viper.Set(constants.ArgProgress, false)
+	}
+	// set color schema
+	err = initialiseCheckColorScheme()
+	if err != nil {
+		initData.Result.Error = err
+		return initData
+	}
+
+	if len(initData.Workspace.GetResourceMaps().Controls) == 0 {
+		initData.Result.AddWarnings("no controls found in current workspace")
+	}
+
+	if err := controldisplay.EnsureTemplates(); err != nil {
+		initData.Result.Error = err
+		return initData
+	}
+
 	return initData
 }
 
-func handleCheckInitResult(ctx context.Context, initData *control.InitData) error {
-	// if there is an error or cancellation we bomb out
-	if initData.Result.Error != nil {
-		return initData.Result.Error
+func initialiseCheckColorScheme() error {
+	theme := viper.GetString(constants.ArgTheme)
+	if !viper.GetBool(constants.ConfigKeyIsTerminalTTY) {
+		// enforce plain output for non-terminals
+		theme = "plain"
 	}
-	// cancelled?
-	if ctx != nil && ctx.Err() != nil {
-		return ctx.Err()
+	themeDef, ok := controldisplay.ColorSchemes[theme]
+	if !ok {
+		return fmt.Errorf("invalid theme '%s'", theme)
 	}
-
-	// if there is a usage warning we display it
-	initData.Result.DisplayMessages()
-
+	scheme, err := controldisplay.NewControlColorScheme(themeDef)
+	if err != nil {
+		return err
+	}
+	controldisplay.ControlColors = scheme
 	return nil
 }
 
@@ -266,7 +283,7 @@ func shouldPrintTiming() bool {
 	outputFormat := viper.GetString(constants.ArgOutput)
 
 	return (viper.GetBool(constants.ArgTiming) && !viper.GetBool(constants.ArgDryRun)) &&
-		(outputFormat == constants.CheckOutputFormatText || outputFormat == constants.CheckOutputFormatBrief)
+		(outputFormat == constants.OutputFormatText || outputFormat == constants.OutputFormatBrief)
 }
 
 func exportCheckResult(ctx context.Context, d *control.ExportData) {
@@ -328,7 +345,7 @@ func exportControlResults(ctx context.Context, executionTree *controlexecute.Exe
 			continue
 		}
 		// tactical solution to prettify the json output
-		if target.Formatter.GetFormatName() == "json" {
+		if target.Formatter.GetFormatName() == constants.OutputFormatJSON {
 			dataToExport, err = prettifyJsonFromReader(dataToExport)
 			if err != nil {
 				errors = append(errors, err)

cmd/dashboard.go

@@ -2,9 +2,13 @@ package cmd
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"github.com/turbot/steampipe/pkg/cloud"
+	"github.com/turbot/steampipe/pkg/initialisation"
 	"log"
 	"os"
+	"strings"
 
 	"github.com/turbot/steampipe/pkg/statushooks"
 	"github.com/turbot/steampipe/pkg/workspace"
@@ -16,20 +20,21 @@ import (
 	"github.com/turbot/steampipe/pkg/cmdconfig"
 	"github.com/turbot/steampipe/pkg/constants"
 	"github.com/turbot/steampipe/pkg/contexthelpers"
-	"github.com/turbot/steampipe/pkg/dashboard"
 	"github.com/turbot/steampipe/pkg/dashboard/dashboardassets"
 	"github.com/turbot/steampipe/pkg/dashboard/dashboardserver"
+	"github.com/turbot/steampipe/pkg/interactive"
+	"github.com/turbot/steampipe/pkg/snapshot"
 	"github.com/turbot/steampipe/pkg/utils"
 )
 
 func dashboardCmd() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:              "dashboard",
+		Use:              "dashboard [flags] [benchmark/dashboard]",
 		TraverseChildren: true,
 		Args:             cobra.ArbitraryArgs,
 		Run:              runDashboardCmd,
-		Short:            "Start the local dashboard UI",
-		Long: `Starts a local web server that enables real-time development of dashboards within the current mod.
+		Short:            "Start the local dashboard UI or run a named dashboard",
+		Long: `Either runs the a named dashboard or benchmark, or starts a local web server that enables real-time development of dashboards within the current mod.
 
 The current mod is the working directory, or the directory specified by the --workspace-chdir flag.`,
 	}
@@ -43,11 +48,17 @@ The current mod is the working directory, or the directory specified by the --wo
 		AddStringSliceFlag(constants.ArgSearchPath, "", nil, "Set a custom search_path for the steampipe user for a check session (comma-separated)").
 		AddStringSliceFlag(constants.ArgSearchPathPrefix, "", nil, "Set a prefix to the current search path for a check session (comma-separated)").
 		AddStringSliceFlag(constants.ArgVarFile, "", nil, "Specify an .spvar file containing variable values").
+		AddBoolFlag(constants.ArgProgress, "", true, "Display dashboard execution progress respected when a dashboard name argument is passed").
 		// NOTE: use StringArrayFlag for ArgVariable, not StringSliceFlag
-		// Cobra will interpret values passed to a StringSliceFlag as CSV,
-		// where args passed to StringArrayFlag are not parsed and used raw
+		// Cobra will interpret values passed to a StringSliceFlag as CSV, where args passed to StringArrayFlag are not parsed and used raw
 		AddStringArrayFlag(constants.ArgVariable, "", nil, "Specify the value of a variable").
 		AddBoolFlag(constants.ArgInput, "", true, "Enable interactive prompts").
+		AddStringFlag(constants.ArgOutput, "", constants.OutputFormatSnapshot, "Select a console output format: snapshot").
+		AddStringFlag(constants.ArgSnapshot, "", "", "Create snapshot in Steampipe Cloud with the default (workspace) visibility.", cmdconfig.FlagOptions.NoOptDefVal(constants.ArgShareNoOptDefault)).
+		AddStringFlag(constants.ArgShare, "", "", "Create snapshot in Steampipe Cloud with 'anyone_with_link' visibility.", cmdconfig.FlagOptions.NoOptDefVal(constants.ArgShareNoOptDefault)).
+		// NOTE: use StringArrayFlag for ArgDashboardInput, not StringSliceFlag
+		// Cobra will interpret values passed to a StringSliceFlag as CSV, where args passed to StringArrayFlag are not parsed and used raw
+		AddStringArrayFlag(constants.ArgDashboardInput, "", nil, "Specify the value of a dashboard input").
 		// hidden flags that are used internally
 		AddBoolFlag(constants.ArgServiceMode, "", false, "Hidden flag to specify whether this is starting as a service", cmdconfig.FlagOptions.Hidden())
 
@@ -57,17 +68,36 @@ The current mod is the working directory, or the directory specified by the --wo
 func runDashboardCmd(cmd *cobra.Command, args []string) {
 	dashboardCtx := cmd.Context()
 
+	var err error
 	logging.LogTime("runDashboardCmd start")
 	defer func() {
 		logging.LogTime("runDashboardCmd end")
 		if r := recover(); r != nil {
-			utils.ShowError(dashboardCtx, helpers.ToError(r))
+			err = helpers.ToError(r)
+			utils.ShowError(dashboardCtx, err)
 			if isRunningAsService() {
-				saveErrorToDashboardState(helpers.ToError(r))
+				saveErrorToDashboardState(err)
 			}
 		}
+		setExitCodeForDashboardError(err)
+
 	}()
 
+	// first check whether a dashboard name has been passed as an arg
+	dashboardName, err := validateDashboardArgs(args)
+	utils.FailOnError(err)
+	if dashboardName != "" {
+		inputs, err := collectInputs()
+		utils.FailOnError(err)
+
+		// run just this dashboard
+		err = runSingleDashboard(dashboardCtx, dashboardName, inputs)
+		utils.FailOnError(err)
+		// and we are done
+		return
+	}
+
+	// retrieve server params
 	serverPort := dashboardserver.ListenPort(viper.GetInt(constants.ArgDashboardPort))
 	utils.FailOnError(serverPort.IsValid())
 
@@ -84,29 +114,24 @@ func runDashboardCmd(cmd *cobra.Command, args []string) {
 	contexthelpers.StartCancelHandler(cancel)
 
 	// ensure dashboard assets are present and extract if not
-	err := dashboardassets.Ensure(dashboardCtx)
+	err = dashboardassets.Ensure(dashboardCtx)
 	utils.FailOnError(err)
 
 	// disable all status messages
 	dashboardCtx = statushooks.DisableStatusHooks(dashboardCtx)
 
 	// load the workspace
-	dashboardserver.OutputWait(dashboardCtx, "Loading Workspace")
-	w, err := loadWorkspacePromptingForVariables(dashboardCtx)
-	utils.FailOnErrorWithMessage(err, "failed to load workspace")
-
-	initData := dashboard.NewInitData(dashboardCtx, w)
-	// shutdown the service on exit
+	initData := initDashboard(dashboardCtx, err)
 	defer initData.Cleanup(dashboardCtx)
+	utils.FailOnError(initData.Result.Error)
 
-	err = handleDashboardInitResult(dashboardCtx, initData)
-	// if there was an error, display it
+	// if there is a usage warning we display it
+	initData.Result.DisplayMessages()
+
+	// create the server
+	server, err := dashboardserver.NewServer(dashboardCtx, initData.Client, initData.Workspace)
 	utils.FailOnError(err)
 
-	server, err := dashboardserver.NewServer(dashboardCtx, initData.Client, initData.Workspace)
-	if err != nil {
-		utils.FailOnError(err)
-	}
 	// start the server asynchronously - this returns a chan which is signalled when the internal API server terminates
 	doneChan := server.Start()
 
@@ -122,24 +147,103 @@ func runDashboardCmd(cmd *cobra.Command, args []string) {
 	log.Println("[TRACE] runDashboardCmd exiting")
 }
 
-// inspect the init result ands
-func handleDashboardInitResult(ctx context.Context, initData *dashboard.InitData) error {
-	// if there is an error or cancellation we bomb out
-	if err := initData.Result.Error; err != nil {
-		setExitCodeForDashboardError(err)
-		return initData.Result.Error
-	}
-	// cancelled?
-	if ctx != nil && ctx.Err() != nil {
-		return ctx.Err()
-	}
-	// if there is a usage warning we display it
-	initData.Result.DisplayMessages()
+func initDashboard(dashboardCtx context.Context, err error) *initialisation.InitData {
+	dashboardserver.OutputWait(dashboardCtx, "Loading Workspace")
+	w, err := interactive.LoadWorkspacePromptingForVariables(dashboardCtx)
+	utils.FailOnErrorWithMessage(err, "failed to load workspace")
 
+	// initialise
+	initData := initialisation.NewInitData(dashboardCtx, w)
+	// there must be a modfile
+	if !w.ModfileExists() {
+		initData.Result.Error = workspace.ErrorNoModDefinition
+	}
+
+	return initData
+}
+
+func runSingleDashboard(ctx context.Context, dashboardName string, inputs map[string]interface{}) error {
+	// so a dashboard name was specified - just call GenerateSnapshot
+	snapshot, err := snapshot.GenerateSnapshot(ctx, dashboardName, inputs)
+	if err != nil {
+		return err
+	}
+
+	shouldShare := viper.IsSet(constants.ArgShare)
+	shouldUpload := viper.IsSet(constants.ArgSnapshot)
+	if shouldShare || shouldUpload {
+		snapshotUrl, err := cloud.UploadSnapshot(snapshot, shouldShare)
+		statushooks.Done(ctx)
+		if err != nil {
+			return err
+		} else {
+			fmt.Printf("Snapshot uploaded to %s\n", snapshotUrl)
+		}
+		return err
+	}
+
+	// just display result
+	snapshotText, err := json.MarshalIndent(snapshot, "", "  ")
+	utils.FailOnError(err)
+	fmt.Println(string(snapshotText))
+	fmt.Println("")
 	return nil
 }
 
+func validateDashboardArgs(args []string) (string, error) {
+	if len(args) > 1 {
+		return "", fmt.Errorf("dashboard command accepts 0 or 1 argument")
+	}
+	dashboardName := ""
+	if len(args) == 1 {
+		dashboardName = args[0]
+	}
+
+	// only 1 of 'share' and 'snapshot' may be set
+	shareArg := viper.GetString(constants.ArgShare)
+	snapshotArg := viper.GetString(constants.ArgSnapshot)
+	if shareArg != "" && snapshotArg != "" {
+		return "", fmt.Errorf("only 1 of --share and --dashboard may be set")
+	}
+
+	// if either share' or 'snapshot' are set, a dashboard name an dcloud token must be provided
+	if shareArg != "" || snapshotArg != "" {
+		if dashboardName == "" {
+			return "", fmt.Errorf("dashboard name must be provided if --share or --snapshot arg is used")
+		}
+		snapshotWorkspace := shareArg
+		argName := "share"
+		if snapshotWorkspace == "" {
+			snapshotWorkspace = snapshotArg
+			argName = "snapshot"
+		}
+
+		// is this is the no-option default, use the workspace arg
+		if snapshotWorkspace == constants.ArgShareNoOptDefault {
+			snapshotWorkspace = viper.GetString(constants.ArgWorkspace)
+		}
+		if snapshotWorkspace == "" {
+			return "", fmt.Errorf("a Steampipe Cloud workspace name must be provided, either by setting %s=<workspace> or --workspace=<workspace>", argName)
+		}
+
+		// now write back the workspace to viper
+		viper.Set(constants.ArgWorkspace, snapshotWorkspace)
+
+		// verify cloud token
+		if !viper.IsSet(constants.ArgCloudToken) {
+			return "", fmt.Errorf("a Steampipe Cloud token must be provided")
+		}
+	}
+
+	return dashboardName, nil
+}
+
 func setExitCodeForDashboardError(err error) {
+	// if exit code already set, leave as is
+	if exitCode != 0 {
+		return
+	}
+
 	if err == workspace.ErrorNoModDefinition {
 		exitCode = constants.ExitCodeNoModFile
 	} else {
@@ -198,3 +302,27 @@ func saveDashboardState(serverPort dashboardserver.ListenPort, serverListen dash
 	}
 	utils.FailOnError(dashboardserver.WriteServiceStateFile(state))
 }
+
+func collectInputs() (map[string]interface{}, error) {
+	res := make(map[string]interface{})
+	inputArgs := viper.GetStringSlice(constants.ArgDashboardInput)
+	for _, variableArg := range inputArgs {
+		// Value should be in the form "name=value", where value is a string
+		raw := variableArg
+		eq := strings.Index(raw, "=")
+		if eq == -1 {
+			return nil, fmt.Errorf("the --dashboard-input argument '%s' is not correctly specified. It must be an input name and value separated an equals sign: --dashboard-input key=value", raw)
+		}
+		name := raw[:eq]
+		rawVal := raw[eq+1:]
+		if _, ok := res[name]; ok {
+			return nil, fmt.Errorf("the dashboard-input option '%s' is provided more than once", name)
+		}
+		// TACTICAL: add `input. to start of name
+		key := fmt.Sprintf("input.%s", name)
+		res[key] = rawVal
+	}
+
+	return res, nil
+
+}

cmd/query.go

@@ -2,9 +2,7 @@ package cmd
 
 import (
 	"bufio"
-	"context"
 	"fmt"
-	"log"
 	"os"
 	"strings"
 
@@ -17,7 +15,6 @@ import (
 	"github.com/turbot/steampipe/pkg/query"
 	"github.com/turbot/steampipe/pkg/query/queryexecute"
 	"github.com/turbot/steampipe/pkg/statushooks"
-	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
 	"github.com/turbot/steampipe/pkg/utils"
 	"github.com/turbot/steampipe/pkg/workspace"
 )
@@ -75,7 +72,10 @@ Examples:
 		// Cobra will interpret values passed to a StringSliceFlag as CSV,
 		// where args passed to StringArrayFlag are not parsed and used raw
 		AddStringArrayFlag(constants.ArgVariable, "", nil, "Specify the value of a variable").
-		AddBoolFlag(constants.ArgInput, "", true, "Enable interactive prompts")
+		AddBoolFlag(constants.ArgInput, "", true, "Enable interactive prompts").
+		AddStringFlag(constants.ArgSnapshot, "", "", "Create snapshot in Steampipe Cloud with the default (workspace) visibility.", cmdconfig.FlagOptions.NoOptDefVal(constants.ArgShareNoOptDefault)).
+		AddStringFlag(constants.ArgShare, "", "", "Create snapshot in Steampipe Cloud with 'anyone_with_link' visibility.", cmdconfig.FlagOptions.NoOptDefVal(constants.ArgShareNoOptDefault))
+
 	return cmd
 }
 
@@ -93,6 +93,9 @@ func runQueryCmd(cmd *cobra.Command, args []string) {
 		args = append(args, stdinData)
 	}
 
+	// validate args
+	utils.FailOnError(validateQueryArgs())
+
 	cloudMetadata, err := cmdconfig.GetCloudMetadata()
 	utils.FailOnError(err)
 
@@ -102,7 +105,7 @@ func runQueryCmd(cmd *cobra.Command, args []string) {
 	viper.Set(constants.ConfigKeyInteractive, interactiveMode)
 
 	// load the workspace
-	w, err := loadWorkspacePromptingForVariables(ctx)
+	w, err := interactive.LoadWorkspacePromptingForVariables(ctx)
 	utils.FailOnErrorWithMessage(err, "failed to load workspace")
 
 	// set cloud metadata (may be nil)
@@ -124,6 +127,14 @@ func runQueryCmd(cmd *cobra.Command, args []string) {
 	}
 }
 
+func validateQueryArgs() error {
+	// only 1 of 'share' and 'snapshot' may be set
+	if len(viper.GetString(constants.ArgShare)) > 0 && len(viper.GetString(constants.ArgShare)) > 0 {
+		return fmt.Errorf("only 1 of 'share' and 'dashboard' may be set")
+	}
+	return nil
+}
+
 // getPipedStdinData reads the Standard Input and returns the available data as a string
 // if and only if the data was piped to the process
 func getPipedStdinData() string {
@@ -141,30 +152,3 @@ func getPipedStdinData() string {
 	}
 	return stdinData
 }
-
-func loadWorkspacePromptingForVariables(ctx context.Context) (*workspace.Workspace, error) {
-	workspacePath := viper.GetString(constants.ArgWorkspaceChDir)
-
-	w, err := workspace.Load(ctx, workspacePath)
-	if err == nil {
-		return w, nil
-	}
-	missingVariablesError, ok := err.(modconfig.MissingVariableError)
-	// if there was an error which is NOT a MissingVariableError, return it
-	if !ok {
-		return nil, err
-	}
-	// if interactive inp[ut is disabled, return the missing variables error
-	if !viper.GetBool(constants.ArgInput) {
-		return nil, missingVariablesError
-	}
-	// so we have missing variables - prompt for them
-	// first hide spinner if it is there
-	statushooks.Done(ctx)
-	if err := interactive.PromptForMissingVariables(ctx, missingVariablesError.MissingVariables, workspacePath); err != nil {
-		log.Printf("[TRACE] Interactive variables prompting returned error %v", err)
-		return nil, err
-	}
-	// ok we should have all variables now - reload workspace
-	return workspace.Load(ctx, workspacePath)
-}

cmd/root.go

@@ -82,19 +82,14 @@ func InitCmd() {
 	defer utils.LogTime("cmd.root.InitCmd end")
 
 	rootCmd.PersistentFlags().String(constants.ArgInstallDir, filepaths.DefaultInstallDir, fmt.Sprintf("Path to the Config Directory (defaults to %s)", filepaths.DefaultInstallDir))
-	rootCmd.PersistentFlags().String(constants.ArgWorkspace, "", "Path to the workspace working directory")
 	rootCmd.PersistentFlags().String(constants.ArgWorkspaceChDir, "", "Path to the workspace working directory")
 	rootCmd.PersistentFlags().String(constants.ArgCloudHost, "cloud.steampipe.io", "Steampipe Cloud host")
 	rootCmd.PersistentFlags().String(constants.ArgCloudToken, "", "Steampipe Cloud authentication token")
 	rootCmd.PersistentFlags().String(constants.ArgWorkspaceDatabase, "local", "Steampipe Cloud workspace database")
 	rootCmd.PersistentFlags().Bool(constants.ArgSchemaComments, true, "Include schema comments when importing connection schemas")
 
-	rootCmd.Flag(constants.ArgWorkspace).Deprecated = "please use workspace-chdir"
-
 	err := viper.BindPFlag(constants.ArgInstallDir, rootCmd.PersistentFlags().Lookup(constants.ArgInstallDir))
 	utils.FailOnError(err)
-	err = viper.BindPFlag(constants.ArgWorkspace, rootCmd.PersistentFlags().Lookup(constants.ArgWorkspace))
-	utils.FailOnError(err)
 	err = viper.BindPFlag(constants.ArgWorkspaceChDir, rootCmd.PersistentFlags().Lookup(constants.ArgWorkspaceChDir))
 	utils.FailOnError(err)
 	err = viper.BindPFlag(constants.ArgCloudHost, rootCmd.PersistentFlags().Lookup(constants.ArgCloudHost))
@@ -188,11 +183,6 @@ func validateConfig() error {
 
 func setWorkspaceChDir() string {
 	workspaceChdir := viper.GetString(constants.ArgWorkspaceChDir)
-	workspace := viper.GetString(constants.ArgWorkspace)
-	if workspace != "" {
-		workspaceChdir = workspace
-
-	}
 	if workspaceChdir == "" {
 		cwd, err := os.Getwd()
 		utils.FailOnError(err)

pkg/cloud/snapshot.go

@@ -0,0 +1,92 @@
+package cloud
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/spf13/viper"
+	"github.com/turbot/steampipe/pkg/constants"
+	"github.com/turbot/steampipe/pkg/dashboard/dashboardtypes"
+	"io"
+	"net/http"
+	"strings"
+)
+
+func UploadSnapshot(snapshot *dashboardtypes.SteampipeSnapshot, share bool) (string, error) {
+
+	cloudWorkspace := viper.GetString(constants.ArgWorkspace)
+	parts := strings.Split(cloudWorkspace, "/")
+	if len(parts) != 2 {
+		return "", fmt.Errorf("failed to resolve username and workspace handle from workspace %s", cloudWorkspace)
+	}
+	user := parts[0]
+	worskpaceHandle := parts[1]
+
+	url := fmt.Sprintf("https://%s/api/v0/user/%s/workspace/%s/snapshot",
+		viper.GetString(constants.ArgCloudHost),
+		user,
+		worskpaceHandle)
+
+	// get the cloud token (we have already verifuied this was provided)
+	token := viper.GetString(constants.ArgCloudToken)
+	// create a 'bearer' string by appending the access token
+	var bearer = "Bearer " + token
+
+	client := &http.Client{}
+
+	// set the visibility
+	visibility := "workspace"
+	if share {
+		visibility = "anyone_with_link"
+	}
+
+	body := struct {
+		Data       *dashboardtypes.SteampipeSnapshot `json:"data"`
+		Tags       map[string]interface{}            `json:"tags"`
+		Visibility string                            `json:"visibility"`
+	}{
+		Data:       snapshot,
+		Tags:       map[string]interface{}{"generated_by": "cli"},
+		Visibility: visibility,
+	}
+
+	bodyStr, err := json.Marshal(body)
+	if err != nil {
+		return "", err
+	}
+
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(bodyStr))
+	if err != nil {
+		return "", err
+	}
+	req.Header.Add("Accept", "application/json")
+	req.Header.Add("Content-Type", "application/json")
+	req.Header.Add("Authorization", bearer)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	bodyBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	if resp.StatusCode < 200 || resp.StatusCode > 206 {
+		return "", fmt.Errorf("%s", resp.Status)
+	}
+
+	var result map[string]interface{}
+	err = json.Unmarshal(bodyBytes, &result)
+	if err != nil {
+		return "", err
+	}
+	snapshotId := result["id"].(string)
+	snapshotUrl := fmt.Sprintf("https://%s/user/%s/workspace/%s/snapshot/%s",
+		viper.GetString(constants.ArgCloudHost),
+		user,
+		worskpaceHandle,
+		snapshotId)
+
+	return snapshotUrl, nil
+}

pkg/cmdconfig/cmd_flags.go

@@ -14,13 +14,15 @@ type flagOpt func(c *cobra.Command, name string, key string)
 
 // FlagOptions :: shortcut for common flag options
 var FlagOptions = struct {
-	Required   func() flagOpt
-	Hidden     func() flagOpt
-	Deprecated func(string) flagOpt
+	Required    func() flagOpt
+	Hidden      func() flagOpt
+	Deprecated  func(string) flagOpt
+	NoOptDefVal func(string) flagOpt
 }{
-	Required:   requiredOpt,
-	Hidden:     hiddenOpt,
-	Deprecated: deprecatedOpt,
+	Required:    requiredOpt,
+	Hidden:      hiddenOpt,
+	Deprecated:  deprecatedOpt,
+	NoOptDefVal: noOptDefValOpt,
 }
 
 // Helper function to mark a flag as required
@@ -45,3 +47,9 @@ func deprecatedOpt(replacement string) flagOpt {
 		c.Flag(name).Deprecated = fmt.Sprintf("please use %s", replacement)
 	}
 }
+
+func noOptDefValOpt(noOptDefVal string) flagOpt {
+	return func(c *cobra.Command, name, key string) {
+		c.Flag(name).NoOptDefVal = noOptDefVal
+	}
+}

pkg/constants/args.go

@@ -47,10 +47,16 @@ const (
 	ArgServiceMode         = "service-mode"
 	ArgBrowser             = "browser"
 	ArgInput               = "input"
+	ArgDashboardInput      = "dashboard-input"
 	ArgMaxCacheSizeMb      = "max-cache-size-mb"
 	ArgIntrospection       = "introspection"
+	ArgShare               = "share"
+	ArgSnapshot            = "snapshot"
 )
 
+// the default value for ArgShare and ArgSnapshot if no value is provided
+const ArgShareNoOptDefault = "__workspace__"
+
 /// metaquery mode arguments
 
 var ArgOutput = ArgFromMetaquery(CmdOutput)

pkg/constants/extensions.go

@@ -12,6 +12,7 @@ const (
 	AutoVariablesExtension = ".auto.spvars"
 	JsonExtension          = ".json"
 	CsvExtension           = ".csv"
+	TextExtension          = ".txt"
 )
 
 var YamlExtensions = []string{".yml", ".yaml"}

pkg/constants/output_format.go

@@ -1,20 +1,12 @@
 package constants
 
 const (
-	// query output format
-	OutputFormatCSV   = "csv"
-	OutputFormatJSON  = "json"
-	OutputFormatTable = "table"
-	OutputFormatLine  = "line"
-
-	// check output format
-	CheckOutputFormatNone     = "none"
-	CheckOutputFormatText     = "text"
-	CheckOutputFormatBrief    = "brief"
-	CheckOutputFormatCSV      = "csv"
-	CheckOutputFormatJSON     = "json"
-	CheckOutputFormatHTML     = "html"
-	CheckOutputFormatMarkdown = "md"
-	CheckOutputFormatNUnit3   = "nunit3"
-	CheckOutputFormatAsffJson = "json-asff"
+	OutputFormatCSV      = "csv"
+	OutputFormatJSON     = "json"
+	OutputFormatTable    = "table"
+	OutputFormatLine     = "line"
+	OutputFormatNone     = "none"
+	OutputFormatText     = "text"
+	OutputFormatBrief    = "brief"
+	OutputFormatSnapshot = "snapshot"
 )

pkg/control/controldisplay/formatter.go

@@ -15,9 +15,9 @@ var ErrFormatterNotFound = errors.New("Formatter not found")
 type FormatterMap map[string]Formatter
 
 var outputFormatters FormatterMap = FormatterMap{
-	constants.CheckOutputFormatNone:  &NullFormatter{},
-	constants.CheckOutputFormatText:  &TextFormatter{},
-	constants.CheckOutputFormatBrief: &TextFormatter{},
+	constants.OutputFormatNone:  &NullFormatter{},
+	constants.OutputFormatText:  &TextFormatter{},
+	constants.OutputFormatBrief: &TextFormatter{},
 }
 
 type CheckExportTarget struct {
@@ -35,6 +35,7 @@ func NewCheckExportTarget(formatter Formatter, file string) CheckExportTarget {
 type Formatter interface {
 	Format(ctx context.Context, tree *controlexecute.ExecutionTree) (io.Reader, error)
 	FileExtension() string
+	// TODO THIS SEEMS TO BE ONLY USED FOR PRETTIFYING JSON???
 	GetFormatName() string
 }
 

pkg/control/controldisplay/formatter_snapshot.go

@@ -0,0 +1,27 @@
+package controldisplay
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/turbot/steampipe/pkg/constants"
+	"github.com/turbot/steampipe/pkg/control/controlexecute"
+)
+
+type SnapshotFormatter struct{}
+
+func (j *SnapshotFormatter) Format(ctx context.Context, tree *controlexecute.ExecutionTree) (io.Reader, error) {
+	var outputString = ""
+	res := strings.NewReader(fmt.Sprintf("\n%s\n", outputString))
+	return res, nil
+}
+
+func (j *SnapshotFormatter) FileExtension() string {
+	return constants.JsonExtension
+}
+
+func (tf SnapshotFormatter) GetFormatName() string {
+	return constants.OutputFormatSnapshot
+}

pkg/control/controldisplay/formatter_text.go

@@ -16,23 +16,23 @@ const MaxColumns = 200
 
 type TextFormatter struct{}
 
-func (j *TextFormatter) Format(ctx context.Context, tree *controlexecute.ExecutionTree) (io.Reader, error) {
+func (tf *TextFormatter) Format(ctx context.Context, tree *controlexecute.ExecutionTree) (io.Reader, error) {
 	renderer := NewTableRenderer(tree)
 	widthConstraint := NewRangeConstraint(renderer.MinimumWidth(), MaxColumns)
-	renderedText := renderer.Render(j.getMaxCols(widthConstraint))
+	renderedText := renderer.Render(tf.getMaxCols(widthConstraint))
 	res := strings.NewReader(fmt.Sprintf("\n%s\n", renderedText))
 	return res, nil
 }
 
-func (j *TextFormatter) FileExtension() string {
-	return ".txt"
+func (tf *TextFormatter) FileExtension() string {
+	return constants.TextExtension
 }
 
 func (tf TextFormatter) GetFormatName() string {
-	return "txt"
+	return constants.OutputFormatText
 }
 
-func (j *TextFormatter) getMaxCols(constraint RangeConstraint) int {
+func (tf *TextFormatter) getMaxCols(constraint RangeConstraint) int {
 	colsAvailable, _, _ := gows.GetWinSize()
 	// check if STEAMPIPE_CHECK_DISPLAY_WIDTH env variable is set
 	if viper.IsSet(constants.ArgCheckDisplayWidth) {

pkg/control/controlexecute/execution_tree.go

@@ -48,7 +48,6 @@ func NewExecutionTree(ctx context.Context, workspace *workspace.Workspace, clien
 	// create a context with status hooks disabled
 	noStatusCtx := statushooks.DisableStatusHooks(ctx)
 	err := executionTree.populateControlFilterMap(noStatusCtx)
-
 	if err != nil {
 		return nil, err
 	}

pkg/control/controlstatus/context.go

@@ -11,6 +11,12 @@ var (
 )
 
 func AddControlHooksToContext(ctx context.Context, statusHooks ControlHooks) context.Context {
+	// if the context already contains ControlHooks, do nothing
+	// this may happen when executing a dashboard snapshot -
+	if _, ok := ctx.Value(contextKeyControlHook).(ControlHooks); ok {
+		return ctx
+	}
+
 	return context.WithValue(ctx, contextKeyControlHook, statusHooks)
 }
 

pkg/control/controlstatus/control_hooks_snapshot.go

@@ -0,0 +1,36 @@
+package controlstatus
+
+import (
+	"context"
+	"github.com/spf13/viper"
+	"github.com/turbot/steampipe/pkg/constants"
+	"github.com/turbot/steampipe/pkg/statushooks"
+)
+
+// SnapshotControlHooks is a struct which implements ControlHooks, and displays the control progress as a status message
+type SnapshotControlHooks struct {
+	Enabled bool
+}
+
+func NewSnapshotControlHooks() *SnapshotControlHooks {
+	return &SnapshotControlHooks{
+		Enabled: viper.GetBool(constants.ArgProgress),
+	}
+}
+
+func (c *SnapshotControlHooks) OnStart(context.Context, *ControlProgress) {
+}
+
+func (c *SnapshotControlHooks) OnControlStart(context.Context, ControlRunStatusProvider, *ControlProgress) {
+}
+
+func (c *SnapshotControlHooks) OnControlComplete(ctx context.Context, _ ControlRunStatusProvider, progress *ControlProgress) {
+	statushooks.UpdateSnapshotProgress(ctx, progress.StatusSummaries.TotalCount())
+}
+
+func (c *SnapshotControlHooks) OnControlError(ctx context.Context, _ ControlRunStatusProvider, _ *ControlProgress) {
+	statushooks.SnapshotError(ctx)
+}
+
+func (c *SnapshotControlHooks) OnComplete(_ context.Context, _ *ControlProgress) {
+}

pkg/control/controlstatus/control_hooks_status.go

@@ -10,18 +10,18 @@ import (
 	"github.com/turbot/steampipe/pkg/utils"
 )
 
-// ControlStatusHooks is a struct which implements ControlHooks, and displays the control progress as a status message
-type ControlStatusHooks struct {
+// StatusControlHooks is a struct which implements ControlHooks, and displays the control progress as a status message
+type StatusControlHooks struct {
 	Enabled bool
 }
 
-func NewControlStatusHooks() *ControlStatusHooks {
-	return &ControlStatusHooks{
+func NewStatusControlHooks() *StatusControlHooks {
+	return &StatusControlHooks{
 		Enabled: viper.GetBool(constants.ArgProgress),
 	}
 }
 
-func (c *ControlStatusHooks) OnStart(ctx context.Context, _ *ControlProgress) {
+func (c *StatusControlHooks) OnStart(ctx context.Context, _ *ControlProgress) {
 	if !c.Enabled {
 		return
 	}
@@ -29,7 +29,7 @@ func (c *ControlStatusHooks) OnStart(ctx context.Context, _ *ControlProgress) {
 	statushooks.SetStatus(ctx, "Starting controls...")
 }
 
-func (c *ControlStatusHooks) OnControlStart(ctx context.Context, _ ControlRunStatusProvider, p *ControlProgress) {
+func (c *StatusControlHooks) OnControlStart(ctx context.Context, _ ControlRunStatusProvider, p *ControlProgress) {
 	if !c.Enabled {
 		return
 	}
@@ -37,7 +37,7 @@ func (c *ControlStatusHooks) OnControlStart(ctx context.Context, _ ControlRunSta
 	c.setStatusFromProgress(ctx, p)
 }
 
-func (c *ControlStatusHooks) OnControlComplete(ctx context.Context, _ ControlRunStatusProvider, p *ControlProgress) {
+func (c *StatusControlHooks) OnControlComplete(ctx context.Context, _ ControlRunStatusProvider, p *ControlProgress) {
 	if !c.Enabled {
 		return
 	}
@@ -45,7 +45,7 @@ func (c *ControlStatusHooks) OnControlComplete(ctx context.Context, _ ControlRun
 	c.setStatusFromProgress(ctx, p)
 }
 
-func (c *ControlStatusHooks) OnControlError(ctx context.Context, _ ControlRunStatusProvider, p *ControlProgress) {
+func (c *StatusControlHooks) OnControlError(ctx context.Context, _ ControlRunStatusProvider, p *ControlProgress) {
 	if !c.Enabled {
 		return
 	}
@@ -53,7 +53,7 @@ func (c *ControlStatusHooks) OnControlError(ctx context.Context, _ ControlRunSta
 	c.setStatusFromProgress(ctx, p)
 }
 
-func (c *ControlStatusHooks) OnComplete(ctx context.Context, _ *ControlProgress) {
+func (c *StatusControlHooks) OnComplete(ctx context.Context, _ *ControlProgress) {
 	if !c.Enabled {
 		return
 	}
@@ -61,7 +61,7 @@ func (c *ControlStatusHooks) OnComplete(ctx context.Context, _ *ControlProgress)
 	statushooks.Done(ctx)
 }
 
-func (c *ControlStatusHooks) setStatusFromProgress(ctx context.Context, p *ControlProgress) {
+func (c *StatusControlHooks) setStatusFromProgress(ctx context.Context, p *ControlProgress) {
 	message := fmt.Sprintf("Running %d %s. (%d complete, %d running, %d pending, %d %s)",
 		p.Total,
 		utils.Pluralize("control", p.Total),

pkg/control/init_data.go

@@ -1,149 +0,0 @@
-package control
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/spf13/viper"
-	"github.com/turbot/steampipe-plugin-sdk/v4/telemetry"
-	"github.com/turbot/steampipe/pkg/cmdconfig"
-	"github.com/turbot/steampipe/pkg/constants"
-	"github.com/turbot/steampipe/pkg/control/controldisplay"
-	"github.com/turbot/steampipe/pkg/db/db_client"
-	"github.com/turbot/steampipe/pkg/db/db_common"
-	"github.com/turbot/steampipe/pkg/db/db_local"
-	"github.com/turbot/steampipe/pkg/modinstaller"
-	"github.com/turbot/steampipe/pkg/statushooks"
-	"github.com/turbot/steampipe/pkg/workspace"
-)
-
-type InitData struct {
-	Workspace         *workspace.Workspace
-	Client            db_common.Client
-	Result            *db_common.InitResult
-	ShutdownTelemetry func()
-}
-
-func NewInitData(ctx context.Context, w *workspace.Workspace) *InitData {
-	initData := &InitData{
-		Workspace: w,
-		Result:    &db_common.InitResult{},
-	}
-
-	if err := controldisplay.EnsureTemplates(); err != nil {
-		initData.Result.Error = err
-		return initData
-	}
-
-	// initialise telemetry
-	shutdownTelemetry, err := telemetry.Init(constants.AppName)
-	if err != nil {
-		initData.Result.AddWarnings(err.Error())
-	} else {
-		initData.ShutdownTelemetry = shutdownTelemetry
-	}
-
-	if viper.GetBool(constants.ArgModInstall) {
-		opts := &modinstaller.InstallOpts{WorkspacePath: viper.GetString(constants.ArgWorkspaceChDir)}
-		_, err := modinstaller.InstallWorkspaceDependencies(opts)
-		if err != nil {
-			initData.Result.Error = err
-			return initData
-		}
-	}
-
-	if viper.GetString(constants.ArgOutput) == constants.CheckOutputFormatNone {
-		// set progress to false
-		viper.Set(constants.ArgProgress, false)
-	}
-
-	cloudMetadata, err := cmdconfig.GetCloudMetadata()
-	if err != nil {
-		initData.Result.Error = err
-		return initData
-	}
-
-	// set cloud metadata (may be nil)
-	initData.Workspace.CloudMetadata = cloudMetadata
-	// set color schema
-	err = initialiseColorScheme()
-	if err != nil {
-		initData.Result.Error = err
-		return initData
-	}
-
-	// check if the required plugins are installed
-	err = initData.Workspace.CheckRequiredPluginsInstalled()
-	if err != nil {
-		initData.Result.Error = err
-		return initData
-	}
-
-	if len(initData.Workspace.GetResourceMaps().Controls) == 0 {
-		initData.Result.AddWarnings("no controls found in current workspace")
-	}
-
-	statushooks.SetStatus(ctx, "Connecting to service...")
-	// get a client
-	var client db_common.Client
-	if connectionString := viper.GetString(constants.ArgConnectionString); connectionString != "" {
-		client, err = db_client.NewDbClient(ctx, connectionString)
-	} else {
-		// when starting the database, installers may trigger their own spinners
-		client, err = db_local.GetLocalClient(ctx, constants.InvokerCheck)
-	}
-
-	if err != nil {
-		initData.Result.Error = err
-		return initData
-	}
-	initData.Client = client
-
-	refreshResult := initData.Client.RefreshConnectionAndSearchPaths(ctx)
-	if refreshResult.Error != nil {
-		initData.Result.Error = refreshResult.Error
-		return initData
-	}
-	initData.Result.AddWarnings(refreshResult.Warnings...)
-
-	// setup the session data - prepared statements and introspection tables
-	sessionDataSource := workspace.NewSessionDataSource(initData.Workspace, nil)
-
-	// register EnsureSessionData as a callback on the client.
-	// if the underlying SQL client has certain errors (for example context expiry) it will reset the session
-	// so our client object calls this callback to restore the session data
-	initData.Client.SetEnsureSessionDataFunc(func(localCtx context.Context, conn *db_common.DatabaseSession) (error, []string) {
-		return workspace.EnsureSessionData(localCtx, sessionDataSource, conn)
-	})
-
-	return initData
-
-}
-
-func (i InitData) Cleanup(ctx context.Context) {
-	if i.Client != nil {
-		i.Client.Close(ctx)
-	}
-
-	if i.ShutdownTelemetry != nil {
-		i.ShutdownTelemetry()
-	}
-}
-
-func initialiseColorScheme() error {
-	theme := viper.GetString(constants.ArgTheme)
-	if !viper.GetBool(constants.ConfigKeyIsTerminalTTY) {
-		// enforce plain output for non-terminals
-		theme = "plain"
-	}
-	themeDef, ok := controldisplay.ColorSchemes[theme]
-	if !ok {
-		return fmt.Errorf("invalid theme '%s'", theme)
-	}
-	scheme, err := controldisplay.NewControlColorScheme(themeDef)
-	if err != nil {
-		return err
-	}
-	controldisplay.ControlColors = scheme
-	return nil
-}

pkg/dashboard/dashboardexecute/check_run.go

@@ -98,7 +98,7 @@ func (r *CheckRun) Initialise(ctx context.Context) {
 	executionTree, err := controlexecute.NewExecutionTree(ctx, r.executionTree.workspace, r.executionTree.client, r.DashboardNode.Name())
 	if err != nil {
 		// set the error status on the counter - this will raise counter error event
-		r.SetError(err)
+		r.SetError(ctx, err)
 		return
 	}
 	r.controlExecutionTree = executionTree
@@ -110,15 +110,15 @@ func (r *CheckRun) Execute(ctx context.Context) {
 	utils.LogTime("CheckRun.execute start")
 	defer utils.LogTime("CheckRun.execute end")
 
-	// create a context with a ControlEventHooks to report control execution progress
-	ctx = controlstatus.AddControlHooksToContext(ctx, NewControlEventHooks(r))
+	// create a context with a DashboardEventControlHooks to report control execution progress
+	ctx = controlstatus.AddControlHooksToContext(ctx, NewDashboardEventControlHooks(r))
 	r.controlExecutionTree.Execute(ctx)
 
 	// set the summary on the CheckRun
 	r.Summary = r.controlExecutionTree.Root.Summary
 
 	// set complete status on counter - this will raise counter complete event
-	r.SetComplete()
+	r.SetComplete(ctx)
 }
 
 // GetName implements DashboardNodeRun
@@ -132,7 +132,7 @@ func (r *CheckRun) GetRunStatus() dashboardtypes.DashboardRunStatus {
 }
 
 // SetError implements DashboardNodeRun
-func (r *CheckRun) SetError(err error) {
+func (r *CheckRun) SetError(ctx context.Context, err error) {
 	r.error = err
 	// error type does not serialise to JSON so copy into a string
 	r.ErrorString = err.Error()
@@ -155,7 +155,7 @@ func (r *CheckRun) GetError() error {
 }
 
 // SetComplete implements DashboardNodeRun
-func (r *CheckRun) SetComplete() {
+func (r *CheckRun) SetComplete(ctx context.Context) {
 	r.runStatus = dashboardtypes.DashboardRunComplete
 	// raise counter complete event
 	r.executionTree.workspace.PublishDashboardEvent(&dashboardevents.LeafNodeComplete{

pkg/dashboard/dashboardexecute/container_run.go

@@ -132,7 +132,7 @@ func (r *DashboardContainerRun) Initialise(ctx context.Context) {
 	for _, child := range r.Children {
 		child.Initialise(ctx)
 		if err := child.GetError(); err != nil {
-			r.SetError(err)
+			r.SetError(ctx, err)
 			return
 		}
 	}
@@ -161,9 +161,9 @@ func (r *DashboardContainerRun) Execute(ctx context.Context) {
 	err := utils.CombineErrors(errors...)
 	if err == nil {
 		// set complete status on dashboard
-		r.SetComplete()
+		r.SetComplete(ctx)
 	} else {
-		r.SetError(err)
+		r.SetError(ctx, err)
 	}
 }
 
@@ -179,7 +179,7 @@ func (r *DashboardContainerRun) GetRunStatus() dashboardtypes.DashboardRunStatus
 
 // SetError implements DashboardNodeRun
 // tell parent we are done
-func (r *DashboardContainerRun) SetError(err error) {
+func (r *DashboardContainerRun) SetError(_ context.Context, err error) {
 	r.error = err
 	// error type does not serialise to JSON so copy into a string
 	r.ErrorString = err.Error()
@@ -199,7 +199,7 @@ func (r *DashboardContainerRun) GetError() error {
 }
 
 // SetComplete implements DashboardNodeRun
-func (r *DashboardContainerRun) SetComplete() {
+func (r *DashboardContainerRun) SetComplete(context.Context) {
 	r.Status = dashboardtypes.DashboardRunComplete
 	// raise container complete event
 	r.executionTree.workspace.PublishDashboardEvent(&dashboardevents.ContainerComplete{
@@ -237,5 +237,5 @@ func (r *DashboardContainerRun) ChildCompleteChan() chan dashboardtypes.Dashboar
 }
 
 // GetInputsDependingOn implements DashboardNodeRun
-//return nothing for DashboardContainerRun
+// return nothing for DashboardContainerRun
 func (r *DashboardContainerRun) GetInputsDependingOn(changedInputName string) []string { return nil }

pkg/dashboard/dashboardexecute/control_event_hooks.go

@@ -1,52 +0,0 @@
-package dashboardexecute
-
-import (
-	"context"
-
-	"github.com/turbot/steampipe/pkg/control/controlstatus"
-	"github.com/turbot/steampipe/pkg/dashboard/dashboardevents"
-)
-
-// ControlEventHooks is a struct which implements ControlHooks, and displays the control progress as a status message
-type ControlEventHooks struct {
-	CheckRun *CheckRun
-}
-
-func NewControlEventHooks(r *CheckRun) *ControlEventHooks {
-	return &ControlEventHooks{
-		CheckRun: r,
-	}
-}
-
-func (c *ControlEventHooks) OnStart(ctx context.Context, _ *controlstatus.ControlProgress) {
-	// nothing to do
-}
-
-func (c *ControlEventHooks) OnControlStart(context.Context, controlstatus.ControlRunStatusProvider, *controlstatus.ControlProgress) {
-}
-
-func (c *ControlEventHooks) OnControlComplete(ctx context.Context, controlRun controlstatus.ControlRunStatusProvider, progress *controlstatus.ControlProgress) {
-	event := &dashboardevents.ControlComplete{
-		Control:     controlRun,
-		Progress:    progress,
-		Name:        c.CheckRun.Name,
-		ExecutionId: c.CheckRun.executionTree.id,
-		Session:     c.CheckRun.SessionId,
-	}
-	c.CheckRun.executionTree.workspace.PublishDashboardEvent(event)
-}
-
-func (c *ControlEventHooks) OnControlError(ctx context.Context, controlRun controlstatus.ControlRunStatusProvider, progress *controlstatus.ControlProgress) {
-	var event = &dashboardevents.ControlError{
-		Control:     controlRun,
-		Progress:    progress,
-		Name:        c.CheckRun.Name,
-		ExecutionId: c.CheckRun.executionTree.id,
-		Session:     c.CheckRun.SessionId,
-	}
-	c.CheckRun.executionTree.workspace.PublishDashboardEvent(event)
-}
-
-func (c *ControlEventHooks) OnComplete(ctx context.Context, _ *controlstatus.ControlProgress) {
-	// nothing to do - LeafNodeDone will be sent anyway
-}

pkg/dashboard/dashboardexecute/control_hooks_dashboard_event.go

@@ -0,0 +1,53 @@
+package dashboardexecute
+
+import (
+	"context"
+	"github.com/turbot/steampipe/pkg/control/controlstatus"
+
+	"github.com/turbot/steampipe/pkg/dashboard/dashboardevents"
+)
+
+// DashboardEventControlHooks is a struct which implements ControlHooks,
+// and raises ControlComplete and ControlError dashboard events
+type DashboardEventControlHooks struct {
+	CheckRun *CheckRun
+}
+
+func NewDashboardEventControlHooks(r *CheckRun) *DashboardEventControlHooks {
+	return &DashboardEventControlHooks{
+		CheckRun: r,
+	}
+}
+
+func (c *DashboardEventControlHooks) OnStart(ctx context.Context, _ *controlstatus.ControlProgress) {
+	// nothing to do
+}
+
+func (c *DashboardEventControlHooks) OnControlStart(context.Context, controlstatus.ControlRunStatusProvider, *controlstatus.ControlProgress) {
+}
+
+func (c *DashboardEventControlHooks) OnControlComplete(ctx context.Context, controlRun controlstatus.ControlRunStatusProvider, progress *controlstatus.ControlProgress) {
+	event := &dashboardevents.ControlComplete{
+		Control:     controlRun,
+		Progress:    progress,
+		Name:        c.CheckRun.Name,
+		ExecutionId: c.CheckRun.executionTree.id,
+		Session:     c.CheckRun.SessionId,
+	}
+	c.CheckRun.executionTree.workspace.PublishDashboardEvent(event)
+}
+
+func (c *DashboardEventControlHooks) OnControlError(ctx context.Context, controlRun controlstatus.ControlRunStatusProvider, progress *controlstatus.ControlProgress) {
+	var event = &dashboardevents.ControlError{
+		Control:     controlRun,
+		Progress:    progress,
+		Name:        c.CheckRun.Name,
+		ExecutionId: c.CheckRun.executionTree.id,
+		Session:     c.CheckRun.SessionId,
+	}
+	c.CheckRun.executionTree.workspace.PublishDashboardEvent(event)
+}
+
+func (c *DashboardEventControlHooks) OnComplete(ctx context.Context, _ *controlstatus.ControlProgress) {
+	// nothing to do - LeafNodeDone will be sent anyway
+}

pkg/dashboard/dashboardexecute/dashboard_execution_tree.go

@@ -64,6 +64,11 @@ func (e *DashboardExecutionTree) createRootItem(rootName string) (dashboardtypes
 	if err != nil {
 		return nil, err
 	}
+	// if no mod is specified, assume the workspace mod
+	if parsedName.Mod == "" {
+		parsedName.Mod = e.workspace.Mod.ShortName
+		rootName = parsedName.ToFullName()
+	}
 	switch parsedName.ItemType {
 	case modconfig.BlockTypeDashboard:
 		dashboard, ok := e.workspace.GetResourceMaps().Dashboards[rootName]
@@ -143,8 +148,8 @@ func (e *DashboardExecutionTree) GetRunStatus() dashboardtypes.DashboardRunStatu
 }
 
 // SetError sets the error on the Root run
-func (e *DashboardExecutionTree) SetError(err error) {
-	e.Root.SetError(err)
+func (e *DashboardExecutionTree) SetError(ctx context.Context, err error) {
+	e.Root.SetError(ctx, err)
 }
 
 // GetName implements DashboardNodeParent

pkg/dashboard/dashboardexecute/dashboard_run.go

@@ -147,7 +147,7 @@ func (r *DashboardRun) Initialise(ctx context.Context) {
 	for _, child := range r.Children {
 		child.Initialise(ctx)
 		if err := child.GetError(); err != nil {
-			r.SetError(err)
+			r.SetError(ctx, err)
 			return
 		}
 	}
@@ -176,9 +176,9 @@ func (r *DashboardRun) Execute(ctx context.Context) {
 	err := utils.CombineErrors(errors...)
 	if err == nil {
 		// set complete status on dashboard
-		r.SetComplete()
+		r.SetComplete(ctx)
 	} else {
-		r.SetError(err)
+		r.SetError(ctx, err)
 	}
 }
 
@@ -197,7 +197,7 @@ func (r *DashboardRun) GetRunStatus() dashboardtypes.DashboardRunStatus {
 
 // SetError implements DashboardNodeRun
 // tell parent we are done
-func (r *DashboardRun) SetError(err error) {
+func (r *DashboardRun) SetError(_ context.Context, err error) {
 	r.error = err
 	// error type does not serialise to JSON so copy into a string
 	r.ErrorString = err.Error()
@@ -217,7 +217,7 @@ func (r *DashboardRun) GetError() error {
 }
 
 // SetComplete implements DashboardNodeRun
-func (r *DashboardRun) SetComplete() {
+func (r *DashboardRun) SetComplete(context.Context) {
 	r.Status = dashboardtypes.DashboardRunComplete
 	// raise container complete event
 	r.executionTree.workspace.PublishDashboardEvent(&dashboardevents.ContainerComplete{

pkg/dashboard/dashboardexecute/executor.go

@@ -67,7 +67,7 @@ func (e *DashboardExecutor) OnInputChanged(ctx context.Context, sessionId string
 		return fmt.Errorf("no dashboard running for session %s", sessionId)
 	}
 
-	// get the previous value oif this input
+	// get the previous value of this input
 	inputPrevValue := executionTree.inputValues[changedInput]
 	// first see if any other inputs rely on the one which was just changed
 	clearedInputs := e.clearDependentInputs(executionTree.Root, changedInput, inputs)
@@ -79,7 +79,7 @@ func (e *DashboardExecutor) OnInputChanged(ctx context.Context, sessionId string
 		}
 		executionTree.workspace.PublishDashboardEvent(event)
 	}
-	// oif there are any dependent inputs, set their value to nil and send an event to the UI
+	// if there are any dependent inputs, set their value to nil and send an event to the UI
 	// if the dashboard run is complete, just re-execute
 	if executionTree.GetRunStatus() == dashboardtypes.DashboardRunComplete || inputPrevValue != nil {
 		return e.ExecuteDashboard(

pkg/dashboard/dashboardexecute/leaf_run.go

@@ -3,6 +3,7 @@ package dashboardexecute
 import (
 	"context"
 	"fmt"
+	"github.com/turbot/steampipe/pkg/statushooks"
 	"log"
 
 	"github.com/turbot/steampipe/pkg/dashboard/dashboardevents"
@@ -118,13 +119,13 @@ func (r *LeafRun) Execute(ctx context.Context) {
 	// if there are any unresolved runtime dependencies, wait for them
 	if len(r.runtimeDependencies) > 0 {
 		if err := r.waitForRuntimeDependencies(ctx); err != nil {
-			r.SetError(err)
+			r.SetError(ctx, err)
 			return
 		}
 
 		// ok now we have runtime dependencies, we can resolve the query
 		if err := r.resolveSQL(); err != nil {
-			r.SetError(err)
+			r.SetError(ctx, err)
 			return
 		}
 	}
@@ -135,7 +136,7 @@ func (r *LeafRun) Execute(ctx context.Context) {
 	if err != nil {
 		log.Printf("[TRACE] LeafRun '%s' query failed: %s", r.DashboardNode.Name(), err.Error())
 		// set the error status on the counter - this will raise counter error event
-		r.SetError(err)
+		r.SetError(ctx, err)
 		return
 
 	}
@@ -143,7 +144,7 @@ func (r *LeafRun) Execute(ctx context.Context) {
 
 	r.Data = dashboardtypes.NewLeafData(queryResult)
 	// set complete status on counter - this will raise counter complete event
-	r.SetComplete()
+	r.SetComplete(ctx)
 }
 
 // GetName implements DashboardNodeRun
@@ -157,12 +158,13 @@ func (r *LeafRun) GetRunStatus() dashboardtypes.DashboardRunStatus {
 }
 
 // SetError implements DashboardNodeRun
-func (r *LeafRun) SetError(err error) {
+func (r *LeafRun) SetError(ctx context.Context, err error) {
 	r.error = err
 	// error type does not serialise to JSON so copy into a string
 	r.ErrorString = err.Error()
-
 	r.Status = dashboardtypes.DashboardRunError
+	// increment error count for snapshot hook
+	statushooks.SnapshotError(ctx)
 	// raise counter error event
 	r.executionTree.workspace.PublishDashboardEvent(&dashboardevents.LeafNodeError{
 		LeafNode:    r,
@@ -178,7 +180,7 @@ func (r *LeafRun) GetError() error {
 }
 
 // SetComplete implements DashboardNodeRun
-func (r *LeafRun) SetComplete() {
+func (r *LeafRun) SetComplete(ctx context.Context) {
 	r.Status = dashboardtypes.DashboardRunComplete
 	// raise counter complete event
 	r.executionTree.workspace.PublishDashboardEvent(&dashboardevents.LeafNodeComplete{
@@ -186,6 +188,10 @@ func (r *LeafRun) SetComplete() {
 		Session:     r.executionTree.sessionId,
 		ExecutionId: r.executionTree.id,
 	})
+
+	// call snapshot hooks with progress
+	statushooks.UpdateSnapshotProgress(ctx, 1)
+
 	// tell parent we are done
 	r.parent.ChildCompleteChan() <- r
 }
@@ -209,7 +215,7 @@ func (r *LeafRun) ChildrenComplete() bool {
 func (*LeafRun) IsSnapshotPanel() {}
 
 // GetInputsDependingOn implements DashboardNodeRun
-//return nothing for LeafRun
+// return nothing for LeafRun
 func (r *LeafRun) GetInputsDependingOn(changedInputName string) []string { return nil }
 
 func (r *LeafRun) waitForRuntimeDependencies(ctx context.Context) error {

pkg/dashboard/dashboardserver/payload.go

@@ -199,20 +199,7 @@ func buildExecutionErrorPayload(event *dashboardevents.ExecutionError) ([]byte,
 }
 
 func buildExecutionCompletePayload(event *dashboardevents.ExecutionComplete) ([]byte, error) {
-	payload := ExecutionCompletePayload{
-		SchemaVersion: fmt.Sprintf("%d", ExecutionCompleteSchemaVersion),
-		Action:        "execution_complete",
-		DashboardNode: event.Root,
-		ExecutionId:   event.ExecutionId,
-		Panels:        event.Panels,
-		Layout:        event.Root.AsTreeNode(),
-		Inputs:        event.Inputs,
-		Variables:     event.Variables,
-		SearchPath:    event.SearchPath,
-		StartTime:     event.StartTime,
-		EndTime:       event.EndTime,
-	}
-
+	payload := ExecutionCompleteToSnapshot(event)
 	return json.Marshal(payload)
 }
 

pkg/dashboard/dashboardserver/server.go

@@ -50,7 +50,7 @@ func NewServer(ctx context.Context, dbClient db_common.Client, w *workspace.Work
 		workspace:        w,
 	}
 
-	w.RegisterDashboardEventHandler(server.HandleWorkspaceUpdate)
+	w.RegisterDashboardEventHandler(server.HandleDashboardEvent)
 	err := w.SetupWatcher(ctx, dbClient, func(c context.Context, e error) {})
 	OutputMessage(ctx, "Workspace loaded")
 
@@ -76,16 +76,11 @@ func (s *Server) Shutdown() {
 		log.Println("[TRACE] closed websocket")
 	}
 
-	// Close the workspace
-	if s.workspace != nil {
-		s.workspace.Close()
-	}
-
 	log.Println("[TRACE] Server shutdown complete")
 
 }
 
-func (s *Server) HandleWorkspaceUpdate(event dashboardevents.DashboardEvent) {
+func (s *Server) HandleDashboardEvent(event dashboardevents.DashboardEvent) {
 	var payloadError error
 	var payload []byte
 	defer func() {

pkg/dashboard/dashboardserver/snapshot.go

@@ -0,0 +1,24 @@
+package dashboardserver
+
+import (
+	"fmt"
+	"github.com/turbot/steampipe/pkg/dashboard/dashboardevents"
+	"github.com/turbot/steampipe/pkg/dashboard/dashboardtypes"
+)
+
+// ExecutionCompleteToSnapshot transforms the ExecutionComplete event into a SteampipeSnapshot
+func ExecutionCompleteToSnapshot(event *dashboardevents.ExecutionComplete) *dashboardtypes.SteampipeSnapshot {
+	return &dashboardtypes.SteampipeSnapshot{
+		SchemaVersion: fmt.Sprintf("%d", dashboardtypes.SteampipeSnapshotSchemaVersion),
+		Action:        "execution_complete",
+		DashboardNode: event.Root,
+		ExecutionId:   event.ExecutionId,
+		Panels:        event.Panels,
+		Layout:        event.Root.AsTreeNode(),
+		Inputs:        event.Inputs,
+		Variables:     event.Variables,
+		SearchPath:    event.SearchPath,
+		StartTime:     event.StartTime,
+		EndTime:       event.EndTime,
+	}
+}

pkg/dashboard/dashboardserver/types.go

@@ -2,8 +2,6 @@ package dashboardserver
 
 import (
 	"fmt"
-	"time"
-
 	"github.com/turbot/steampipe/pkg/control/controlstatus"
 	"github.com/turbot/steampipe/pkg/dashboard/dashboardtypes"
 	"github.com/turbot/steampipe/pkg/steampipeconfig"
@@ -71,22 +69,6 @@ type ExecutionErrorPayload struct {
 	Error  string `json:"error"`
 }
 
-var ExecutionCompleteSchemaVersion int64 = 20220614
-
-type ExecutionCompletePayload struct {
-	SchemaVersion string                                  `json:"schema_version"`
-	Action        string                                  `json:"action"`
-	DashboardNode dashboardtypes.DashboardNodeRun         `json:"dashboard_node"`
-	Panels        map[string]dashboardtypes.SnapshotPanel `json:"panels"`
-	ExecutionId   string                                  `json:"execution_id"`
-	Inputs        map[string]interface{}                  `json:"inputs"`
-	Variables     map[string]string                       `json:"variables"`
-	SearchPath    []string                                `json:"search_path"`
-	StartTime     time.Time                               `json:"start_time"`
-	EndTime       time.Time                               `json:"end_time"`
-	Layout        *dashboardtypes.SnapshotTreeNode        `json:"layout"`
-}
-
 type InputValuesClearedPayload struct {
 	Action        string   `json:"action"`
 	ClearedInputs []string `json:"cleared_inputs"`

pkg/dashboard/dashboardtypes/dashboard_node_run.go

@@ -18,13 +18,13 @@ type DashboardNodeRun interface {
 	Execute(ctx context.Context)
 	GetName() string
 	GetRunStatus() DashboardRunStatus
-	SetError(err error)
+	SetError(context.Context, error)
 	GetError() error
-	SetComplete()
+	SetComplete(context.Context)
 	RunComplete() bool
 	GetChildren() []DashboardNodeRun
 	ChildrenComplete() bool
-	GetInputsDependingOn(changedInputName string) []string
+	GetInputsDependingOn(string) []string
 	AsTreeNode() *SnapshotTreeNode
 }
 

pkg/dashboard/dashboardtypes/snapshot.go

@@ -0,0 +1,21 @@
+package dashboardtypes
+
+import (
+	"time"
+)
+
+var SteampipeSnapshotSchemaVersion int64 = 20220614
+
+type SteampipeSnapshot struct {
+	SchemaVersion string                   `json:"schema_version"`
+	Action        string                   `json:"action"`
+	DashboardNode DashboardNodeRun         `json:"dashboard_node"`
+	Panels        map[string]SnapshotPanel `json:"panels"`
+	ExecutionId   string                   `json:"execution_id"`
+	Inputs        map[string]interface{}   `json:"inputs"`
+	Variables     map[string]string        `json:"variables"`
+	SearchPath    []string                 `json:"search_path"`
+	StartTime     time.Time                `json:"start_time"`
+	EndTime       time.Time                `json:"end_time"`
+	Layout        *SnapshotTreeNode        `json:"layout"`
+}

pkg/initialisation/init_data.go

@@ -1,4 +1,4 @@
-package dashboard
+package initialisation
 
 import (
 	"context"
@@ -28,6 +28,12 @@ func NewInitData(ctx context.Context, w *workspace.Workspace) *InitData {
 		Result:    &db_common.InitResult{},
 	}
 
+	defer func() {
+		// if there is no error, return context cancellation error (if any)
+		if initData.Result.Error == nil {
+			initData.Result.Error = ctx.Err()
+		}
+	}()
 	// initialise telemetry
 	shutdownTelemetry, err := telemetry.Init(constants.AppName)
 	if err != nil {
@@ -36,11 +42,7 @@ func NewInitData(ctx context.Context, w *workspace.Workspace) *InitData {
 		initData.ShutdownTelemetry = shutdownTelemetry
 	}
 
-	if !w.ModfileExists() {
-		initData.Result.Error = workspace.ErrorNoModDefinition
-		return initData
-	}
-
+	// install mod dependencies if needed
 	if viper.GetBool(constants.ArgModInstall) {
 		opts := &modinstaller.InstallOpts{WorkspacePath: viper.GetString(constants.ArgWorkspaceChDir)}
 		_, err := modinstaller.InstallWorkspaceDependencies(opts)
@@ -49,6 +51,8 @@ func NewInitData(ctx context.Context, w *workspace.Workspace) *InitData {
 			return initData
 		}
 	}
+
+	// retrieve cloud metadata
 	cloudMetadata, err := cmdconfig.GetCloudMetadata()
 	if err != nil {
 		initData.Result.Error = err
@@ -65,8 +69,8 @@ func NewInitData(ctx context.Context, w *workspace.Workspace) *InitData {
 		return initData
 	}
 
-	statushooks.SetStatus(ctx, "Connecting to service...")
 	// get a client
+	statushooks.SetStatus(ctx, "Connecting to service...")
 	var client db_common.Client
 	if connectionString := viper.GetString(constants.ArgConnectionString); connectionString != "" {
 		client, err = db_client.NewDbClient(ctx, connectionString)
@@ -74,13 +78,14 @@ func NewInitData(ctx context.Context, w *workspace.Workspace) *InitData {
 		// when starting the database, installers may trigger their own spinners
 		client, err = db_local.GetLocalClient(ctx, constants.InvokerDashboard)
 	}
-
 	if err != nil {
 		initData.Result.Error = err
 		return initData
 	}
 	initData.Client = client
+	statushooks.Done(ctx)
 
+	// refresh connections
 	refreshResult := initData.Client.RefreshConnectionAndSearchPaths(ctx)
 	if refreshResult.Error != nil {
 		initData.Result.Error = refreshResult.Error
@@ -99,11 +104,9 @@ func NewInitData(ctx context.Context, w *workspace.Workspace) *InitData {
 	})
 
 	return initData
-
 }
 
 func (i InitData) Cleanup(ctx context.Context) {
-	// if a client was initialised, close it
 	if i.Client != nil {
 		i.Client.Close(ctx)
 	}
@@ -111,4 +114,7 @@ func (i InitData) Cleanup(ctx context.Context) {
 	if i.ShutdownTelemetry != nil {
 		i.ShutdownTelemetry()
 	}
+	if i.Workspace != nil {
+		i.Workspace.Close()
+	}
 }

pkg/interactive/load_workspace.go

@@ -0,0 +1,39 @@
+package interactive
+
+import (
+	"context"
+	"log"
+
+	"github.com/spf13/viper"
+	"github.com/turbot/steampipe/pkg/constants"
+	"github.com/turbot/steampipe/pkg/statushooks"
+	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
+	"github.com/turbot/steampipe/pkg/workspace"
+)
+
+func LoadWorkspacePromptingForVariables(ctx context.Context) (*workspace.Workspace, error) {
+	workspacePath := viper.GetString(constants.ArgWorkspaceChDir)
+
+	w, err := workspace.Load(ctx, workspacePath)
+	if err == nil {
+		return w, nil
+	}
+	missingVariablesError, ok := err.(modconfig.MissingVariableError)
+	// if there was an error which is NOT a MissingVariableError, return it
+	if !ok {
+		return nil, err
+	}
+	// if interactive input is disabled, return the missing variables error
+	if !viper.GetBool(constants.ArgInput) {
+		return nil, missingVariablesError
+	}
+	// so we have missing variables - prompt for them
+	// first hide spinner if it is there
+	statushooks.Done(ctx)
+	if err := PromptForMissingVariables(ctx, missingVariablesError.MissingVariables, workspacePath); err != nil {
+		log.Printf("[TRACE] Interactive variables prompting returned error %v", err)
+		return nil, err
+	}
+	// ok we should have all variables now - reload workspace
+	return workspace.Load(ctx, workspacePath)
+}

pkg/snapshot/progress_reporter.go

@@ -0,0 +1,53 @@
+package snapshot
+
+import (
+	"context"
+	"fmt"
+	"github.com/turbot/steampipe/pkg/statushooks"
+	"github.com/turbot/steampipe/pkg/utils"
+	"strings"
+	"sync"
+)
+
+// SnapshotProgressReporter is an empty implementation of SnapshotProgress
+type SnapshotProgressReporter struct {
+	rows     int
+	errors   int
+	nodeType string
+	name     string
+	mut      sync.Mutex
+}
+
+func NewSnapshotProgressReporter(target string) *SnapshotProgressReporter {
+	res := &SnapshotProgressReporter{
+		name: target,
+	}
+	return res
+}
+
+func (r *SnapshotProgressReporter) UpdateRowCount(ctx context.Context, rows int) {
+	r.mut.Lock()
+	defer r.mut.Unlock()
+
+	r.rows += rows
+	r.showProgress(ctx)
+}
+func (r *SnapshotProgressReporter) UpdateErrorCount(ctx context.Context, errors int) {
+	r.mut.Lock()
+	defer r.mut.Unlock()
+	r.errors += errors
+	r.showProgress(ctx)
+}
+
+func (r *SnapshotProgressReporter) showProgress(ctx context.Context) {
+	var msg strings.Builder
+	msg.WriteString(fmt.Sprintf("Running %s", r.name))
+	if r.rows > 0 {
+		msg.WriteString(fmt.Sprintf(", %d %s returned", r.rows, utils.Pluralize("row", r.rows)))
+	}
+	if r.errors > 0 {
+		msg.WriteString(fmt.Sprintf(", %d %s, ", r.errors, utils.Pluralize("error", r.errors)))
+	}
+
+	statushooks.SetStatus(ctx, msg.String())
+}

pkg/snapshot/snapshot.go

@@ -0,0 +1,84 @@
+package snapshot
+
+import (
+	"context"
+	"github.com/turbot/steampipe/pkg/contexthelpers"
+	"github.com/turbot/steampipe/pkg/control/controlstatus"
+	"github.com/turbot/steampipe/pkg/dashboard/dashboardevents"
+	"github.com/turbot/steampipe/pkg/dashboard/dashboardexecute"
+	"github.com/turbot/steampipe/pkg/dashboard/dashboardserver"
+	"github.com/turbot/steampipe/pkg/dashboard/dashboardtypes"
+	"github.com/turbot/steampipe/pkg/initialisation"
+	"github.com/turbot/steampipe/pkg/interactive"
+	"github.com/turbot/steampipe/pkg/statushooks"
+	"github.com/turbot/steampipe/pkg/utils"
+	"log"
+)
+
+func GenerateSnapshot(ctx context.Context, target string, inputs map[string]interface{}) (snapshot *dashboardtypes.SteampipeSnapshot, err error) {
+	// create context for the dashboard execution
+	snapshotCtx, cancel := createSnapshotContext(ctx, target)
+
+	contexthelpers.StartCancelHandler(cancel)
+
+	w, err := interactive.LoadWorkspacePromptingForVariables(snapshotCtx)
+	utils.FailOnErrorWithMessage(err, "failed to load workspace")
+
+	// todo do we require a mod file?
+
+	initData := initialisation.NewInitData(snapshotCtx, w)
+	// shutdown the service on exit
+	defer initData.Cleanup(snapshotCtx)
+	if err := initData.Result.Error; err != nil {
+		return nil, initData.Result.Error
+	}
+
+	// if there is a usage warning we display it
+	initData.Result.DisplayMessages()
+
+	sessionId := "generateSnapshot"
+
+	errorChannel := make(chan error)
+	resultChannel := make(chan *dashboardtypes.SteampipeSnapshot)
+	dashboardEventHandler := func(event dashboardevents.DashboardEvent) {
+		handleDashboardEvent(event, resultChannel, errorChannel)
+	}
+	w.RegisterDashboardEventHandler(dashboardEventHandler)
+	dashboardexecute.Executor.ExecuteDashboard(snapshotCtx, sessionId, target, inputs, w, initData.Client)
+
+	select {
+	case err = <-errorChannel:
+	case snapshot = <-resultChannel:
+	}
+
+	return snapshot, err
+
+}
+
+// create the context for the check run - add a control status renderer
+func createSnapshotContext(ctx context.Context, target string) (context.Context, context.CancelFunc) {
+	// create context for the dashboard execution
+	snapshotCtx, cancel := context.WithCancel(ctx)
+	contexthelpers.StartCancelHandler(cancel)
+
+	snapshotProgressReporter := NewSnapshotProgressReporter(target)
+	snapshotCtx = statushooks.AddSnapshotProgressToContext(snapshotCtx, snapshotProgressReporter)
+
+	// create a context with a SnapshotControlHooks to report execution progress of any controls in this snapshot
+	snapshotCtx = controlstatus.AddControlHooksToContext(snapshotCtx, controlstatus.NewSnapshotControlHooks())
+	return snapshotCtx, cancel
+}
+
+func handleDashboardEvent(event dashboardevents.DashboardEvent, resultChannel chan *dashboardtypes.SteampipeSnapshot, errorChannel chan error) {
+
+	switch e := event.(type) {
+
+	case *dashboardevents.ExecutionError:
+		errorChannel <- e.Error
+	case *dashboardevents.ExecutionComplete:
+		log.Println("[TRACE] execution complete event", *e)
+		snapshot := dashboardserver.ExecutionCompleteToSnapshot(e)
+
+		resultChannel <- snapshot
+	}
+}

pkg/snapshot/upload.go

@@ -0,0 +1 @@
+package snapshot

pkg/statushooks/context.go

@@ -8,8 +8,9 @@ import (
 )
 
 var (
-	contextKeyStatusHook      = contexthelpers.ContextKey("status_hook")
-	contextKeyMessageRenderer = contexthelpers.ContextKey("meddage_renderer")
+	contextKeySnapshotProgress = contexthelpers.ContextKey("snapshot_progress")
+	contextKeyStatusHook       = contexthelpers.ContextKey("status_hook")
+	contextKeyMessageRenderer  = contexthelpers.ContextKey("meddage_renderer")
 )
 
 func DisableStatusHooks(ctx context.Context) context.Context {
@@ -20,10 +21,6 @@ func AddStatusHooksToContext(ctx context.Context, statusHooks StatusHooks) conte
 	return context.WithValue(ctx, contextKeyStatusHook, statusHooks)
 }
 
-func AddMessageRendererToContext(ctx context.Context, messageRenderer MessageRenderer) context.Context {
-	return context.WithValue(ctx, contextKeyMessageRenderer, messageRenderer)
-}
-
 func StatusHooksFromContext(ctx context.Context) StatusHooks {
 	if ctx == nil {
 		return NullHooks
@@ -35,6 +32,25 @@ func StatusHooksFromContext(ctx context.Context) StatusHooks {
 	return NullHooks
 }
 
+func AddSnapshotProgressToContext(ctx context.Context, snapshotProgress SnapshotProgress) context.Context {
+	return context.WithValue(ctx, contextKeySnapshotProgress, snapshotProgress)
+}
+
+func SnapshotProgressFromContext(ctx context.Context) SnapshotProgress {
+	if ctx == nil {
+		return NullProgress
+	}
+	if val, ok := ctx.Value(contextKeySnapshotProgress).(SnapshotProgress); ok {
+		return val
+	}
+	// no snapshot progress in context - return null progress
+	return NullProgress
+}
+
+func AddMessageRendererToContext(ctx context.Context, messageRenderer MessageRenderer) context.Context {
+	return context.WithValue(ctx, contextKeyMessageRenderer, messageRenderer)
+}
+
 func SetStatus(ctx context.Context, msg string) {
 	StatusHooksFromContext(ctx).SetStatus(msg)
 }

pkg/statushooks/null_snapshot_progress.go

@@ -0,0 +1,11 @@
+package statushooks
+
+import "context"
+
+// NullProgress is an empty implementation of SnapshotProgress
+var NullProgress = &NullSnapshotProgress{}
+
+type NullSnapshotProgress struct{}
+
+func (*NullSnapshotProgress) UpdateRowCount(context.Context, int)   {}
+func (*NullSnapshotProgress) UpdateErrorCount(context.Context, int) {}

pkg/statushooks/snapshot_progress.go

@@ -0,0 +1,16 @@
+package statushooks
+
+import "context"
+
+type SnapshotProgress interface {
+	UpdateRowCount(context.Context, int)
+	UpdateErrorCount(context.Context, int)
+}
+
+func SnapshotError(ctx context.Context) {
+	SnapshotProgressFromContext(ctx).UpdateErrorCount(ctx, 1)
+}
+
+func UpdateSnapshotProgress(ctx context.Context, completedRows int) {
+	SnapshotProgressFromContext(ctx).UpdateRowCount(ctx, completedRows)
+}

pkg/steampipeconfig/load_mod_variables.go

@@ -6,13 +6,12 @@ import (
 
 	"sort"
 
-	"github.com/turbot/steampipe/pkg/steampipeconfig/parse"
-
 	"github.com/hashicorp/terraform/tfdiags"
 	"github.com/spf13/viper"
 	"github.com/turbot/steampipe/pkg/constants"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/inputvars"
 	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
+	"github.com/turbot/steampipe/pkg/steampipeconfig/parse"
 	"github.com/turbot/steampipe/pkg/utils"
 )
 

pkg/steampipeconfig/modconfig/parsed_name.go

@@ -42,6 +42,14 @@ func (p *ParsedResourceName) ToResourceName() string {
 	return BuildModResourceName(p.ItemType, p.Name)
 }
 
+func (p *ParsedResourceName) ToFullName() string {
+	return BuildFullResourceName(p.Mod, p.ItemType, p.Name)
+}
+
+func BuildFullResourceName(mod, blockType, name string) string {
+	return fmt.Sprintf("%s.%s.%s", mod, blockType, name)
+}
+
 // UnqualifiedResourceName removes the mod prefix from the given name
 func UnqualifiedResourceName(fullName string) string {
 	parts := strings.Split(fullName, ".")
@@ -53,6 +61,6 @@ func UnqualifiedResourceName(fullName string) string {
 	}
 }
 
-func BuildModResourceName(blockType string, name string) string {
+func BuildModResourceName(blockType, name string) string {
 	return fmt.Sprintf("%s.%s", blockType, name)
 }

tests/acceptance/test_data/templates/expected_plugin_help_output.txt

@@ -33,7 +33,6 @@ Global Flags:
       --cloud-host string           Steampipe Cloud host (default "cloud.steampipe.io")
       --cloud-token string          Steampipe Cloud authentication token
       --install-dir string          Path to the Config Directory (defaults to ~/.steampipe) (default "~/.steampipe")
-      --workspace string            Path to the workspace working directory (DEPRECATED: please use workspace-chdir)
       --workspace-chdir string      Path to the workspace working directory
       --workspace-database string   Steampipe Cloud workspace database (default "local")
 

tests/acceptance/test_data/templates/expected_service_help_output.txt

@@ -19,7 +19,6 @@ Global Flags:
       --cloud-host string           Steampipe Cloud host (default "cloud.steampipe.io")
       --cloud-token string          Steampipe Cloud authentication token
       --install-dir string          Path to the Config Directory (defaults to ~/.steampipe) (default "~/.steampipe")
-      --workspace string            Path to the workspace working directory (DEPRECATED: please use workspace-chdir)
       --workspace-chdir string      Path to the workspace working directory
       --workspace-database string   Steampipe Cloud workspace database (default "local")