Deprecate cloud-token and cloud-host args and params. Write token to new ~/.pipes location (#4147)

cmd/login.go

@@ -36,7 +36,7 @@ func loginCmd() *cobra.Command {
 func runLoginCmd(cmd *cobra.Command, _ []string) {
 	ctx := cmd.Context()
 
-	log.Printf("[TRACE] login, cloud host %s", viper.Get(constants.ArgCloudHost))
+	log.Printf("[TRACE] login, pipes host %s", viper.Get(constants.ArgPipesHost))
 	log.Printf("[TRACE] opening login web page")
 	// start login flow - this will open a web page prompting user to login, and will give the user a code to enter
 	var id, err = cloud.WebLogin(ctx)

cmd/root.go

@@ -8,6 +8,7 @@ import (
 	"github.com/mattn/go-isatty"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	filehelpers "github.com/turbot/go-kit/files"
 	"github.com/turbot/steampipe/pkg/constants"
 	"github.com/turbot/steampipe/pkg/error_helpers"
 	"github.com/turbot/steampipe/pkg/filepaths"
@@ -53,11 +54,13 @@ func InitCmd() {
 	utils.LogTime("cmd.root.InitCmd start")
 	defer utils.LogTime("cmd.root.InitCmd end")
 
+	defaultInstallDir, err := filehelpers.Tildefy(filepaths.DefaultInstallDir)
+	error_helpers.FailOnError(err)
 	rootCmd.SetVersionTemplate(fmt.Sprintf("Steampipe v%s\n", version.SteampipeVersion.String()))
 
 	// global flags
 	rootCmd.PersistentFlags().String(constants.ArgWorkspaceProfile, "default", "The workspace profile to use") // workspace profile profile is a global flag since install-dir(global) can be set through the workspace profile
-	rootCmd.PersistentFlags().String(constants.ArgInstallDir, filepaths.DefaultInstallDir, "Path to the Config Directory")
+	rootCmd.PersistentFlags().String(constants.ArgInstallDir, defaultInstallDir, "Path to the Config Directory")
 	rootCmd.PersistentFlags().Bool(constants.ArgSchemaComments, true, "Include schema comments when importing connection schemas")
 
 	error_helpers.FailOnError(viper.BindPFlag(constants.ArgInstallDir, rootCmd.PersistentFlags().Lookup(constants.ArgInstallDir)))

pkg/cloud/api.go

@@ -12,7 +12,7 @@ import (
 func newSteampipeCloudClient(token string) *steampipecloud.APIClient {
 	// Create a default configuration
 	configuration := steampipecloud.NewConfiguration()
-	configuration.Host = viper.GetString(constants.ArgCloudHost)
+	configuration.Host = viper.GetString(constants.ArgPipesHost)
 
 	// Add your Turbot Pipes user token as an auth header
 	if token != "" {
@@ -24,10 +24,10 @@ func newSteampipeCloudClient(token string) *steampipecloud.APIClient {
 }
 
 func getLoginTokenConfirmUIUrl() string {
-	url := url.URL{
+	confirmUrl := url.URL{
 		Scheme: "https",
-		Host:   viper.GetString(constants.ArgCloudHost),
+		Host:   viper.GetString(constants.ArgPipesHost),
 		Path:   "/login/token",
 	}
-	return url.String()
+	return confirmUrl.String()
 }

pkg/cloud/login.go

@@ -3,6 +3,7 @@ package cloud
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
 	"os"
@@ -22,7 +23,7 @@ var UnconfirmedError = "Not confirmed"
 // WebLogin POSTs to ${envBaseUrl}/api/latest/login/token to retrieve a login is
 // it then opens the login webpage and returns th eid
 func WebLogin(ctx context.Context) (string, error) {
-	client := newSteampipeCloudClient(viper.GetString(constants.ArgCloudToken))
+	client := newSteampipeCloudClient(viper.GetString(constants.ArgPipesToken))
 
 	tempTokenReq, _, err := client.Auth.LoginTokenCreate(ctx).Execute()
 	if err != nil {
@@ -32,8 +33,7 @@ func WebLogin(ctx context.Context) (string, error) {
 	// add in id query string
 	browserUrl := fmt.Sprintf("%s?r=%s", getLoginTokenConfirmUIUrl(), id)
 
-	fmt.Println()
+	fmt.Printf("\nVerify login at %s\n", browserUrl)
-	fmt.Printf("Verify login at %s\n", browserUrl)
 
 	if err = utils.OpenBrowser(browserUrl); err != nil {
 		log.Println("[INFO] failed to open login web page")
@@ -47,7 +47,8 @@ func GetLoginToken(ctx context.Context, id, code string) (string, error) {
 	client := newSteampipeCloudClient("")
 	tokenResp, _, err := client.Auth.LoginTokenGet(ctx, id).Code(code).Execute()
 	if err != nil {
-		if apiErr, ok := err.(steampipecloud.GenericOpenAPIError); ok {
+		var apiErr steampipecloud.GenericOpenAPIError
+		if errors.As(err, &apiErr) {
 			var body = map[string]any{}
 			if err := json.Unmarshal(apiErr.Body(), &body); err == nil {
 				return "", sperr.New("%s", body["detail"])
@@ -63,7 +64,7 @@ func GetLoginToken(ctx context.Context, id, code string) (string, error) {
 
 // SaveToken writes the token to  ~/.steampipe/internal/{cloud-host}.tptt
 func SaveToken(token string) error {
-	tokenPath := tokenFilePath(viper.GetString(constants.ArgCloudHost))
+	tokenPath := tokenFilePath(viper.GetString(constants.ArgPipesHost))
 	return sperr.Wrap(os.WriteFile(tokenPath, []byte(token), 0600))
 }
 
@@ -71,7 +72,7 @@ func LoadToken() (string, error) {
 	if err := migrateDefaultTokenFile(); err != nil {
 		log.Println("[TRACE] ERROR during migrating token file", err)
 	}
-	tokenPath := tokenFilePath(viper.GetString(constants.ArgCloudHost))
+	tokenPath := tokenFilePath(viper.GetString(constants.ArgPipesHost))
 	if !filehelpers.FileExists(tokenPath) {
 		return "", nil
 	}
@@ -84,25 +85,28 @@ func LoadToken() (string, error) {
 
 // migrateDefaultTokenFile migrates the cloud.steampipe.io.sptt token file
 // to the pipes.turbot.com.tptt token file
+// it also migrates the token file from the	~/.steampipe/internal directory to the ~/.pipes/internal directory
 func migrateDefaultTokenFile() error {
-	defaultTokenPath := tokenFilePath(constants.DefaultCloudHost)
+	defaultTokenPath := tokenFilePath(constants.DefaultPipesHost)
-	defaultLegacyTokenPath := legacyTokenFilePath(constants.LegacyDefaultCloudHost)
+	defaultLegacyTokenPaths := legacyTokenFilePaths()
 
-	if filehelpers.FileExists(defaultTokenPath) {
+	tokenExists := filehelpers.FileExists(defaultTokenPath)
-		// try removing the old legacy file - no worries if os.Remove fails
+
-		os.Remove(defaultLegacyTokenPath)
+	for _, legacyPath := range defaultLegacyTokenPaths {
-		// we have the new token file
+		if filehelpers.FileExists(legacyPath) {
-		return nil
+			if tokenExists {
+				// try removing the old legacy file - no worries if os.Remove fails
+				_ = os.Remove(legacyPath)
+			} else {
+				if err := utils.MoveFile(legacyPath, defaultTokenPath); err != nil {
+					return err
+				}
+				// set token exists flag so any other legacy files are removed (we do not expect any more)
+				tokenExists = true
+			}
+		}
 	}
 
-	// the default file does not exist
-	// check if the legacy one exists
-	if filehelpers.FileExists(defaultLegacyTokenPath) {
-		// move the legacy to the new
-		return utils.MoveFile(defaultLegacyTokenPath, defaultTokenPath)
-	}
-
-	// none exists - nothing to do
 	return nil
 }
 
@@ -122,12 +126,12 @@ func getActorName(actor steampipecloud.User) string {
 	return actor.Handle
 }
 
-func tokenFilePath(cloudHost string) string {
+func tokenFilePath(pipesHost string) string {
-	tokenPath := path.Join(filepaths.EnsureInternalDir(), fmt.Sprintf("%s%s", cloudHost, constants.TokenExtension))
+	tokenPath := path.Join(filepaths.EnsurePipesInternalDir(), fmt.Sprintf("%s%s", pipesHost, constants.TokenExtension))
 	return tokenPath
 }
 
-func legacyTokenFilePath(cloudHost string) string {
+func legacyTokenFilePaths() []string {
-	tokenPath := path.Join(filepaths.EnsureInternalDir(), fmt.Sprintf("%s%s", cloudHost, constants.LegacyTokenExtension))
+	return []string{path.Join(filepaths.EnsureInternalDir(), fmt.Sprintf("%s%s", constants.LegacyDefaultPipesHost, constants.LegacyTokenExtension)),
-	return tokenPath
+		path.Join(filepaths.EnsureInternalDir(), fmt.Sprintf("%s%s", constants.DefaultPipesHost, constants.TokenExtension))}
 }

pkg/cloud/snapshot.go

@@ -55,7 +55,7 @@ func exportSnapshot(snapshot *dashboardtypes.SteampipeSnapshot) (string, error)
 }
 
 func uploadSnapshot(ctx context.Context, snapshot *dashboardtypes.SteampipeSnapshot, share bool) (string, error) {
-	client := newSteampipeCloudClient(viper.GetString(constants.ArgCloudToken))
+	client := newSteampipeCloudClient(viper.GetString(constants.ArgPipesToken))
 
 	cloudWorkspace := viper.GetString(constants.ArgSnapshotLocation)
 	parts := strings.Split(cloudWorkspace, "/")
@@ -109,7 +109,7 @@ func uploadSnapshot(ctx context.Context, snapshot *dashboardtypes.SteampipeSnaps
 
 	snapshotId := uploadedSnapshot.Id
 	snapshotUrl := fmt.Sprintf("https://%s/%s/%s/workspace/%s/snapshot/%s",
-		viper.GetString(constants.ArgCloudHost),
+		viper.GetString(constants.ArgPipesHost),
 		workspaceType,
 		identityHandle,
 		workspaceHandle,

pkg/cmdconfig/builder.go

@@ -111,8 +111,10 @@ func (c *CmdBuilder) AddBoolFlag(name string, defaultValue bool, desc string, op
 // AddCloudFlags is helper function to add the cloud flags to a command
 func (c *CmdBuilder) AddCloudFlags() *CmdBuilder {
 	return c.
-		AddStringFlag(constants.ArgCloudHost, constants.DefaultCloudHost, "Turbot Pipes host").
+		AddStringFlag(constants.ArgPipesHost, constants.DefaultPipesHost, "Turbot Pipes host").
-		AddStringFlag(constants.ArgCloudToken, "", "Turbot Pipes authentication token")
+		AddStringFlag(constants.ArgPipesToken, "", "Turbot Pipes authentication token").
+		AddStringFlag(constants.ArgCloudHost, constants.DefaultPipesHost, "Turbot Pipes host", FlagOptions.Deprecated(constants.ArgPipesHost)).
+		AddStringFlag(constants.ArgCloudToken, "", "Turbot Pipes authentication token", FlagOptions.Deprecated(constants.ArgPipesToken))
 }
 
 // AddWorkspaceDatabaseFlag is helper function to add the workspace-databse flag to a command

pkg/cmdconfig/cmd_flags.go

@@ -13,7 +13,7 @@ var requiredColor = color.New(color.Bold).SprintfFunc()
 
 type flagOpt func(c *cobra.Command, name string, key string)
 
-// FlagOptions :: shortcut for common flag options
+// FlagOptions - shortcut for common flag options
 var FlagOptions = struct {
 	Required      func() flagOpt
 	Hidden        func() flagOpt

pkg/cmdconfig/cmd_hooks.go

@@ -207,7 +207,7 @@ func initGlobalConfig() *error_helpers.ErrorAndWarnings {
 	}
 
 	// set global containing the configured install dir (create directory if needed)
-	ensureInstallDir(viper.GetString(constants.ArgInstallDir))
+	ensureInstallDir()
 
 	// load the connection config and HCL options
 	config, loadConfigErrorsAndWarnings := steampipeconfig.LoadSteampipeConfig(ctx, viper.GetString(constants.ArgModLocation), cmd.Name())
@@ -232,8 +232,14 @@ func initGlobalConfig() *error_helpers.ErrorAndWarnings {
 		SetDefaultsFromConfig(loader.ConfiguredProfile.ConfigMap(cmd))
 	}
 
+	// handle deprecated cloud-host and cloud-token args and env vars
+	ew := handleDeprecations()
+	if ew.Error != nil {
+		return ew
+	}
+
 	// NOTE: we need to resolve the token separately
-	// - that is because we need the resolved value of ArgCloudHost in order to load any saved token
+	// - that is because we need the resolved value of ArgPipesHost in order to load any saved token
 	// and we cannot get this until the other config has been resolved
 	err = setCloudTokenDefault(loader)
 	if err != nil {
@@ -241,15 +247,56 @@ func initGlobalConfig() *error_helpers.ErrorAndWarnings {
 		return loadConfigErrorsAndWarnings
 	}
 
+	loadConfigErrorsAndWarnings.Merge(ew)
 	// now validate all config values have appropriate values
-	ew := validateConfig()
+	ew = validateConfig()
-	error_helpers.FailOnErrorWithMessage(ew.Error, "failed to validate config")
+	if ew.Error != nil {
-
+		return ew
+	}
 	loadConfigErrorsAndWarnings.Merge(ew)
 
 	return loadConfigErrorsAndWarnings
 }
 
+func handleDeprecations() *error_helpers.ErrorAndWarnings {
+	var ew = &error_helpers.ErrorAndWarnings{}
+	// if deprecated cloud-token or cloud-host is set, show a warning and copy the value to the new arg
+	if viper.IsSet(constants.ArgCloudToken) {
+		if viper.IsSet(constants.ArgPipesToken) {
+			ew.Error = sperr.New("Only one of flags --%s and --%s may be set", constants.ArgCloudToken, constants.ArgPipesToken)
+			return ew
+		}
+		viper.Set(constants.ArgPipesToken, viper.GetString(constants.ArgCloudToken))
+	}
+	if viper.IsSet(constants.ArgCloudHost) {
+		if viper.IsSet(constants.ArgPipesHost) {
+			ew.Error = sperr.New("Only one of flags --%s and --%s may be set", constants.ArgCloudHost, constants.ArgPipesHost)
+			return ew
+		}
+		viper.Set(constants.ArgPipesHost, viper.GetString(constants.ArgCloudHost))
+	}
+
+	// is deprecated STEAMPIPE_CLOUD_TOKEN env var set?
+	if _, isCloudTokenSet := os.LookupEnv(constants.EnvCloudToken); isCloudTokenSet {
+		// is PIPES_TOKEN also set? This is an error
+		if _, isPipesTokenSet := os.LookupEnv(constants.EnvPipesToken); isPipesTokenSet {
+			ew.Error = sperr.New("Only one of env vars %s and %s may be set", constants.EnvCloudToken, constants.EnvPipesToken)
+			return ew
+		}
+		// otherwise, show a warning
+		ew.AddWarning(fmt.Sprintf("The %s env var is deprecated - use %s", constants.EnvCloudToken, constants.EnvPipesToken))
+	}
+	// the same for STEAMPIPE_CLOUD_HOST
+	if _, isCloudTokenSet := os.LookupEnv(constants.EnvCloudHost); isCloudTokenSet {
+		if _, isPipesTokenSet := os.LookupEnv(constants.EnvPipesHost); isPipesTokenSet {
+			ew.Error = sperr.New("Only one of env vars %s and %s may be set", constants.EnvCloudHost, constants.EnvPipesHost)
+			return ew
+		}
+		ew.AddWarning(fmt.Sprintf("The %s env var is deprecated - use %s", constants.EnvCloudHost, constants.EnvPipesHost))
+	}
+	return ew
+}
+
 func setCloudTokenDefault(loader *steampipeconfig.WorkspaceProfileLoader) error {
 	/*
 	   saved cloud token
@@ -264,18 +311,28 @@ func setCloudTokenDefault(loader *steampipeconfig.WorkspaceProfileLoader) error
 		return err
 	}
 	if savedToken != "" {
-		viper.SetDefault(constants.ArgCloudToken, savedToken)
+		viper.SetDefault(constants.ArgPipesToken, savedToken)
 	}
-	// 2) default profile cloud token
+	// 2) default profile pipes token
+	if loader.DefaultProfile.PipesToken != nil {
+		viper.SetDefault(constants.ArgPipesToken, *loader.DefaultProfile.PipesToken)
+	}
+	// deprecated - cloud token
 	if loader.DefaultProfile.CloudToken != nil {
-		viper.SetDefault(constants.ArgCloudToken, *loader.DefaultProfile.CloudToken)
+		viper.SetDefault(constants.ArgPipesToken, *loader.DefaultProfile.CloudToken)
 	}
 	// 3) env var (STEAMIPE_CLOUD_TOKEN )
-	SetDefaultFromEnv(constants.EnvCloudToken, constants.ArgCloudToken, String)
+	SetDefaultFromEnv(constants.EnvPipesToken, constants.ArgPipesToken, String)
+	// deprecated env var
+	SetDefaultFromEnv(constants.EnvCloudToken, constants.ArgPipesToken, String)
 
 	// 4) explicit workspace profile
+	if p := loader.ConfiguredProfile; p != nil && p.PipesToken != nil {
+		viper.SetDefault(constants.ArgPipesToken, *p.PipesToken)
+	}
+	// deprecated - cloud token
 	if p := loader.ConfiguredProfile; p != nil && p.CloudToken != nil {
-		viper.SetDefault(constants.ArgCloudToken, *p.CloudToken)
+		viper.SetDefault(constants.ArgPipesToken, *p.CloudToken)
 	}
 	return nil
 }
@@ -374,7 +431,10 @@ func createLogger(logBuffer *bytes.Buffer, cmd *cobra.Command) {
 	}
 }
 
-func ensureInstallDir(installDir string) {
+func ensureInstallDir() {
+	pipesInstallDir := viper.GetString(constants.ArgPipesInstallDir)
+	installDir := viper.GetString(constants.ArgInstallDir)
+
 	log.Printf("[TRACE] ensureInstallDir %s", installDir)
 	if _, err := os.Stat(installDir); os.IsNotExist(err) {
 		log.Printf("[TRACE] creating install dir")
@@ -382,8 +442,15 @@ func ensureInstallDir(installDir string) {
 		error_helpers.FailOnErrorWithMessage(err, fmt.Sprintf("could not create installation directory: %s", installDir))
 	}
 
-	// store as SteampipeDir
+	if _, err := os.Stat(pipesInstallDir); os.IsNotExist(err) {
+		log.Printf("[TRACE] creating install dir")
+		err = os.MkdirAll(pipesInstallDir, 0755)
+		error_helpers.FailOnErrorWithMessage(err, fmt.Sprintf("could not create pipes installation directory: %s", pipesInstallDir))
+	}
+
+	// store as SteampipeDir and PipesInstallDir
 	filepaths.SteampipeDir = installDir
+	filepaths.PipesInstallDir = pipesInstallDir
 }
 
 // displayDeprecationWarnings shows the deprecated warnings in a formatted way

pkg/cmdconfig/validate.go

@@ -25,7 +25,7 @@ func ValidateSnapshotArgs(ctx context.Context) error {
 		return nil
 	}
 
-	token := viper.GetString(constants.ArgCloudToken)
+	token := viper.GetString(constants.ArgPipesToken)
 
 	// determine whether snapshot location is a cloud workspace or a file location
 	// if a file location, check it exists
@@ -43,7 +43,7 @@ func ValidateSnapshotArgs(ctx context.Context) error {
 	}
 
 	// should never happen as there is a default set
-	if viper.GetString(constants.ArgCloudHost) == "" {
+	if viper.GetString(constants.ArgPipesHost) == "" {
 		return fmt.Errorf("to share snapshots, cloud host must be set")
 	}
 

pkg/cmdconfig/viper.go

@@ -2,6 +2,7 @@ package cmdconfig
 
 import (
 	"fmt"
+	"github.com/turbot/steampipe/pkg/filepaths"
 	"log"
 	"os"
 
@@ -22,7 +23,9 @@ func Viper() *viper.Viper {
 // bootstrapViper sets up viper with the essential path config (workspace-chdir and install-dir)
 func bootstrapViper(loader *steampipeconfig.WorkspaceProfileLoader, cmd *cobra.Command) error {
 	// set defaults  for keys which do not have a corresponding command flag
-	setBaseDefaults()
+	if err := setBaseDefaults(); err != nil {
+		return err
+	}
 
 	// set defaults from defaultWorkspaceProfile
 	SetDefaultsFromConfig(loader.DefaultProfile.ConfigMap(cmd))
@@ -88,11 +91,16 @@ func SetDefaultsFromConfig(configMap map[string]interface{}) {
 //
 // Do not add keys here which have command line defaults - the way this is setup, this value takes
 // precedence over command line default
-func setBaseDefaults() {
+func setBaseDefaults() error {
+	pipesInstallDir, err := filehelpers.Tildefy(filepaths.DefaultPipesInstallDir)
+	if err != nil {
+		return err
+	}
 	defaults := map[string]interface{}{
 		// global general options
-		constants.ArgTelemetry:   constants.TelemetryInfo,
+		constants.ArgTelemetry:       constants.TelemetryInfo,
-		constants.ArgUpdateCheck: true,
+		constants.ArgUpdateCheck:     true,
+		constants.ArgPipesInstallDir: pipesInstallDir,
 
 		// workspace profile
 		constants.ArgAutoComplete:  true,
@@ -115,6 +123,7 @@ func setBaseDefaults() {
 	for k, v := range defaults {
 		viper.SetDefault(k, v)
 	}
+	return nil
 }
 
 type envMapping struct {
@@ -148,15 +157,11 @@ func setDefaultsFromEnv() {
 		constants.EnvIntrospection:  {[]string{constants.ArgIntrospection}, String},
 		constants.EnvTelemetry:      {[]string{constants.ArgTelemetry}, String},
 		constants.EnvUpdateCheck:    {[]string{constants.ArgUpdateCheck}, Bool},
-		// PIPES_HOST needs to be defined before STEAMPIPE_CLOUD_HOST,
+		// deprecated
-		// so that if STEAMPIPE_CLOUD_HOST is defined, it can override PIPES_HOST
+		constants.EnvCloudHost:             {[]string{constants.ArgPipesHost}, String},
-		constants.EnvPipesHost: {[]string{constants.ArgCloudHost}, String},
+		constants.EnvCloudToken:            {[]string{constants.ArgPipesToken}, String},
-		constants.EnvCloudHost: {[]string{constants.ArgCloudHost}, String},
+		constants.EnvPipesHost:             {[]string{constants.ArgPipesHost}, String},
-		// PIPES_TOKEN needs to be defined before STEAMPIPE_CLOUD_TOKEN,
+		constants.EnvPipesToken:            {[]string{constants.ArgPipesToken}, String},
-		// so that if STEAMPIPE_CLOUD_TOKEN is defined, it can override PIPES_TOKEN
-		constants.EnvPipesToken: {[]string{constants.ArgCloudToken}, String},
-		constants.EnvCloudToken: {[]string{constants.ArgCloudToken}, String},
-		//
 		constants.EnvSnapshotLocation:      {[]string{constants.ArgSnapshotLocation}, String},
 		constants.EnvWorkspaceDatabase:     {[]string{constants.ArgWorkspaceDatabase}, String},
 		constants.EnvServicePassword:       {[]string{constants.ArgServicePassword}, String},

pkg/constants/args.go

@@ -25,10 +25,13 @@ const (
 	ArgUpdateCheck             = "update-check"
 	ArgTelemetry               = "telemetry"
 	ArgInstallDir              = "install-dir"
+	ArgPipesInstallDir         = "pipes-install-dir"
 	ArgWorkspaceDatabase       = "workspace-database"
 	ArgSchemaComments          = "schema-comments"
 	ArgCloudHost               = "cloud-host"
 	ArgCloudToken              = "cloud-token"
+	ArgPipesHost               = "pipes-host"
+	ArgPipesToken              = "pipes-token"
 	ArgSearchPath              = "search-path"
 	ArgSearchPathPrefix        = "search-path-prefix"
 	ArgWatch                   = "watch"

pkg/constants/workspace_profile.go

@@ -1,7 +1,7 @@
 package constants
 
 const (
-	DefaultCloudHost         = "pipes.turbot.com"
+	DefaultPipesHost         = "pipes.turbot.com"
-	LegacyDefaultCloudHost   = "cloud.steampipe.io"
+	LegacyDefaultPipesHost   = "cloud.steampipe.io"
 	DefaultWorkspaceDatabase = "local"
 )

pkg/filepaths/pipes_install_dir.go

@@ -0,0 +1,35 @@
+package filepaths
+
+import (
+	"fmt"
+	"github.com/turbot/pipe-fittings/error_helpers"
+	"os"
+	"path/filepath"
+)
+
+// PipesInstallDir is the location of config files commen between pipelings
+// this must be set by the application at startup
+var PipesInstallDir = ""
+
+func ensurePipesInstallSubDir(dirName string) string {
+	subDir := installPipesSubDir(dirName)
+
+	if _, err := os.Stat(subDir); os.IsNotExist(err) {
+		err = os.MkdirAll(subDir, 0755)
+		error_helpers.FailOnErrorWithMessage(err, fmt.Sprintf("could not create %s directory", dirName))
+	}
+
+	return subDir
+}
+
+func installPipesSubDir(dirName string) string {
+	if PipesInstallDir == "" {
+		panic(fmt.Errorf("cannot call any pipes directory functions before PipesInstallDir is set"))
+	}
+	return filepath.Join(PipesInstallDir, dirName)
+}
+
+// EnsurePipesInternalDir returns the path to the pipes internal directory (creates if missing)
+func EnsurePipesInternalDir() string {
+	return ensurePipesInstallSubDir("internal")
+}

pkg/filepaths/steampipe.go

@@ -12,7 +12,8 @@ import (
 
 // Constants for Config
 const (
-	DefaultInstallDir = "~/.steampipe"
+	DefaultInstallDir      = "~/.steampipe"
+	DefaultPipesInstallDir = "~/.pipes"
 
 	connectionsStateFileName     = "connection.json"
 	versionFileName              = "versions.json"

pkg/initialisation/cloud_metadata.go

@@ -25,7 +25,7 @@ func getCloudMetadata(ctx context.Context) (*steampipeconfig.CloudMetadata, erro
 	workspaceDatabaseIsConnectionString := strings.HasPrefix(workspaceDatabase, "postgresql://") || strings.HasPrefix(workspaceDatabase, "postgres://")
 	if !workspaceDatabaseIsConnectionString {
 		// it must be a database name - verify the cloud token was provided
-		cloudToken := viper.GetString(constants.ArgCloudToken)
+		cloudToken := viper.GetString(constants.ArgPipesToken)
 		if cloudToken == "" {
 			return nil, error_helpers.MissingCloudTokenError
 		}

pkg/steampipeconfig/modconfig/workspace_profile.go

@@ -14,9 +14,13 @@ import (
 )
 
 type WorkspaceProfile struct {
-	ProfileName       string            `hcl:"name,label" cty:"name"`
+	ProfileName string `hcl:"name,label" cty:"name"`
-	CloudHost         *string           `hcl:"cloud_host,optional" cty:"cloud_host"`
+	// deprecated
+	CloudHost *string `hcl:"cloud_host,optional" cty:"cloud_host"`
+	PipesHost *string `hcl:"pipes_host,optional" cty:"pipes_host"`
+	// deprecated
 	CloudToken        *string           `hcl:"cloud_token,optional" cty:"cloud_token"`
+	PipesToken        *string           `hcl:"pipes_token,optional" cty:"pipes_token"`
 	InstallDir        *string           `hcl:"install_dir,optional" cty:"install_dir"`
 	ModLocation       *string           `hcl:"mod_location,optional" cty:"mod_location"`
 	QueryTimeout      *int              `hcl:"query_timeout,optional" cty:"query_timeout"`
@@ -39,12 +43,14 @@ type WorkspaceProfile struct {
 	CheckOptions     *options.Check                     `cty:"check-options"`
 	DashboardOptions *options.WorkspaceProfileDashboard `cty:"dashboard-options"`
 	DeclRange        hcl.Range
+	block            *hcl.Block
 }
 
 func NewWorkspaceProfile(block *hcl.Block) *WorkspaceProfile {
 	return &WorkspaceProfile{
 		ProfileName: block.Labels[0],
-		DeclRange:   block.TypeRange,
+		DeclRange:   hclhelpers.BlockRange(block),
+		block:       block,
 	}
 }
 
@@ -96,7 +102,43 @@ func (p *WorkspaceProfile) CtyValue() (cty.Value, error) {
 
 func (p *WorkspaceProfile) OnDecoded() hcl.Diagnostics {
 	p.setBaseProperties()
-	return nil
+
+	return p.validate()
+}
+
+func (p *WorkspaceProfile) validate() hcl.Diagnostics {
+	var diags hcl.Diagnostics
+	// validate that both deprecated and new versions of propertied have not been set
+	if p.CloudHost != nil && p.PipesHost != nil {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "both cloud_host and pipes_host cannot be set",
+			Subject:  hclhelpers.BlockRangePointer(p.block),
+		})
+	}
+	if p.CloudToken != nil && p.PipesToken != nil {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "both cloud_token and pipes_token cannot be set",
+			Subject:  hclhelpers.BlockRangePointer(p.block),
+		})
+	}
+	// return warnings if deprecated properties are set
+	if p.CloudHost != nil {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagWarning,
+			Summary:  "cloud_host is deprecated, use pipes_host",
+			Subject:  hclhelpers.BlockRangePointer(p.block),
+		})
+	}
+	if p.CloudToken != nil {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagWarning,
+			Summary:  "cloud_token is deprecated, use pipes_token",
+			Subject:  hclhelpers.BlockRangePointer(p.block),
+		})
+	}
+	return diags
 }
 
 func (p *WorkspaceProfile) setBaseProperties() {
@@ -186,8 +228,10 @@ func (p *WorkspaceProfile) ConfigMap(cmd *cobra.Command) map[string]interface{}
 	res := ConfigMap{}
 	// add non-empty properties to config map
 
-	res.SetStringItem(p.CloudHost, constants.ArgCloudHost)
+	res.SetStringItem(p.CloudHost, constants.ArgPipesHost)
-	res.SetStringItem(p.CloudToken, constants.ArgCloudToken)
+	res.SetStringItem(p.CloudToken, constants.ArgPipesToken)
+	res.SetStringItem(p.PipesHost, constants.ArgPipesHost)
+	res.SetStringItem(p.PipesToken, constants.ArgPipesToken)
 	res.SetStringItem(p.InstallDir, constants.ArgInstallDir)
 	res.SetStringItem(p.ModLocation, constants.ArgModLocation)
 	res.SetStringItem(p.SnapshotLocation, constants.ArgSnapshotLocation)

tests/acceptance/test_data/source_files/config_tests/workspace_profiles/workspaces.spc

@@ -1,8 +1,8 @@
 
 workspace "default" {
   introspection = "info"
-  cloud_host = "latestpipe.turbot.io/"
+  pipes_host = "latestpipe.turbot.io/"
-  cloud_token = "spt_012faketoken34567890_012faketoken3456789099999"
+  pipes_token = "spt_012faketoken34567890_012faketoken3456789099999"
   install_dir = "sp_install_dir_default"
   mod_location = "sp_install_dir_default"
   snapshot_location = "snaps"
@@ -11,8 +11,8 @@ workspace "default" {
 
 workspace "sample" {
   introspection = "control"
-  cloud_host = "testpipe.turbot.io"
+  pipes_host = "testpipe.turbot.io"
-  cloud_token = "spt_012faketoken34567890_012faketoken3456789099999"
+  pipes_token = "spt_012faketoken34567890_012faketoken3456789099999"
   install_dir = "sp_install_dir_sample"
   mod_location = "sp_install_dir_sample"
   snapshot_location = "snap"

tests/acceptance/test_data/source_files/config_tests/workspace_profiles_options/workspaces.spc

@@ -1,6 +1,6 @@
 workspace "default" {
-  cloud_host = "latestpipe.turbot.io/"
+  pipes_host = "latestpipe.turbot.io/"
-  cloud_token = "spt_012faketoken34567890_012faketoken3456789099999"
+  pipes_token = "spt_012faketoken34567890_012faketoken3456789099999"
   mod_location = "sp_install_dir_default"
   snapshot_location = "snaps"
   workspace_database = "fk43e7"
@@ -27,8 +27,8 @@ workspace "default" {
 }
 
 workspace "sample" {
-  cloud_host = "latestpipe.turbot.io/"
+  pipes_host = "latestpipe.turbot.io/"
-  cloud_token = "spt_012faketoken34567890_012faketoken3456789099999"
+  pipes_token = "spt_012faketoken34567890_012faketoken3456789099999"
   mod_location = "sp_install_dir_sample"
   snapshot_location = "snaps"
   workspace_database = "fk43e7"

tests/acceptance/test_data/source_files/config_tests/workspace_tests.json

@@ -11,8 +11,8 @@
     },
     "expected": {
       "introspection": "info",
-      "cloud-host": "latestpipe.turbot.io/",
+      "pipes-host": "latestpipe.turbot.io/",
-      "cloud-token": "spt_012faketoken34567890_012faketoken3456789099999",
+      "pipes-token": "spt_012faketoken34567890_012faketoken3456789099999",
       "install-dir": "sp_install_dir_default",
       "mod-location": "sp_install_dir_default",
       "snapshot-location": "snaps",
@@ -27,10 +27,10 @@
     "setup": {
       "env": [
         "STEAMPIPE_WORKSPACE_PROFILES_LOCATION=workspace_profiles",
-        "STEAMPIPE_CLOUD_HOST=testpipe.turbot.io",
+        "PIPES_HOST=testpipe.turbot.io",
         "STEAMPIPE_MOD_LOCATION=sp_install_dir_env",
         "STEAMPIPE_INSTALL_DIR=sp_install_dir_env",
-        "STEAMPIPE_CLOUD_TOKEN=spt_012faketoken34567890_012faketoken3456789099996",
+        "PIPES_TOKEN=spt_012faketoken34567890_012faketoken3456789099996",
         "STEAMPIPE_SNAPSHOT_LOCATION=snapshot",
         "STEAMPIPE_WORKSPACE_DATABASE=fk/43e7",
         "STEAMPIPE_INTROSPECTION=none"
@@ -38,16 +38,16 @@
       "args": [
         "--install-dir=sp_install_dir_default",
         "--mod-location=sp_install_dir_default",
-        "--cloud-host=fastestpipe.turbot.io",
+        "--pipes-host=fastestpipe.turbot.io",
-        "--cloud-token=spt_012faketoken34567890_012faketoken3456789099990",
+        "--pipes-token=spt_012faketoken34567890_012faketoken3456789099990",
         "--snapshot-location=snaps",
         "--workspace-database=fk43e9"
       ]
     },
     "expected": {
       "introspection": "none",
-      "cloud-host": "fastestpipe.turbot.io",
+      "pipes-host": "fastestpipe.turbot.io",
-      "cloud-token": "spt_012faketoken34567890_012faketoken3456789099990",
+      "pipes-token": "spt_012faketoken34567890_012faketoken3456789099990",
       "install-dir": "sp_install_dir_default",
       "mod-location": "sp_install_dir_default",
       "snapshot-location": "snaps",
@@ -61,18 +61,18 @@
     "cmd": "query",
     "setup": {
       "env": [
-        "STEAMPIPE_CLOUD_HOST=latestpipe.turbot.io/",
+        "PIPES_HOST=latestpipe.turbot.io/",
         "STEAMPIPE_INSTALL_DIR=sp_install_dir_env",
         "STEAMPIPE_MOD_LOCATION=sp_install_dir_env",
-        "STEAMPIPE_CLOUD_TOKEN=spt_012faketoken34567890_012faketoken3456789099999",
+        "PIPES_TOKEN=spt_012faketoken34567890_012faketoken3456789099999",
         "STEAMPIPE_SNAPSHOT_LOCATION=snaps",
         "STEAMPIPE_WORKSPACE_DATABASE=fk43e7"
       ],
       "args": []
     },
     "expected": {
-      "cloud-host": "latestpipe.turbot.io/",
+      "pipes-host": "latestpipe.turbot.io/",
-      "cloud-token": "spt_012faketoken34567890_012faketoken3456789099999",
+      "pipes-token": "spt_012faketoken34567890_012faketoken3456789099999",
       "install-dir": "sp_install_dir_env",
       "mod-location": "sp_install_dir_env",
       "snapshot-location": "snaps",
@@ -94,8 +94,8 @@
     },
     "expected": {
       "introspection": "control",
-      "cloud-host": "testpipe.turbot.io",
+      "pipes-host": "testpipe.turbot.io",
-      "cloud-token": "spt_012faketoken34567890_012faketoken3456789099999",
+      "pipes-token": "spt_012faketoken34567890_012faketoken3456789099999",
       "install-dir": "sp_install_dir_sample",
       "mod-location": "sp_install_dir_sample",
       "snapshot-location": "snap",
@@ -112,15 +112,15 @@
       "args": [
         "--install-dir=sp_install_dir_sample",
         "--mod-location=sp_install_dir_sample",
-        "--cloud-host=fastestpipe.turbot.io",
+        "--pipes-host=fastestpipe.turbot.io",
-        "--cloud-token=spt_012faketoken34567890_012faketoken3456789099990",
+        "--pipes-token=spt_012faketoken34567890_012faketoken3456789099990",
         "--snapshot-location=snaps",
         "--workspace-database=fk43e9"
       ]
     },
     "expected": {
-      "cloud-host": "fastestpipe.turbot.io",
+      "pipes-host": "fastestpipe.turbot.io",
-      "cloud-token": "spt_012faketoken34567890_012faketoken3456789099990",
+      "pipes-token": "spt_012faketoken34567890_012faketoken3456789099990",
       "install-dir": "sp_install_dir_sample",
       "mod-location": "sp_install_dir_sample",
       "snapshot-location": "snaps",
@@ -135,18 +135,18 @@
     "setup": {
       "env": [
         "STEAMPIPE_WORKSPACE_PROFILES_LOCATION=workspace_profiles",
-        "STEAMPIPE_CLOUD_HOST=fastestpipe.turbot.io/",
+        "PIPES_HOST=fastestpipe.turbot.io/",
         "STEAMPIPE_INSTALL_DIR=sp_install_dir_env",
         "STEAMPIPE_MOD_LOCATION=sp_install_dir_env",
-        "STEAMPIPE_CLOUD_TOKEN=spt_012faketoken34567890_012faketoken3456789099996",
+        "PIPES_TOKEN=spt_012faketoken34567890_012faketoken3456789099996",
         "STEAMPIPE_SNAPSHOT_LOCATION=snapshot",
         "STEAMPIPE_WORKSPACE_DATABASE=ab43e6"
       ],
       "args": []
     },
     "expected": {
-      "cloud-host": "fastestpipe.turbot.io/",
+      "pipes-host": "fastestpipe.turbot.io/",
-      "cloud-token": "spt_012faketoken34567890_012faketoken3456789099996",
+      "pipes-token": "spt_012faketoken34567890_012faketoken3456789099996",
       "install-dir": "sp_install_dir_env",
       "mod-location": "sp_install_dir_env",
       "snapshot-location": "snapshot",
@@ -161,10 +161,10 @@
     "setup": {
       "env": [
         "STEAMPIPE_WORKSPACE_PROFILES_LOCATION=workspace_profiles",
-        "STEAMPIPE_CLOUD_HOST=fastestpipe.turbot.io/",
+        "PIPES_HOST=fastestpipe.turbot.io/",
         "STEAMPIPE_INSTALL_DIR=sp_install_dir_env",
         "STEAMPIPE_MOD_LOCATION=sp_install_dir_env",
-        "STEAMPIPE_CLOUD_TOKEN=spt_012faketoken34567890_012faketoken3456789099996",
+        "PIPES_TOKEN=spt_012faketoken34567890_012faketoken3456789099996",
         "STEAMPIPE_SNAPSHOT_LOCATION=snapshot",
         "STEAMPIPE_WORKSPACE_DATABASE=ab43e6"
       ],
@@ -173,8 +173,8 @@
       ]
     },
     "expected": {
-      "cloud-host": "testpipe.turbot.io",
+      "pipes-host": "testpipe.turbot.io",
-      "cloud-token": "spt_012faketoken34567890_012faketoken3456789099999",
+      "pipes-token": "spt_012faketoken34567890_012faketoken3456789099999",
       "install-dir": "sp_install_dir_sample",
       "mod-location": "sp_install_dir_sample",
       "snapshot-location": "snap",
@@ -189,10 +189,10 @@
     "setup": {
       "env": [
         "STEAMPIPE_WORKSPACE_PROFILES_LOCATION=workspace_profiles",
-        "STEAMPIPE_CLOUD_HOST=fastestpipe.turbot.io/",
+        "PIPES_HOST=fastestpipe.turbot.io/",
         "STEAMPIPE_INSTALL_DIR=sp_install_dir_env",
         "STEAMPIPE_MOD_LOCATION=sp_install_dir_env",
-        "STEAMPIPE_CLOUD_TOKEN=spt_012faketoken34567890_012faketoken3456789099996",
+        "PIPES_TOKEN=spt_012faketoken34567890_012faketoken3456789099996",
         "STEAMPIPE_SNAPSHOT_LOCATION=snapshot",
         "STEAMPIPE_WORKSPACE_DATABASE=ab43e6"
       ],
@@ -201,8 +201,8 @@
       ]
     },
     "expected": {
-      "cloud-host": "testpipe.turbot.io",
+      "pipes-host": "testpipe.turbot.io",
-      "cloud-token": "spt_012faketoken34567890_012faketoken3456789099999",
+      "pipes-token": "spt_012faketoken34567890_012faketoken3456789099999",
       "install-dir": "sp_install_dir_sample",
       "mod-location": "sp_install_dir_sample",
       "snapshot-location": "snap",
@@ -221,15 +221,15 @@
       "args": [
         "--install-dir=sp_install_dir_default",
         "--mod-location=sp_install_dir_default",
-        "--cloud-host=fastestpipe.turbot.io",
+        "--pipes-host=fastestpipe.turbot.io",
-        "--cloud-token=spt_012faketoken34567890_012faketoken3456789099990",
+        "--pipes-token=spt_012faketoken34567890_012faketoken3456789099990",
         "--snapshot-location=snaps",
         "--workspace-database=fk43e9"
       ]
     },
     "expected": {
-      "cloud-host": "fastestpipe.turbot.io",
+      "pipes-host": "fastestpipe.turbot.io",
-      "cloud-token": "spt_012faketoken34567890_012faketoken3456789099990",
+      "pipes-token": "spt_012faketoken34567890_012faketoken3456789099990",
       "install-dir": "sp_install_dir_default",
       "mod-location": "sp_install_dir_default",
       "snapshot-location": "snaps",

tests/acceptance/test_files/cloud.bats

@@ -20,7 +20,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
 
 @test "connect to cloud workspace - passing the cloud-token arg and the workspace name to workspace-database arg" {
   # run steampipe query and fetch an account from the cloud workspace
-  run steampipe query "select account_aliases from all_aws.aws_account where account_id='632902152528'" --cloud-token $SPIPETOOLS_TOKEN --workspace-database spipetools/toolstest --output json
+  run steampipe query "select account_aliases from all_aws.aws_account where account_id='632902152528'" --pipes-token $SPIPETOOLS_TOKEN --workspace-database spipetools/toolstest --output json
   echo $output
 
   # fetch the value of account_alias to compare
@@ -33,7 +33,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
 
 @test "connect to cloud workspace - passing the cloud-host arg, the cloud-token arg and the workspace name to workspace-database arg" {
   # run steampipe query and fetch an account from the cloud workspace
-  run steampipe query "select account_aliases from all_aws.aws_account where account_id='632902152528'" --cloud-host "pipes.turbot.com" --cloud-token $SPIPETOOLS_TOKEN --workspace-database spipetools/toolstest --output json
+  run steampipe query "select account_aliases from all_aws.aws_account where account_id='632902152528'" --pipes-host "pipes.turbot.com" --pipes-token $SPIPETOOLS_TOKEN --workspace-database spipetools/toolstest --output json
   echo $output
 
   # fetch the value of account_alias to compare

tests/acceptance/test_files/plugin.bats

@@ -1,13 +1,6 @@
 load "$LIB_BATS_ASSERT/load.bash"
 load "$LIB_BATS_SUPPORT/load.bash"
 
-@test "steampipe plugin help is displayed when no sub command given" {
-  steampipe plugin > test.txt
-
-  assert_equal "$(cat test.txt)" "$(cat $TEST_DATA_DIR/expected_plugin_help_output.txt)"
-  rm -f test.txt*
-}
-
 @test "plugin install" {
   run steampipe plugin install net
   assert_success

tests/acceptance/test_files/service.bats

@@ -1,13 +1,6 @@
 load "$LIB_BATS_ASSERT/load.bash"
 load "$LIB_BATS_SUPPORT/load.bash"
 
-@test "steampipe service help is displayed when no sub command given" {
-  steampipe service > test.txt
-
-  assert_equal "$(cat test.txt)" "$(cat $TEST_DATA_DIR/expected_service_help_output.txt)"
-  rm -f test.txt*
-}
-
 @test "steampipe service start" {
     run steampipe service start
     assert_success