From a01776b78f0226ab1e7f925f59b887e154ab0c67 Mon Sep 17 00:00:00 2001 From: Binaek Sarkar Date: Tue, 8 Aug 2023 16:54:38 +0530 Subject: [PATCH] Migrate from 'cloud.steampipe.io' to 'pipes.turbot.com'. Closes #3724 --- README.md | 4 +-- cmd/check.go | 6 ++-- cmd/dashboard.go | 6 ++-- cmd/login.go | 4 +-- cmd/query.go | 9 ++--- cmd/root.go | 6 ++-- design/steampipe_data_files.md | 2 +- pkg/cloud/api.go | 2 +- pkg/cloud/doc.go | 2 +- pkg/cloud/login.go | 34 ++++++++++++++++++- pkg/cmdconfig/viper.go | 23 ++++++++----- pkg/constants/default_workspaces.go | 2 +- pkg/constants/env.go | 3 ++ pkg/constants/extensions.go | 3 +- pkg/constants/workspace_profile.go | 3 +- pkg/error_helpers/errors.go | 2 +- .../workspace_profile_loader.go | 2 +- .../templates/expected_plugin_help_output.txt | 6 ++-- .../expected_service_help_output.txt | 6 ++-- tests/acceptance/test_files/cloud.bats | 4 +-- 20 files changed, 87 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 8a670fbff..4c4abf11e 100644 --- a/README.md +++ b/README.md @@ -330,7 +330,7 @@ We thrive on feedback and community involvement! **Want to work with the team?** → We are [hiring](https://turbot.com/careers?utm_id=gspreadme&utm_source=github&utm_medium=repo&utm_campaign=github&utm_content=readme)! -## Steampipe Cloud +## Turbot Pipes -Want a hosted version of Steampipe? Bring your team to [Steampipe Cloud](https://cloud.steampipe.io?utm_id=gspreadme&utm_source=github&utm_medium=repo&utm_campaign=github&utm_content=readme). +Want a hosted version of Steampipe? Bring your team to [Turbot Pipes](https://pipes.turbot.com?utm_id=gspreadme&utm_source=github&utm_medium=repo&utm_campaign=github&utm_content=readme). diff --git a/cmd/check.go b/cmd/check.go index 4270e88de..4705a61aa 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -78,10 +78,10 @@ You may specify one or more benchmarks or controls to run (separated by a space) AddIntFlag(constants.ArgMaxParallel, constants.DefaultMaxConnections, "The maximum number of concurrent database connections to open"). AddBoolFlag(constants.ArgModInstall, true, "Specify whether to install mod dependencies before running the check"). AddBoolFlag(constants.ArgInput, true, "Enable interactive prompts"). - AddBoolFlag(constants.ArgSnapshot, false, "Create snapshot in Steampipe Cloud with the default (workspace) visibility"). - AddBoolFlag(constants.ArgShare, false, "Create snapshot in Steampipe Cloud with 'anyone_with_link' visibility"). + AddBoolFlag(constants.ArgSnapshot, false, "Create snapshot in Turbot Pipes with the default (workspace) visibility"). + AddBoolFlag(constants.ArgShare, false, "Create snapshot in Turbot Pipes with 'anyone_with_link' visibility"). AddStringArrayFlag(constants.ArgSnapshotTag, nil, "Specify tags to set on the snapshot"). - AddStringFlag(constants.ArgSnapshotLocation, "", "The location to write snapshots - either a local file path or a Steampipe Cloud workspace"). + AddStringFlag(constants.ArgSnapshotLocation, "", "The location to write snapshots - either a local file path or a Turbot Pipes workspace"). AddStringFlag(constants.ArgSnapshotTitle, "", "The title to give a snapshot") cmd.AddCommand(getListSubCmd(listSubCmdOptions{parentCmd: cmd})) diff --git a/cmd/dashboard.go b/cmd/dashboard.go index 345f384c8..3d1590e27 100644 --- a/cmd/dashboard.go +++ b/cmd/dashboard.go @@ -58,9 +58,9 @@ The current mod is the working directory, or the directory specified by the --mo AddStringArrayFlag(constants.ArgVariable, nil, "Specify the value of a variable"). AddBoolFlag(constants.ArgInput, true, "Enable interactive prompts"). AddStringFlag(constants.ArgOutput, constants.OutputFormatNone, "Select a console output format: none, snapshot"). - AddBoolFlag(constants.ArgSnapshot, false, "Create snapshot in Steampipe Cloud with the default (workspace) visibility"). - AddBoolFlag(constants.ArgShare, false, "Create snapshot in Steampipe Cloud with 'anyone_with_link' visibility"). - AddStringFlag(constants.ArgSnapshotLocation, "", "The location to write snapshots - either a local file path or a Steampipe Cloud workspace"). + AddBoolFlag(constants.ArgSnapshot, false, "Create snapshot in Turbot Pipes with the default (workspace) visibility"). + AddBoolFlag(constants.ArgShare, false, "Create snapshot in Turbot Pipes with 'anyone_with_link' visibility"). + AddStringFlag(constants.ArgSnapshotLocation, "", "The location to write snapshots - either a local file path or a Turbot Pipes workspace"). AddStringFlag(constants.ArgSnapshotTitle, "", "The title to give a snapshot"). // NOTE: use StringArrayFlag for ArgDashboardInput, not StringSliceFlag // Cobra will interpret values passed to a StringSliceFlag as CSV, where args passed to StringArrayFlag are not parsed and used raw diff --git a/cmd/login.go b/cmd/login.go index dc555855e..b5fe6d68d 100644 --- a/cmd/login.go +++ b/cmd/login.go @@ -22,8 +22,8 @@ func loginCmd() *cobra.Command { TraverseChildren: true, Args: cobra.NoArgs, Run: runLoginCmd, - Short: "Login to Steampipe Cloud", - Long: `Login to Steampipe Cloud.`, + Short: "Login to Turbot Pipes", + Long: `Login to Turbot Pipes.`, } cmdconfig.OnCmd(cmd). diff --git a/cmd/query.go b/cmd/query.go index f206fba37..6cb1a6129 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -5,11 +5,12 @@ import ( "context" "encoding/json" "fmt" - "github.com/turbot/steampipe/pkg/connection_sync" "os" "path" "strings" + "github.com/turbot/steampipe/pkg/connection_sync" + "github.com/hashicorp/hcl/v2" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -85,13 +86,13 @@ Examples: // where args passed to StringArrayFlag are not parsed and used raw AddStringArrayFlag(constants.ArgVariable, nil, "Specify the value of a variable"). AddBoolFlag(constants.ArgInput, true, "Enable interactive prompts"). - AddBoolFlag(constants.ArgSnapshot, false, "Create snapshot in Steampipe Cloud with the default (workspace) visibility"). - AddBoolFlag(constants.ArgShare, false, "Create snapshot in Steampipe Cloud with 'anyone_with_link' visibility"). + AddBoolFlag(constants.ArgSnapshot, false, "Create snapshot in Turbot Pipes with the default (workspace) visibility"). + AddBoolFlag(constants.ArgShare, false, "Create snapshot in Turbot Pipes with 'anyone_with_link' visibility"). AddStringArrayFlag(constants.ArgSnapshotTag, nil, "Specify tags to set on the snapshot"). AddStringFlag(constants.ArgSnapshotTitle, "", "The title to give a snapshot"). AddIntFlag(constants.ArgDatabaseQueryTimeout, 0, "The query timeout"). AddStringSliceFlag(constants.ArgExport, nil, "Export output to file, supported format: sps (snapshot)"). - AddStringFlag(constants.ArgSnapshotLocation, "", "The location to write snapshots - either a local file path or a Steampipe Cloud workspace"). + AddStringFlag(constants.ArgSnapshotLocation, "", "The location to write snapshots - either a local file path or a Turbot Pipes workspace"). AddBoolFlag(constants.ArgProgress, true, "Display snapshot upload status") cmd.AddCommand(getListSubCmd(listSubCmdOptions{parentCmd: cmd})) diff --git a/cmd/root.go b/cmd/root.go index 5bf74c05a..ba89c7379 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -180,9 +180,9 @@ func InitCmd() { rootCmd.PersistentFlags().Bool(constants.ArgSchemaComments, true, "Include schema comments when importing connection schemas") // TODO elevate these to specific command? they are not used for plugin or mod commands // or else put validation for plugin commands or at least a warning - rootCmd.PersistentFlags().String(constants.ArgCloudHost, constants.DefaultCloudHost, "Steampipe Cloud host") - rootCmd.PersistentFlags().String(constants.ArgCloudToken, "", "Steampipe Cloud authentication token") - rootCmd.PersistentFlags().String(constants.ArgWorkspaceDatabase, constants.DefaultWorkspaceDatabase, "Steampipe Cloud workspace database") + rootCmd.PersistentFlags().String(constants.ArgCloudHost, constants.DefaultCloudHost, "Turbot Pipes host") + rootCmd.PersistentFlags().String(constants.ArgCloudToken, "", "Turbot Pipes authentication token") + rootCmd.PersistentFlags().String(constants.ArgWorkspaceDatabase, constants.DefaultWorkspaceDatabase, "Turbot Pipes workspace database") rootCmd.PersistentFlags().String(constants.ArgWorkspaceProfile, "default", "The workspace profile to use") error_helpers.FailOnError(viper.BindPFlag(constants.ArgInstallDir, rootCmd.PersistentFlags().Lookup(constants.ArgInstallDir))) diff --git a/design/steampipe_data_files.md b/design/steampipe_data_files.md index b21bc0dfd..c028977ea 100644 --- a/design/steampipe_data_files.md +++ b/design/steampipe_data_files.md @@ -8,7 +8,7 @@ - `.passwd` - Stores the database password. Deleting the file does not effect steampipe, you can view your password by using the --show-password flag along with the service commands. Starting the service would re-create the file. -- `cloud.steampipe.io.sptt` - Stores the steampipe cloud token. Deleting the file would require you to run steampipe login again. +- `pipes.turbot.com.sptt` - Stores the [Turbot Pipes](https://pipes.turbot.com) token. Deleting the file would require you to run steampipe login again. - `connection.json` - Stores the connection config information. This file gets re-generated everytime RefreshConnections is called. diff --git a/pkg/cloud/api.go b/pkg/cloud/api.go index 4d5f55ccc..27fffecbb 100644 --- a/pkg/cloud/api.go +++ b/pkg/cloud/api.go @@ -14,7 +14,7 @@ func newSteampipeCloudClient(token string) *steampipecloud.APIClient { configuration := steampipecloud.NewConfiguration() configuration.Host = viper.GetString(constants.ArgCloudHost) - // Add your Steampipe Cloud user token as an auth header + // Add your Turbot Pipes user token as an auth header if token != "" { configuration.AddDefaultHeader("Authorization", fmt.Sprintf("Bearer %s", token)) } diff --git a/pkg/cloud/doc.go b/pkg/cloud/doc.go index 5024005db..caa4690c1 100644 --- a/pkg/cloud/doc.go +++ b/pkg/cloud/doc.go @@ -1,2 +1,2 @@ -// Package cloud contains logic to support connecting to a steampipe cloud database +// Package cloud contains logic to support connecting to a Turbot Pipes database package cloud diff --git a/pkg/cloud/login.go b/pkg/cloud/login.go index a54d9b818..baec5453f 100644 --- a/pkg/cloud/login.go +++ b/pkg/cloud/login.go @@ -61,13 +61,16 @@ func GetLoginToken(ctx context.Context, id, code string) (string, error) { return tokenResp.GetToken(), nil } -// SaveToken writes the token to ~/.steampipe/internal/{cloud-host}.sptt +// SaveToken writes the token to ~/.steampipe/internal/{cloud-host}.tptt func SaveToken(token string) error { tokenPath := tokenFilePath(viper.GetString(constants.ArgCloudHost)) return sperr.Wrap(os.WriteFile(tokenPath, []byte(token), 0600)) } func LoadToken() (string, error) { + if err := migrateDefaultTokenFile(); err != nil { + log.Println("[TRACE] ERROR during migrating token file", err) + } tokenPath := tokenFilePath(viper.GetString(constants.ArgCloudHost)) if !filehelpers.FileExists(tokenPath) { return "", nil @@ -79,6 +82,30 @@ func LoadToken() (string, error) { return string(tokenBytes), nil } +// migrateDefaultTokenFile migrates the cloud.steampipe.io.sptt token file +// to the pipes.turbot.com.tptt token file +func migrateDefaultTokenFile() error { + defaultTokenPath := tokenFilePath(constants.DefaultCloudHost) + defaultLegacyTokenPath := legacyTokenFilePath(constants.LegacyDefaultCloudHost) + + if filehelpers.FileExists(defaultTokenPath) { + // try removing the old legacy file - no worries if os.Remove fails + os.Remove(defaultLegacyTokenPath) + // we have the new token file + return nil + } + + // the default file does not exist + // check if the legacy one exists + if filehelpers.FileExists(defaultLegacyTokenPath) { + // move the legacy to the new + return utils.MoveFile(defaultLegacyTokenPath, defaultTokenPath) + } + + // none exists - nothing to do + return nil +} + func GetUserName(ctx context.Context, token string) (string, error) { client := newSteampipeCloudClient(token) actor, _, err := client.Actors.Get(ctx).Execute() @@ -99,3 +126,8 @@ func tokenFilePath(cloudHost string) string { tokenPath := path.Join(filepaths.EnsureInternalDir(), fmt.Sprintf("%s%s", cloudHost, constants.TokenExtension)) return tokenPath } + +func legacyTokenFilePath(cloudHost string) string { + tokenPath := path.Join(filepaths.EnsureInternalDir(), fmt.Sprintf("%s%s", cloudHost, constants.LegacyTokenExtension)) + return tokenPath +} diff --git a/pkg/cmdconfig/viper.go b/pkg/cmdconfig/viper.go index 9409deb89..a5f0ad275 100644 --- a/pkg/cmdconfig/viper.go +++ b/pkg/cmdconfig/viper.go @@ -135,14 +135,21 @@ func SetDefaultsFromEnv() { // a map of known environment variables to map to viper keys envMappings := map[string]envMapping{ - constants.EnvInstallDir: {[]string{constants.ArgInstallDir}, String}, - constants.EnvWorkspaceChDir: {[]string{constants.ArgModLocation}, String}, - constants.EnvModLocation: {[]string{constants.ArgModLocation}, String}, - constants.EnvIntrospection: {[]string{constants.ArgIntrospection}, String}, - constants.EnvTelemetry: {[]string{constants.ArgTelemetry}, String}, - constants.EnvUpdateCheck: {[]string{constants.ArgUpdateCheck}, Bool}, - constants.EnvCloudHost: {[]string{constants.ArgCloudHost}, String}, - constants.EnvCloudToken: {[]string{constants.ArgCloudToken}, String}, + constants.EnvInstallDir: {[]string{constants.ArgInstallDir}, String}, + constants.EnvWorkspaceChDir: {[]string{constants.ArgModLocation}, String}, + constants.EnvModLocation: {[]string{constants.ArgModLocation}, String}, + constants.EnvIntrospection: {[]string{constants.ArgIntrospection}, String}, + constants.EnvTelemetry: {[]string{constants.ArgTelemetry}, String}, + constants.EnvUpdateCheck: {[]string{constants.ArgUpdateCheck}, Bool}, + // PIPES_HOST needs to be defined before STEAMPIPE_CLOUD_HOST, + // so that if STEAMPIPE_CLOUD_HOST is defined, it can override PIPES_HOST + constants.EnvPipesHost: {[]string{constants.ArgCloudHost}, String}, + constants.EnvCloudHost: {[]string{constants.ArgCloudHost}, String}, + // PIPES_TOKEN needs to be defined before STEAMPIPE_CLOUD_TOKEN, + // so that if STEAMPIPE_CLOUD_TOKEN is defined, it can override PIPES_TOKEN + constants.EnvPipesToken: {[]string{constants.ArgCloudToken}, String}, + constants.EnvCloudToken: {[]string{constants.ArgCloudToken}, String}, + // constants.EnvSnapshotLocation: {[]string{constants.ArgSnapshotLocation}, String}, constants.EnvWorkspaceDatabase: {[]string{constants.ArgWorkspaceDatabase}, String}, constants.EnvServicePassword: {[]string{constants.ArgServicePassword}, String}, diff --git a/pkg/constants/default_workspaces.go b/pkg/constants/default_workspaces.go index 4c6009dde..6bf000347 100644 --- a/pkg/constants/default_workspaces.go +++ b/pkg/constants/default_workspaces.go @@ -9,7 +9,7 @@ const DefaultWorkspaceContent = ` # # workspace "all_options" { -# cloud_host = "cloud.steampipe.io" +# cloud_host = "pipes.turbot.com" # cloud_token = "spt_999faketoken99999999_111faketoken1111111111111" # install_dir = "~/steampipe2" # mod_location = "~/src/steampipe-mod-aws-insights" diff --git a/pkg/constants/env.go b/pkg/constants/env.go index c7d7d0210..2c6d9f4c6 100644 --- a/pkg/constants/env.go +++ b/pkg/constants/env.go @@ -16,6 +16,9 @@ const ( EnvCloudHost = "STEAMPIPE_CLOUD_HOST" EnvCloudToken = "STEAMPIPE_CLOUD_TOKEN" + EnvPipesHost = "PIPES_HOST" + EnvPipesToken = "PIPES_TOKEN" + EnvDisplayWidth = "STEAMPIPE_DISPLAY_WIDTH" EnvCacheEnabled = "STEAMPIPE_CACHE" EnvCacheTTL = "STEAMPIPE_CACHE_TTL" diff --git a/pkg/constants/extensions.go b/pkg/constants/extensions.go index ba0e8002d..e8fa87462 100644 --- a/pkg/constants/extensions.go +++ b/pkg/constants/extensions.go @@ -14,7 +14,8 @@ const ( CsvExtension = ".csv" TextExtension = ".txt" SnapshotExtension = ".sps" - TokenExtension = ".sptt" + TokenExtension = ".tptt" + LegacyTokenExtension = ".sptt" ) var YamlExtensions = []string{".yml", ".yaml"} diff --git a/pkg/constants/workspace_profile.go b/pkg/constants/workspace_profile.go index f49bdd43c..a36855e44 100644 --- a/pkg/constants/workspace_profile.go +++ b/pkg/constants/workspace_profile.go @@ -1,6 +1,7 @@ package constants const ( - DefaultCloudHost = "cloud.steampipe.io" + DefaultCloudHost = "pipes.turbot.com" + LegacyDefaultCloudHost = "cloud.steampipe.io" DefaultWorkspaceDatabase = "local" ) diff --git a/pkg/error_helpers/errors.go b/pkg/error_helpers/errors.go index 6ee19405c..0442691d6 100644 --- a/pkg/error_helpers/errors.go +++ b/pkg/error_helpers/errors.go @@ -7,6 +7,6 @@ import ( "github.com/turbot/steampipe/pkg/constants" ) -var MissingCloudTokenError = fmt.Errorf("Not authenticated for Steampipe Cloud.\nPlease run %s or setup a token.", constants.Bold("steampipe login")) +var MissingCloudTokenError = fmt.Errorf("Not authenticated for Turbot Pipes.\nPlease run %s or setup a token.", constants.Bold("steampipe login")) var InvalidCloudTokenError = fmt.Errorf("Invalid token.\nPlease run %s or setup a token.", constants.Bold("steampipe login")) var InvalidStateError = errors.New("invalid state") diff --git a/pkg/steampipeconfig/workspace_profile_loader.go b/pkg/steampipeconfig/workspace_profile_loader.go index 4bb798a59..f26557956 100644 --- a/pkg/steampipeconfig/workspace_profile_loader.go +++ b/pkg/steampipeconfig/workspace_profile_loader.go @@ -105,7 +105,7 @@ it will be interpreted as an implicit workspace. Implicit workspaces, as the name suggests, do not need to be specified in the workspaces.spc file. -Instead they will be assumed to refer to a Steampipe Cloud workspace, +Instead they will be assumed to refer to a Turbot Pipes workspace, which will be used as both the database and snapshot location. Essentially, --workspace acme/dev is equivalent to: diff --git a/tests/acceptance/test_data/templates/expected_plugin_help_output.txt b/tests/acceptance/test_data/templates/expected_plugin_help_output.txt index 930d0905e..541237d2f 100644 --- a/tests/acceptance/test_data/templates/expected_plugin_help_output.txt +++ b/tests/acceptance/test_data/templates/expected_plugin_help_output.txt @@ -30,10 +30,10 @@ Flags: -h, --help Help for plugin Global Flags: - --cloud-host string Steampipe Cloud host (default "cloud.steampipe.io") - --cloud-token string Steampipe Cloud authentication token + --cloud-host string Turbot Pipes host (default "pipes.turbot.com") + --cloud-token string Turbot Pipes authentication token --install-dir string Path to the Config Directory (default "~/.steampipe") --workspace string The workspace profile to use (default "default") - --workspace-database string Steampipe Cloud workspace database (default "local") + --workspace-database string Turbot Pipes workspace database (default "local") Use "steampipe plugin [command] --help" for more information about a command. diff --git a/tests/acceptance/test_data/templates/expected_service_help_output.txt b/tests/acceptance/test_data/templates/expected_service_help_output.txt index ad65c3edb..4d6ad3890 100644 --- a/tests/acceptance/test_data/templates/expected_service_help_output.txt +++ b/tests/acceptance/test_data/templates/expected_service_help_output.txt @@ -16,10 +16,10 @@ Flags: -h, --help Help for service Global Flags: - --cloud-host string Steampipe Cloud host (default "cloud.steampipe.io") - --cloud-token string Steampipe Cloud authentication token + --cloud-host string Turbot Pipes host (default "pipes.turbot.com") + --cloud-token string Turbot Pipes authentication token --install-dir string Path to the Config Directory (default "~/.steampipe") --workspace string The workspace profile to use (default "default") - --workspace-database string Steampipe Cloud workspace database (default "local") + --workspace-database string Turbot Pipes workspace database (default "local") Use "steampipe service [command] --help" for more information about a command. diff --git a/tests/acceptance/test_files/cloud.bats b/tests/acceptance/test_files/cloud.bats index 66955ccdf..02e31e978 100644 --- a/tests/acceptance/test_files/cloud.bats +++ b/tests/acceptance/test_files/cloud.bats @@ -33,7 +33,7 @@ load "$LIB_BATS_SUPPORT/load.bash" @test "connect to cloud workspace - passing the cloud-host arg, the cloud-token arg and the workspace name to workspace-database arg" { # run steampipe query and fetch an account from the cloud workspace - run steampipe query "select account_aliases from all_aws.aws_account where account_id='632902152528'" --cloud-host "cloud.steampipe.io" --cloud-token $SPIPETOOLS_TOKEN --workspace-database spipetools/toolstest --output json + run steampipe query "select account_aliases from all_aws.aws_account where account_id='632902152528'" --cloud-host "pipes.turbot.com" --cloud-token $SPIPETOOLS_TOKEN --workspace-database spipetools/toolstest --output json echo $output # fetch the value of account_alias to compare @@ -50,7 +50,7 @@ load "$LIB_BATS_SUPPORT/load.bash" echo $output # check the error message - assert_output --partial 'Error: Not authenticated for Steampipe Cloud.' + assert_output --partial 'Error: Not authenticated for Turbot Pipes.' } @test "install a large mod, query and check if time taken is less than 20s" {