jprdonnelly/steampipe
1
0
Fork 0
mirror of https://github.com/turbot/steampipe.git synced 2026-05-09 21:00:27 -04:00
Code Issues Packages Projects Releases Wiki Activity

Rename CloudMetadata to PipesMetadata, move to pipe-fittings pipes package

Browse Source
This commit is contained in:
kaidaguerre
2024-10-16 15:48:14 +01:00
committed by Puskar Basu
parent 3b7f6b2fdc
commit 8ade67d8d7
16 changed files with 35 additions and 309 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/login.go
View File

@@ -10,8 +10,8 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/viper"
pconstants "github.com/turbot/pipe-fittings/constants"
"github.com/turbot/pipe-fittings/pipes"
"github.com/turbot/steampipe-plugin-sdk/v5/sperr"
"github.com/turbot/steampipe/pkg/cloud"
"github.com/turbot/steampipe/pkg/cmdconfig"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/error_helpers"
@@ -40,7 +40,7 @@ func runLoginCmd(cmd *cobra.Command, _ []string) {
log.Printf("[TRACE] login, pipes host %s", viper.Get(pconstants.ArgPipesHost))
log.Printf("[TRACE] opening login web page")
// start login flow - this will open a web page prompting user to login, and will give the user a code to enter
var id, err = cloud.WebLogin(ctx)
var id, err = pipes.WebLogin(ctx)
if err != nil {
error_helpers.ShowError(ctx, err)
exitCode = constants.ExitCodeLoginCloudConnectionFailed
@@ -55,7 +55,7 @@ func runLoginCmd(cmd *cobra.Command, _ []string) {
}
// save token
err = cloud.SaveToken(token)
err = pipes.SaveToken(token)
if err != nil {
error_helpers.ShowError(ctx, err)
exitCode = constants.ExitCodeLoginCloudConnectionFailed
@@ -77,7 +77,7 @@ func getToken(ctx context.Context, id string) (loginToken string, err error) {
if code != "" {
log.Printf("[TRACE] get login token")
// use this code to get a login token and store it
loginToken, err = cloud.GetLoginToken(ctx, id, code)
loginToken, err = pipes.GetLoginToken(ctx, id, code)
if err == nil {
return loginToken, nil
}
@@ -101,7 +101,7 @@ func getToken(ctx context.Context, id string) (loginToken string, err error) {
}
func displayLoginMessage(ctx context.Context, token string) {
userName, err := cloud.GetUserName(ctx, token)
userName, err := pipes.GetUserName(ctx, token)
error_helpers.FailOnError(sperr.WrapWithMessage(err, "failed to read user name"))
fmt.Println()

4
go.mod
View File

@@ -6,8 +6,8 @@ replace (
github.com/c-bata/go-prompt => github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50
github.com/docker/distribution => github.com/distribution/distribution v2.7.1+incompatible
github.com/docker/docker => github.com/moby/moby v20.10.17+incompatible
// github.com/turbot/pipe-fittings => ../pipe-fittings
// github.com/turbot/steampipe-plugin-sdk/v5 => ../steampipe-plugin-sdk
github.com/turbot/pipe-fittings => ../pipe-fittings
)

33
pkg/cloud/api.go
View File

@@ -1,33 +0,0 @@
package cloud
import (
"fmt"
"github.com/turbot/pipe-fittings/constants"
"net/url"
"github.com/spf13/viper"
steampipecloud "github.com/turbot/steampipe-cloud-sdk-go"
)
func newSteampipeCloudClient(token string) *steampipecloud.APIClient {
// Create a default configuration
configuration := steampipecloud.NewConfiguration()
configuration.Host = viper.GetString(constants.ArgPipesHost)
// Add your Turbot Pipes user token as an auth header
if token != "" {
configuration.AddDefaultHeader("Authorization", fmt.Sprintf("Bearer %s", token))
}
// Create a client
return steampipecloud.NewAPIClient(configuration)
}
func getLoginTokenConfirmUIUrl() string {
confirmUrl := url.URL{
Scheme: "https",
Host: viper.GetString(constants.ArgPipesHost),
Path: "/login/token",
}
return confirmUrl.String()
}

2
pkg/cloud/doc.go
View File

@@ -1,2 +0,0 @@
// Package cloud contains logic to support connecting to a Turbot Pipes database
package cloud

136
pkg/cloud/login.go
View File

@@ -1,136 +0,0 @@
package cloud
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"os"
"path"
"github.com/spf13/viper"
filehelpers "github.com/turbot/go-kit/files"
pconstants "github.com/turbot/pipe-fittings/constants"
"github.com/turbot/pipe-fittings/utils"
steampipecloud "github.com/turbot/steampipe-cloud-sdk-go"
"github.com/turbot/steampipe-plugin-sdk/v5/sperr"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/filepaths"
)
// WebLogin POSTs to ${envBaseUrl}/api/latest/login/token to retrieve a login is
// it then opens the login webpage and returns th eid
func WebLogin(ctx context.Context) (string, error) {
client := newSteampipeCloudClient(viper.GetString(pconstants.ArgPipesToken))
tempTokenReq, _, err := client.Auth.LoginTokenCreate(ctx).Execute()
if err != nil {
return "", sperr.WrapWithMessage(err, "failed to create login token")
}
id := tempTokenReq.Id
// add in id query string
browserUrl := fmt.Sprintf("%s?r=%s", getLoginTokenConfirmUIUrl(), id)
fmt.Printf("\nVerify login at %s\n", browserUrl)
if err = utils.OpenBrowser(browserUrl); err != nil {
log.Println("[INFO] failed to open login web page")
}
return id, nil
}
// GetLoginToken uses the login id and code and retrieves an authentication token
func GetLoginToken(ctx context.Context, id, code string) (string, error) {
client := newSteampipeCloudClient("")
tokenResp, _, err := client.Auth.LoginTokenGet(ctx, id).Code(code).Execute()
if err != nil {
var apiErr steampipecloud.GenericOpenAPIError
if errors.As(err, &apiErr) {
var body = map[string]any{}
if err := json.Unmarshal(apiErr.Body(), &body); err == nil {
return "", sperr.New("%s", body["detail"])
}
}
return "", sperr.Wrap(err)
}
if tokenResp.GetToken() == "" && tokenResp.GetState() == "pending" {
return "", sperr.New("login request has not been confirmed - select 'Verify' and enter the verification code")
}
return tokenResp.GetToken(), nil
}
// SaveToken writes the token to ~/.steampipe/internal/{cloud-host}.tptt
func SaveToken(token string) error {
tokenPath := tokenFilePath(viper.GetString(pconstants.ArgPipesHost))
return sperr.Wrap(os.WriteFile(tokenPath, []byte(token), 0600))
}
func LoadToken() (string, error) {
if err := migrateDefaultTokenFile(); err != nil {
log.Println("[TRACE] ERROR during migrating token file", err)
}
tokenPath := tokenFilePath(viper.GetString(pconstants.ArgPipesHost))
if !filehelpers.FileExists(tokenPath) {
return "", nil
}
tokenBytes, err := os.ReadFile(tokenPath)
if err != nil {
return "", sperr.WrapWithMessage(err, "failed to load token file '%s'", tokenPath)
}
return string(tokenBytes), nil
}
// migrateDefaultTokenFile migrates the cloud.steampipe.io.sptt token file
// to the pipes.turbot.com.tptt token file
// it also migrates the token file from the ~/.steampipe/internal directory to the ~/.pipes/internal directory
func migrateDefaultTokenFile() error {
defaultTokenPath := tokenFilePath(constants.DefaultPipesHost)
defaultLegacyTokenPaths := legacyTokenFilePaths()
tokenExists := filehelpers.FileExists(defaultTokenPath)
for _, legacyPath := range defaultLegacyTokenPaths {
if filehelpers.FileExists(legacyPath) {
if tokenExists {
// try removing the old legacy file - no worries if os.Remove fails
_ = os.Remove(legacyPath)
} else {
if err := utils.MoveFile(legacyPath, defaultTokenPath); err != nil {
return err
}
// set token exists flag so any other legacy files are removed (we do not expect any more)
tokenExists = true
}
}
}
return nil
}
func GetUserName(ctx context.Context, token string) (string, error) {
client := newSteampipeCloudClient(token)
actor, _, err := client.Actors.Get(ctx).Execute()
if err != nil {
return "", sperr.Wrap(err)
}
return getActorName(actor), nil
}
func getActorName(actor steampipecloud.User) string {
if name, ok := actor.GetDisplayNameOk(); ok {
return *name
}
return actor.Handle
}
func tokenFilePath(pipesHost string) string {
tokenPath := path.Join(filepaths.EnsurePipesInternalDir(), fmt.Sprintf("%s%s", pipesHost, constants.TokenExtension))
return tokenPath
}
func legacyTokenFilePaths() []string {
return []string{path.Join(filepaths.EnsureInternalDir(), fmt.Sprintf("%s%s", constants.LegacyDefaultPipesHost, constants.LegacyTokenExtension)),
path.Join(filepaths.EnsureInternalDir(), fmt.Sprintf("%s%s", constants.DefaultPipesHost, constants.TokenExtension))}
}

4
pkg/cmdconfig/app_specific.go
View File

@@ -1,6 +1,7 @@
package cmdconfig
import (
pfilepaths "github.com/turbot/pipe-fittings/filepaths"
"os"
"github.com/turbot/go-kit/files"
@@ -28,6 +29,9 @@ func SetAppSpecificConstants() {
defaultInstallDir, err := files.Tildefy("~/.steampipe")
error_helpers.FailOnError(err)
app_specific.DefaultInstallDir = defaultInstallDir
defaultPipesInstallDir, err := files.Tildefy("~/.pipes")
pfilepaths.DefaultPipesInstallDir = defaultPipesInstallDir
error_helpers.FailOnError(err)
// check whether install-dir env has been set - if so, respect it
if envInstallDir, ok := os.LookupEnv(app_specific.EnvInstallDir); ok {

7
pkg/cmdconfig/cmd_hooks.go
View File

@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"fmt"
pfilepaths "github.com/turbot/pipe-fittings/filepaths"
"io"
"log"
"os"
@@ -23,13 +24,13 @@ import (
pconstants "github.com/turbot/pipe-fittings/constants"
perror_helpers "github.com/turbot/pipe-fittings/error_helpers"
"github.com/turbot/pipe-fittings/parse"
"github.com/turbot/pipe-fittings/pipes"
"github.com/turbot/pipe-fittings/utils"
"github.com/turbot/pipe-fittings/versionfile"
"github.com/turbot/pipe-fittings/workspace_profile"
sdklogging "github.com/turbot/steampipe-plugin-sdk/v5/logging"
"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
"github.com/turbot/steampipe-plugin-sdk/v5/sperr"
"github.com/turbot/steampipe/pkg/cloud"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/constants/runtime"
"github.com/turbot/steampipe/pkg/error_helpers"
@@ -269,7 +270,7 @@ func setCloudTokenDefault(loader *parse.WorkspaceProfileLoader[*workspace_profil
*/
// set viper defaults in order of increasing precedence
// 1) saved cloud token
savedToken, err := cloud.LoadToken()
savedToken, err := pipes.LoadToken()
if err != nil {
return err
}
@@ -403,7 +404,7 @@ func ensureInstallDir() {
// store as app_specific.InstallDir and PipesInstallDir
app_specific.InstallDir = installDir
filepaths.PipesInstallDir = pipesInstallDir
pfilepaths.PipesInstallDir = pipesInstallDir
}
// displayDeprecationWarnings shows the deprecated warnings in a formatted way

10
pkg/cmdconfig/validate.go
View File

@@ -7,8 +7,8 @@ import (
"github.com/spf13/viper"
filehelpers "github.com/turbot/go-kit/files"
"github.com/turbot/pipe-fittings/cloud"
pconstants "github.com/turbot/pipe-fittings/constants"
"github.com/turbot/pipe-fittings/pipes"
"github.com/turbot/pipe-fittings/steampipeconfig"
"github.com/turbot/steampipe/pkg/error_helpers"
)
@@ -35,8 +35,8 @@ func ValidateSnapshotArgs(ctx context.Context) error {
}
// if workspace-database or snapshot-location are a cloud workspace handle, cloud token must be set
requireCloudToken := steampipeconfig.IsCloudWorkspaceIdentifier(viper.GetString(pconstants.ArgWorkspaceDatabase)) ||
steampipeconfig.IsCloudWorkspaceIdentifier(viper.GetString(pconstants.ArgSnapshotLocation))
requireCloudToken := steampipeconfig.IsPipesWorkspaceIdentifier(viper.GetString(pconstants.ArgWorkspaceDatabase)) ||
steampipeconfig.IsPipesWorkspaceIdentifier(viper.GetString(pconstants.ArgSnapshotLocation))
// verify cloud token and workspace has been set
if requireCloudToken && token == "" {
@@ -64,7 +64,7 @@ func validateSnapshotLocation(ctx context.Context, cloudToken string) error {
// if it is NOT a workspace handle, assume it is a local file location:
// tildefy it and ensure it exists
if !steampipeconfig.IsCloudWorkspaceIdentifier(snapshotLocation) {
if !steampipeconfig.IsPipesWorkspaceIdentifier(snapshotLocation) {
var err error
snapshotLocation, err = filehelpers.Tildefy(snapshotLocation)
if err != nil {
@@ -82,7 +82,7 @@ func validateSnapshotLocation(ctx context.Context, cloudToken string) error {
}
func setSnapshotLocationFromDefaultWorkspace(ctx context.Context, cloudToken string) error {
workspaceHandle, err := cloud.GetUserWorkspaceHandle(ctx, cloudToken)
workspaceHandle, err := pipes.GetUserWorkspaceHandle(ctx, cloudToken)
if err != nil {
return err
}

8
pkg/cmdconfig/viper.go
View File

@@ -2,6 +2,7 @@ package cmdconfig
import (
"fmt"
pfilepaths "github.com/turbot/pipe-fittings/filepaths"
"log"
"os"
@@ -13,7 +14,6 @@ import (
"github.com/turbot/pipe-fittings/parse"
"github.com/turbot/pipe-fittings/workspace_profile"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/filepaths"
)
// Viper fetches the global viper instance
@@ -89,15 +89,11 @@ func SetDefaultsFromConfig(configMap map[string]interface{}) {
// Do not add keys here which have command line defaults - the way this is setup, this value takes
// precedence over command line default
func setBaseDefaults() error {
pipesInstallDir, err := filehelpers.Tildefy(filepaths.DefaultPipesInstallDir)
if err != nil {
return err
}
defaults := map[string]interface{}{
// global general options
pconstants.ArgTelemetry: constants.TelemetryInfo,
pconstants.ArgUpdateCheck: true,
pconstants.ArgPipesInstallDir: pipesInstallDir,
pconstants.ArgPipesInstallDir: pfilepaths.DefaultPipesInstallDir,
// workspace profile
pconstants.ArgAutoComplete: true,

35
pkg/filepaths/pipes_install_dir.go
View File

@@ -1,35 +0,0 @@
package filepaths
import (
"fmt"
"github.com/turbot/pipe-fittings/error_helpers"
"os"
"path/filepath"
)
// PipesInstallDir is the location of config files commen between pipelings
// this must be set by the application at startup
var PipesInstallDir = ""
func ensurePipesInstallSubDir(dirName string) string {
subDir := installPipesSubDir(dirName)
if _, err := os.Stat(subDir); os.IsNotExist(err) {
err = os.MkdirAll(subDir, 0755)
error_helpers.FailOnErrorWithMessage(err, fmt.Sprintf("could not create %s directory", dirName))
}
return subDir
}
func installPipesSubDir(dirName string) string {
if PipesInstallDir == "" {
panic(fmt.Errorf("cannot call any pipes directory functions before PipesInstallDir is set"))
}
return filepath.Join(PipesInstallDir, dirName)
}
// EnsurePipesInternalDir returns the path to the pipes internal directory (creates if missing)
func EnsurePipesInternalDir() string {
return ensurePipesInstallSubDir("internal")
}

2
pkg/filepaths/steampipe.go
View File

@@ -13,8 +13,6 @@ import (
// Constants for Config
const (
DefaultPipesInstallDir = "~/.pipes"
connectionsStateFileName = "connection.json"
versionFileName = "versions.json"
databaseRunningInfoFileName = "steampipe.json"

54
pkg/filepaths/workspace.go
View File

@@ -1,57 +1,5 @@
package filepaths
import (
"fmt"
"path"
"path/filepath"
"strings"
"github.com/turbot/steampipe/pkg/constants/runtime"
)
var ModFileNames = []string{"mod.sp"}
// mod related constants
const (
WorkspaceDataDir = ".steampipe"
WorkspaceModDir = "mods"
WorkspaceModShadowDirPrefix = ".mods."
WorkspaceConfigFileName = "workspace.spc"
WorkspaceIgnoreFile = ".steampipeignore"
DefaultVarsFileName = "steampipe.spvars"
WorkspaceLockFileName = ".mod.cache.json"
WorkspaceConfigFileName = "workspace.spc"
)
func WorkspaceModPath(workspacePath string) string {
return path.Join(workspacePath, WorkspaceDataDir, WorkspaceModDir)
}
func WorkspaceModShadowPath(workspacePath string) string {
return path.Join(workspacePath, WorkspaceDataDir, fmt.Sprintf("%s%s", WorkspaceModShadowDirPrefix, runtime.ExecutionID))
}
func IsModInstallShadowPath(dirName string) bool {
return strings.HasPrefix(dirName, WorkspaceModShadowDirPrefix)
}
func WorkspaceLockPath(workspacePath string) string {
return path.Join(workspacePath, WorkspaceLockFileName)
}
func DefaultVarsFilePath(workspacePath string) string {
return path.Join(workspacePath, DefaultVarsFileName)
}
func DefaultModFilePath(modFolder string) string {
return filepath.Join(modFolder, ModFileNames[0])
}
func ModFilePaths(modFolder string) []string {
var modFilePaths []string
for _, modFileName := range ModFileNames {
modFilePaths = append(modFilePaths, filepath.Join(modFolder, modFileName))
}
return modFilePaths
}

14
pkg/initialisation/cloud_metadata.go
View File

@@ -5,13 +5,13 @@ import (
"strings"
"github.com/spf13/viper"
"github.com/turbot/pipe-fittings/cloud"
"github.com/turbot/pipe-fittings/constants"
"github.com/turbot/pipe-fittings/pipes"
"github.com/turbot/pipe-fittings/steampipeconfig"
"github.com/turbot/steampipe/pkg/error_helpers"
)
func getCloudMetadata(ctx context.Context) (*steampipeconfig.CloudMetadata, error) {
func getPipesMetadata(ctx context.Context) (*steampipeconfig.PipesMetadata, error) {
workspaceDatabase := viper.GetString(constants.ArgWorkspaceDatabase)
if workspaceDatabase == "local" {
// local database - nothing to do here
@@ -19,7 +19,7 @@ func getCloudMetadata(ctx context.Context) (*steampipeconfig.CloudMetadata, erro
}
connectionString := workspaceDatabase
var cloudMetadata *steampipeconfig.CloudMetadata
var pipesMetadata *steampipeconfig.PipesMetadata
// so a backend was set - is it a connection string or a database name
workspaceDatabaseIsConnectionString := strings.HasPrefix(workspaceDatabase, "postgresql://") || strings.HasPrefix(workspaceDatabase, "postgres://")
@@ -32,15 +32,15 @@ func getCloudMetadata(ctx context.Context) (*steampipeconfig.CloudMetadata, erro
// so we have a database and a token - build the connection string and set it in viper
var err error
if cloudMetadata, err = cloud.GetCloudMetadata(ctx, workspaceDatabase, cloudToken); err != nil {
if pipesMetadata, err = pipes.GetPipesMetadata(ctx, workspaceDatabase, cloudToken); err != nil {
return nil, err
}
// read connection string out of cloudMetadata
connectionString = cloudMetadata.ConnectionString
// read connection string out of pipesMetadata
connectionString = pipesMetadata.ConnectionString
}
// now set the connection string in viper
viper.Set(constants.ArgConnectionString, connectionString)
return cloudMetadata, nil
return pipesMetadata, nil
}

6
pkg/initialisation/init_data.go
View File

@@ -23,7 +23,7 @@ import (
type InitData struct {
Client db_common.Client
Result *db_common.InitResult
CloudMetadata *steampipeconfig.CloudMetadata
PipesMetadata *steampipeconfig.PipesMetadata
ShutdownTelemetry func()
ExportManager *export.Manager
@@ -79,14 +79,14 @@ func (i *InitData) Init(ctx context.Context, invoker constants.Invoker, opts ...
}
// retrieve cloud metadata
cloudMetadata, err := getCloudMetadata(ctx)
pipesMetadata, err := getPipesMetadata(ctx)
if err != nil {
i.Result.Error = err
return
}
// set cloud metadata (may be nil)
i.CloudMetadata = cloudMetadata
i.PipesMetadata = pipesMetadata
// get a client
// add a message rendering function to the context - this is used for the fdw update message and

4
pkg/query/queryexecute/execute.go
View File

@@ -9,11 +9,11 @@ import (
"time"
"github.com/spf13/viper"
"github.com/turbot/pipe-fittings/cloud"
"github.com/turbot/pipe-fittings/constants"
pconstants "github.com/turbot/pipe-fittings/constants"
"github.com/turbot/pipe-fittings/contexthelpers"
"github.com/turbot/pipe-fittings/modconfig"
"github.com/turbot/pipe-fittings/pipes"
"github.com/turbot/pipe-fittings/querydisplay"
"github.com/turbot/pipe-fittings/queryresult"
"github.com/turbot/pipe-fittings/steampipeconfig"
@@ -218,7 +218,7 @@ func publishSnapshotIfNeeded(ctx context.Context, snapshot *steampipeconfig.Stea
return nil
}
message, err := cloud.PublishSnapshot(ctx, snapshot, shouldShare)
message, err := pipes.PublishSnapshot(ctx, snapshot, shouldShare)
if err != nil {
// reword "402 Payment Required" error
return handlePublishSnapshotError(err)

15
tests/acceptance/test_files/cloud.bats
View File

@@ -53,21 +53,6 @@ load "$LIB_BATS_SUPPORT/load.bash"
assert_output --partial 'Error: Not authenticated for Turbot Pipes.'
}
@test "connect to cloud workspace - passing the workspace name to workspace-database arg (unsetting ENV - the token should get picked from tptt file)" {
# write the pipes.turbot.com.tptt file in internal
# write the token to the file
file_name=$STEAMPIPE_INSTALL_DIR/internal/pipes.turbot.com.tptt
echo -ne $SPIPETOOLS_TOKEN > $file_name
cat $file_name
# this step will create snapshots in the workspace - but that's ok
# workspaces expire snapshots anyway
run steampipe query "select 1" --share
echo $output
assert_success
}
function teardown_file() {
# list running processes
ps -ef | grep steampipe