jprdonnelly/steampipe
1
0
Fork 0
mirror of https://github.com/turbot/steampipe.git synced 2025-12-19 18:12:43 -05:00
Code Issues Packages Projects Releases Wiki Activity

steampipe compiles

Browse Source
This commit is contained in:
kai
2024-09-03 17:31:01 +01:00
committed by Puskar Basu
parent 14e68aa0dd
commit ef21f1452e
280 changed files with 1039 additions and 23885 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
cmd/list.go
View File

@@ -1,18 +1,12 @@
package cmd
import (
"context"
"fmt"
"sort"
"github.com/spf13/cobra"
"github.com/turbot/steampipe/pkg/cmdconfig"
"github.com/turbot/steampipe/pkg/display"
"github.com/turbot/steampipe/pkg/error_helpers"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
"github.com/turbot/steampipe/pkg/workspace"
)
// TODO #kai can we just remove this
type listSubCmdOptions struct {
parentCmd *cobra.Command
}
@@ -41,95 +35,15 @@ func getRunListSubCmd(opts listSubCmdOptions) func(cmd *cobra.Command, args []st
}
return func(cmd *cobra.Command, _ []string) {
ctx := cmd.Context()
w, inputVariables, errAndWarnings := workspace.LoadWorkspaceVars(ctx)
error_helpers.FailOnError(errAndWarnings.GetError())
errAndWarnings = w.LoadWorkspaceMod(ctx, inputVariables)
error_helpers.FailOnError(errAndWarnings.GetError())
modResources, depResources, err := listResourcesInMod(ctx, w.Mod, cmd)
error_helpers.FailOnErrorWithMessage(err, "could not list resources")
if len(modResources)+len(depResources) == 0 {
fmt.Println("No resources available to execute.")
}
sortResources(modResources)
sortResources(depResources)
headers, rows := getOutputDataTable(modResources, depResources)
display.ShowWrappedTable(headers, rows, &display.ShowWrappedTableOptions{
AutoMerge: false,
HideEmptyColumns: true,
Truncate: true,
})
// TODO #v1 list query files? or deprecate list commena
//ctx := cmd.Context()
//
//headers, rows := getOutputDataTable(modResources, depResources)
//
//display.ShowWrappedTable(headers, rows, &display.ShowWrappedTableOptions{
// AutoMerge: false,
// HideEmptyColumns: true,
// Truncate: true,
//})
}
}
func listResourcesInMod(ctx context.Context, mod *modconfig.Mod, cmd *cobra.Command) (modResources, depResources []modconfig.ModTreeItem, err error) {
resourceTypesToDisplay := getResourceTypesToDisplay(cmd)
err = mod.WalkResources(func(item modconfig.HclResource) (bool, error) {
if ctx.Err() != nil {
return false, ctx.Err()
}
// if we are not showing this resource type, return
if !resourceTypesToDisplay[item.BlockType()] {
return true, nil
}
m := item.(modconfig.ModTreeItem)
itemMod := m.GetMod()
if m.GetParents()[0] == itemMod {
// add to the appropriate array
if itemMod.Name() == mod.Name() {
modResources = append(modResources, m)
} else {
depResources = append(depResources, m)
}
}
return true, nil
})
return modResources, depResources, err
}
func sortResources(items []modconfig.ModTreeItem) {
sort.SliceStable(items, func(i, j int) bool {
return items[i].Name() < items[j].Name()
})
}
func getOutputDataTable(modResources, depResources []modconfig.ModTreeItem) ([]string, [][]string) {
rows := make([][]string, len(modResources)+len(depResources))
for i, modItem := range modResources {
rows[i] = []string{modItem.GetUnqualifiedName(), modItem.GetTitle()}
}
offset := len(modResources)
for i, modItem := range depResources {
// use fully qualified name for dependency resources
rows[i+offset] = []string{modItem.Name(), modItem.GetTitle()}
}
return []string{"Name", "Title"}, rows
}
func getResourceTypesToDisplay(cmd *cobra.Command) map[string]bool {
parent := cmd.Parent().Name()
cmdToTypeMapping := map[string][]string{
"check": {"benchmark", "control"},
"dashboard": {"dashboard", "benchmark"},
"query": {"query"},
}
resourceTypesToDisplay, found := cmdToTypeMapping[parent]
if !found {
panic(fmt.Sprintf("could not find resource type lookup list for '%s'", parent))
}
// add resource types to a map for cheap lookup
res := map[string]bool{}
for _, t := range resourceTypesToDisplay {
res[t] = true
}
return res
}

322
cmd/query.go
View File

@@ -3,34 +3,25 @@ package cmd
import (
"bufio"
"context"
"encoding/json"
"fmt"
"os"
"path"
"strings"
"github.com/hashicorp/hcl/v2"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/thediveo/enumflag/v2"
"github.com/turbot/go-kit/helpers"
"github.com/turbot/pipe-fittings/utils"
"github.com/turbot/steampipe-plugin-sdk/v5/sperr"
"github.com/turbot/steampipe/pkg/cloud"
"github.com/turbot/steampipe/pkg/cmdconfig"
"github.com/turbot/steampipe/pkg/connection_sync"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/contexthelpers"
"github.com/turbot/steampipe/pkg/dashboard/dashboardtypes"
"github.com/turbot/steampipe/pkg/display"
"github.com/turbot/steampipe/pkg/error_helpers"
"github.com/turbot/steampipe/pkg/query"
"github.com/turbot/steampipe/pkg/query/queryexecute"
"github.com/turbot/steampipe/pkg/query/queryresult"
"github.com/turbot/steampipe/pkg/snapshot"
"github.com/turbot/steampipe/pkg/statushooks"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
"github.com/turbot/steampipe/pkg/workspace"
)
// variable used to assign the timing mode flag
@@ -59,21 +50,6 @@ Examples:
# Run a specific query directly
steampipe query "select * from cloud"`,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
ctx := cmd.Context()
w, err := workspace.LoadResourceNames(ctx, viper.GetString(constants.ArgModLocation))
if err != nil {
return []string{}, cobra.ShellCompDirectiveError
}
namedQueries := []string{}
for _, name := range w.GetSortedNamedQueryNames() {
if strings.HasPrefix(name, toComplete) {
namedQueries = append(namedQueries, name)
}
}
return namedQueries, cobra.ShellCompDirectiveNoFileComp
},
}
// Notes:
@@ -93,7 +69,8 @@ Examples:
constants.ArgTiming,
fmt.Sprintf("Display query timing; one of: %s", strings.Join(constants.FlagValues(constants.QueryTimingModeIds), ", ")),
cmdconfig.FlagOptions.NoOptDefVal(constants.ArgOn)).
AddBoolFlag(constants.ArgWatch, true, "Watch SQL files in the current workspace (works only in interactive mode)").
// TODO #breakingchange
//AddBoolFlag(constants.ArgWatch, true, "Watch SQL files in the current workspace (works only in interactive mode)").
AddStringSliceFlag(constants.ArgSearchPath, nil, "Set a custom search_path for the steampipe user for a query session (comma-separated)").
AddStringSliceFlag(constants.ArgSearchPathPrefix, nil, "Set a prefix to the current search path for a query session (comma-separated)").
AddStringSliceFlag(constants.ArgVarFile, nil, "Specify a file containing variable values").
@@ -234,129 +211,133 @@ func executeSnapshotQuery(initData *query.InitData, ctx context.Context) int {
}
}
for _, resolvedQuery := range initData.Queries {
// if a manual query is being run (i.e. not a named query), convert into a query and add to workspace
// this is to allow us to use existing dashboard execution code
queryProvider, existingResource := ensureSnapshotQueryResource(resolvedQuery.Name, resolvedQuery, initData.Workspace)
// we need to pass the embedded initData to GenerateSnapshot
baseInitData := &initData.InitData
// so a dashboard name was specified - just call GenerateSnapshot
snap, err := snapshot.GenerateSnapshot(ctx, queryProvider.Name(), baseInitData, nil)
if err != nil {
exitCode = constants.ExitCodeSnapshotCreationFailed
error_helpers.FailOnError(err)
}
// set the filename root for the snapshot (in case needed)
if !existingResource {
snap.FileNameRoot = "query"
}
// display the result
switch viper.GetString(constants.ArgOutput) {
case constants.OutputFormatNone:
// do nothing
case constants.OutputFormatSnapshot, constants.OutputFormatSnapshotShort:
// if the format is snapshot, just dump it out
jsonOutput, err := json.MarshalIndent(snap, "", " ")
if err != nil {
error_helpers.FailOnErrorWithMessage(err, "failed to display result as snapshot")
}
fmt.Println(string(jsonOutput))
default:
// otherwise convert the snapshot into a query result
result, err := snapshotToQueryResult(snap)
error_helpers.FailOnErrorWithMessage(err, "failed to display result as snapshot")
display.ShowOutput(ctx, result, display.WithTimingDisabled())
}
// share the snapshot if necessary
err = publishSnapshotIfNeeded(ctx, snap)
if err != nil {
exitCode = constants.ExitCodeSnapshotUploadFailed
error_helpers.FailOnErrorWithMessage(err, fmt.Sprintf("failed to publish snapshot to %s", viper.GetString(constants.ArgSnapshotLocation)))
}
// export the result if necessary
exportArgs := viper.GetStringSlice(constants.ArgExport)
exportMsg, err := initData.ExportManager.DoExport(ctx, snap.FileNameRoot, snap, exportArgs)
if err != nil {
exitCode = constants.ExitCodeSnapshotCreationFailed
error_helpers.FailOnErrorWithMessage(err, "failed to export snapshot")
}
// print the location where the file is exported
if len(exportMsg) > 0 && viper.GetBool(constants.ArgProgress) {
fmt.Printf("\n")
fmt.Println(strings.Join(exportMsg, "\n"))
fmt.Printf("\n")
}
}
return 0
// TODO fix me
//
//for _, resolvedQuery := range initData.Queries {
// // if a manual query is being run (i.e. not a named query), convert into a query and add to workspace
// // this is to allow us to use existing dashboard execution code
// queryProvider, existingResource := ensureSnapshotQueryResource(resolvedQuery.Name, resolvedQuery, initData.Workspace)
//
// // we need to pass the embedded initData to GenerateSnapshot
// baseInitData := &initData.InitData
//
// // so a dashboard name was specified - just call GenerateSnapshot
// snap, err := snapshot.GenerateSnapshot(ctx, queryProvider.Name(), baseInitData, nil)
// if err != nil {
// exitCode = constants.ExitCodeSnapshotCreationFailed
// error_helpers.FailOnError(err)
// }
//
// // set the filename root for the snapshot (in case needed)
// if !existingResource {
// snap.FileNameRoot = "query"
// }
//
// // display the result
// switch viper.GetString(constants.ArgOutput) {
// case constants.OutputFormatNone:
// // do nothing
// case constants.OutputFormatSnapshot, constants.OutputFormatSnapshotShort:
// // if the format is snapshot, just dump it out
// jsonOutput, err := json.MarshalIndent(snap, "", " ")
// if err != nil {
// error_helpers.FailOnErrorWithMessage(err, "failed to display result as snapshot")
// }
// fmt.Println(string(jsonOutput))
// default:
// // otherwise convert the snapshot into a query result
// result, err := snapshotToQueryResult(snap)
// error_helpers.FailOnErrorWithMessage(err, "failed to display result as snapshot")
// display.ShowOutput(ctx, result, display.WithTimingDisabled())
// }
//
// // share the snapshot if necessary
// err = publishSnapshotIfNeeded(ctx, snap)
// if err != nil {
// exitCode = constants.ExitCodeSnapshotUploadFailed
// error_helpers.FailOnErrorWithMessage(err, fmt.Sprintf("failed to publish snapshot to %s", viper.GetString(constants.ArgSnapshotLocation)))
// }
//
// // export the result if necessary
// exportArgs := viper.GetStringSlice(constants.ArgExport)
// exportMsg, err := initData.ExportManager.DoExport(ctx, snap.FileNameRoot, snap, exportArgs)
// if err != nil {
// exitCode = constants.ExitCodeSnapshotCreationFailed
// error_helpers.FailOnErrorWithMessage(err, "failed to export snapshot")
// }
// // print the location where the file is exported
// if len(exportMsg) > 0 && viper.GetBool(constants.ArgProgress) {
// fmt.Printf("\n")
// fmt.Println(strings.Join(exportMsg, "\n"))
// fmt.Printf("\n")
// }
//}
return 0
}
func snapshotToQueryResult(snap *dashboardtypes.SteampipeSnapshot) (*queryresult.Result, error) {
// the table of a snapshot query has a fixed name
tablePanel, ok := snap.Panels[modconfig.SnapshotQueryTableName]
if !ok {
return nil, sperr.New("dashboard does not contain table result for query")
}
chartRun := tablePanel.(*snapshot.LeafRun)
if !ok {
return nil, sperr.New("failed to read query result from snapshot")
}
// check for error
if err := chartRun.GetError(); err != nil {
return nil, error_helpers.DecodePgError(err)
}
res := queryresult.NewResult(chartRun.Data.Columns)
// start a goroutine to stream the results as rows
go func() {
for _, d := range chartRun.Data.Rows {
// we need to allocate a new slice everytime, since this gets read
// asynchronously on the other end and we need to make sure that we don't overwrite
// data already sent
rowVals := make([]interface{}, len(chartRun.Data.Columns))
for i, c := range chartRun.Data.Columns {
rowVals[i] = d[c.Name]
}
res.StreamRow(rowVals)
}
res.TimingResult <- chartRun.TimingResult
res.Close()
}()
return res, nil
}
// convert the given command line query into a query resource and add to workspace
// this is to allow us to use existing dashboard execution code
func ensureSnapshotQueryResource(name string, resolvedQuery *modconfig.ResolvedQuery, w *workspace.Workspace) (queryProvider modconfig.HclResource, existingResource bool) {
// is this an existing resource?
if parsedName, err := modconfig.ParseResourceName(name); err == nil {
if resource, found := w.GetResource(parsedName); found {
return resource, true
}
}
// build name
shortName := "command_line_query"
// this is NOT a named query - create the query using RawSql
q := modconfig.NewQuery(&hcl.Block{Type: modconfig.BlockTypeQuery}, w.Mod, shortName).(*modconfig.Query)
q.SQL = utils.ToStringPointer(resolvedQuery.RawSQL)
q.SetArgs(resolvedQuery.QueryArgs())
// add empty metadata
q.SetMetadata(&modconfig.ResourceMetadata{})
// add to the workspace mod so the dashboard execution code can find it
w.Mod.AddResource(q)
// return the new resource name
return q, false
}
//
//func snapshotToQueryResult(snap *dashboardtypes.SteampipeSnapshot) (*queryresult.Result, error) {
// // the table of a snapshot query has a fixed name
// tablePanel, ok := snap.Panels[modconfig.SnapshotQueryTableName]
// if !ok {
// return nil, sperr.New("dashboard does not contain table result for query")
// }
// chartRun := tablePanel.(*snapshot.LeafRun)
// if !ok {
// return nil, sperr.New("failed to read query result from snapshot")
// }
// // check for error
// if err := chartRun.GetError(); err != nil {
// return nil, error_helpers.DecodePgError(err)
// }
//
// res := queryresult.NewResult(chartRun.Data.Columns)
//
// // start a goroutine to stream the results as rows
// go func() {
// for _, d := range chartRun.Data.Rows {
// // we need to allocate a new slice everytime, since this gets read
// // asynchronously on the other end and we need to make sure that we don't overwrite
// // data already sent
// rowVals := make([]interface{}, len(chartRun.Data.Columns))
// for i, c := range chartRun.Data.Columns {
// rowVals[i] = d[c.Name]
// }
// res.StreamRow(rowVals)
// }
// res.TimingResult <- chartRun.TimingResult
// res.Close()
// }()
//
// return res, nil
//}
//
//// convert the given command line query into a query resource and add to workspace
//// this is to allow us to use existing dashboard execution code
//func ensureSnapshotQueryResource(name string, resolvedQuery *modconfig.ResolvedQuery, w *workspace.Workspace) (queryProvider modconfig.HclResource, existingResource bool) {
// // is this an existing resource?
// if parsedName, err := modconfig.ParseResourceName(name); err == nil {
// if resource, found := w.GetResource(parsedName); found {
// return resource, true
// }
// }
//
// // build name
// shortName := "command_line_query"
//
// // this is NOT a named query - create the query using RawSql
// q := modconfig.NewQuery(&hcl.Block{Type: modconfig.BlockTypeQuery}, w.Mod, shortName).(*modconfig.Query)
// q.SQL = utils.ToStringPointer(resolvedQuery.RawSQL)
// q.SetArgs(resolvedQuery.QueryArgs())
// // add empty metadata
// q.SetMetadata(&modconfig.ResourceMetadata{})
//
// // add to the workspace mod so the dashboard execution code can find it
// w.Mod.AddResource(q)
// // return the new resource name
// return q, false
//}
func snapshotRequired() bool {
SnapshotFormatNames := []string{constants.OutputFormatSnapshot, constants.OutputFormatSnapshotShort}
@@ -391,28 +372,29 @@ func getPipedStdinData() string {
return stdinData
}
func publishSnapshotIfNeeded(ctx context.Context, snapshot *dashboardtypes.SteampipeSnapshot) error {
shouldShare := viper.GetBool(constants.ArgShare)
shouldUpload := viper.GetBool(constants.ArgSnapshot)
if !(shouldShare || shouldUpload) {
return nil
}
message, err := cloud.PublishSnapshot(ctx, snapshot, shouldShare)
if err != nil {
// reword "402 Payment Required" error
return handlePublishSnapshotError(err)
}
if viper.GetBool(constants.ArgProgress) {
fmt.Println(message)
}
return nil
}
func handlePublishSnapshotError(err error) error {
if err.Error() == "402 Payment Required" {
return fmt.Errorf("maximum number of snapshots reached")
}
return err
}
//
//func publishSnapshotIfNeeded(ctx context.Context, snapshot *dashboardtypes.SteampipeSnapshot) error {
// shouldShare := viper.GetBool(constants.ArgShare)
// shouldUpload := viper.GetBool(constants.ArgSnapshot)
//
// if !(shouldShare || shouldUpload) {
// return nil
// }
//
// message, err := cloud.PublishSnapshot(ctx, snapshot, shouldShare)
// if err != nil {
// // reword "402 Payment Required" error
// return handlePublishSnapshotError(err)
// }
// if viper.GetBool(constants.ArgProgress) {
// fmt.Println(message)
// }
// return nil
//}
//
//func handlePublishSnapshotError(err error) error {
// if err.Error() == "402 Payment Required" {
// return fmt.Errorf("maximum number of snapshots reached")
// }
// return err
//}