jprdonnelly/steampipe
1
0
Fork 0
mirror of https://github.com/turbot/steampipe.git synced 2026-04-30 16:00:08 -04:00
Code Issues Packages Projects Releases Wiki Activity

Add support for mod management commands. Closes #442. Closes #443

Browse Source
This commit is contained in:
kaidaguerre
2021-12-21 14:10:00 +00:00
committed by GitHub
parent d0e10471d8
commit 33f55e584f
97 changed files with 3135 additions and 889 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/check.go
View File

@@ -10,6 +10,8 @@ import (
"sync"
"time"
"github.com/turbot/steampipe/mod_installer"
"github.com/briandowns/spinner"
"github.com/spf13/cobra"
"github.com/spf13/viper"
@@ -95,7 +97,8 @@ You may specify one or more benchmarks or controls to run (separated by a space)
// where args passed to StringArrayFlag are not parsed and used raw
AddStringArrayFlag(constants.ArgVariable, "", nil, "Specify the value of a variable").
AddStringFlag(constants.ArgWhere, "", "", "SQL 'where' clause, or named query, used to filter controls (cannot be used with '--tag')").
AddIntFlag(constants.ArgMaxParallel, "", constants.DefaultMaxConnections, "The maximum number of parallel executions", cmdconfig.FlagOptions.Hidden())
AddIntFlag(constants.ArgMaxParallel, "", constants.DefaultMaxConnections, "The maximum number of parallel executions", cmdconfig.FlagOptions.Hidden()).
AddBoolFlag(constants.ArgModInstall, "", true, "Specify whether to install mod depdencies before runnign the check")
return cmd
}
@@ -198,6 +201,15 @@ func initialiseCheck(ctx context.Context, spinner *spinner.Spinner) *checkInitDa
result: &db_common.InitResult{},
}
if viper.GetBool(constants.ArgModInstall) {
opts := &mod_installer.InstallOpts{WorkspacePath: viper.GetString(constants.ArgWorkspaceChDir)}
_, err := mod_installer.InstallWorkspaceDependencies(opts)
if err != nil {
initData.result.Error = err
return initData
}
}
cmdconfig.Viper().Set(constants.ConfigKeyShowInteractiveOutput, false)
err := validateOutputFormat()

258
cmd/mod.go Normal file
View File

@@ -0,0 +1,258 @@
package cmd
import (
"fmt"
"strings"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/turbot/go-kit/helpers"
"github.com/turbot/steampipe/cmdconfig"
"github.com/turbot/steampipe/constants"
"github.com/turbot/steampipe/mod_installer"
"github.com/turbot/steampipe/steampipeconfig/modconfig"
"github.com/turbot/steampipe/steampipeconfig/parse"
"github.com/turbot/steampipe/utils"
)
// mod management commands
func modCmd() *cobra.Command {
var cmd = &cobra.Command{
Use: "mod [command]",
Args: cobra.NoArgs,
Short: "Steampipe mod management",
Long: `Steampipe mod management.`,
}
cmd.AddCommand(modInstallCmd())
cmd.AddCommand(modUninstallCmd())
cmd.AddCommand(modUpdateCmd())
cmd.AddCommand(modPruneCmd())
cmd.AddCommand(modListCmd())
cmd.AddCommand(modInitCmd())
return cmd
}
// install
func modInstallCmd() *cobra.Command {
var cmd = &cobra.Command{
Use: "install",
Run: runModInstallCmd,
Short: "Install mod dependencies",
Long: `Install mod dependencies.
`,
}
cmdconfig.OnCmd(cmd).
AddBoolFlag(constants.ArgPrune, "", true, "Remove unreferenced mods after installation").
AddBoolFlag(constants.ArgDryRun, "", false, "Show which mods would be installed or uninstalled without performing the installation")
return cmd
}
func runModInstallCmd(cmd *cobra.Command, args []string) {
utils.LogTime("cmd.runModInstallCmd")
defer func() {
utils.LogTime("cmd.runModInstallCmd end")
if r := recover(); r != nil {
utils.ShowError(helpers.ToError(r))
exitCode = 1
}
}()
// if any mod names were passed as args, convert into formed mod names
opts := newInstallOpts(cmd, args...)
installData, err := mod_installer.InstallWorkspaceDependencies(opts)
utils.FailOnError(err)
fmt.Println(mod_installer.BuildInstallSummary(installData))
}
// uninstall
func modUninstallCmd() *cobra.Command {
var cmd = &cobra.Command{
Use: "uninstall",
Run: runModUninstallCmd,
Short: "Uninstall mod dependencies",
Long: `Uninstall mod dependencies.
`,
}
cmdconfig.OnCmd(cmd).
AddBoolFlag(constants.ArgPrune, "", true, "Remove unreferenced mods after uninstallation").
AddBoolFlag(constants.ArgDryRun, "", false, "Show which mods would be uninstalled without removing them")
return cmd
}
func runModUninstallCmd(cmd *cobra.Command, args []string) {
utils.LogTime("cmd.runModInstallCmd")
defer func() {
utils.LogTime("cmd.runModInstallCmd end")
if r := recover(); r != nil {
utils.ShowError(helpers.ToError(r))
exitCode = 1
}
}()
opts := newInstallOpts(cmd, args...)
installData, err := mod_installer.UninstallWorkspaceDependencies(opts)
utils.FailOnError(err)
fmt.Println(mod_installer.BuildUninstallSummary(installData))
}
// update
func modUpdateCmd() *cobra.Command {
var cmd = &cobra.Command{
Use: "update",
Run: runModUpdateCmd,
Short: "Update workspace dependencies",
Long: `Update workspace dependencies.
`,
}
cmdconfig.OnCmd(cmd).
AddBoolFlag(constants.ArgPrune, "", true, "Remove unreferenced mods after installation").
AddBoolFlag(constants.ArgDryRun, "", false, "Show which mods would be updated without updating them")
return cmd
}
func runModUpdateCmd(cmd *cobra.Command, args []string) {
utils.LogTime("cmd.runModUpdateCmd")
defer func() {
utils.LogTime("cmd.runModUpdateCmd end")
if r := recover(); r != nil {
utils.ShowError(helpers.ToError(r))
exitCode = 1
}
}()
opts := newInstallOpts(cmd, args...)
installData, err := mod_installer.InstallWorkspaceDependencies(opts)
utils.FailOnError(err)
fmt.Println(mod_installer.BuildInstallSummary(installData))
}
// list
func modListCmd() *cobra.Command {
var cmd = &cobra.Command{
Use: "list",
Run: runModListCmd,
Short: "List mod dependencies",
Long: `List mod dependencies.
`,
}
cmdconfig.OnCmd(cmd)
return cmd
}
func runModListCmd(cmd *cobra.Command, _ []string) {
utils.LogTime("cmd.runModListCmd")
defer func() {
utils.LogTime("cmd.runModListCmd end")
if r := recover(); r != nil {
utils.ShowError(helpers.ToError(r))
exitCode = 1
}
}()
opts := newInstallOpts(cmd)
installer, err := mod_installer.NewModInstaller(opts)
utils.FailOnError(err)
treeString := installer.GetModList()
if len(strings.Split(treeString, "\n")) > 1 {
fmt.Println()
}
fmt.Println(treeString)
}
// prune
func modPruneCmd() *cobra.Command {
var cmd = &cobra.Command{
Use: "prune",
Run: runModPruneCmd,
Short: "Prune mod dependencies",
Long: `Prune mod dependencies.
`,
}
cmdconfig.OnCmd(cmd)
return cmd
}
func runModPruneCmd(cmd *cobra.Command, args []string) {
utils.LogTime("cmd.runModPruneCmd")
defer func() {
utils.LogTime("cmd.runModPruneCmd end")
if r := recover(); r != nil {
utils.ShowError(helpers.ToError(r))
exitCode = 1
}
}()
opts := &mod_installer.InstallOpts{
WorkspacePath: viper.GetString(constants.ArgWorkspaceChDir),
DryRun: viper.GetBool(constants.ArgDryRun),
}
// install workspace dependencies
installer, err := mod_installer.NewModInstaller(opts)
utils.FailOnError(err)
unusedMods, err := installer.Prune()
utils.FailOnError(err)
if count := len(unusedMods.FlatMap()); count > 0 {
fmt.Println(mod_installer.BuildPruneSummary(unusedMods))
}
}
// init
func modInitCmd() *cobra.Command {
var cmd = &cobra.Command{
Use: "init",
Run: runModInitCmd,
Short: "Create a modfile in the current directory",
Long: `Create a modfile in the current directory`,
}
cmdconfig.OnCmd(cmd)
return cmd
}
func runModInitCmd(cmd *cobra.Command, args []string) {
utils.LogTime("cmd.runModInitCmd")
defer func() {
utils.LogTime("cmd.runModInitCmd end")
if r := recover(); r != nil {
utils.ShowError(helpers.ToError(r))
exitCode = 1
}
}()
workspacePath := viper.GetString(constants.ArgWorkspaceChDir)
if parse.ModfileExists(workspacePath) {
fmt.Println("Working folder already contains a mod definition file")
return
}
mod := modconfig.CreateDefaultMod(workspacePath)
utils.FailOnError(mod.Save())
fmt.Printf("Created mod definition file '%s'\n", constants.ModFilePath(workspacePath))
}
// helpers
func newInstallOpts(cmd *cobra.Command, args ...string) *mod_installer.InstallOpts {
opts := &mod_installer.InstallOpts{
WorkspacePath: viper.GetString(constants.ArgWorkspaceChDir),
DryRun: viper.GetBool(constants.ArgDryRun),
ModArgs: args,
Command: cmd.Name(),
}
return opts
}

42
cmd/plugin.go
View File

@@ -112,7 +112,7 @@ Examples:
cmdconfig.
OnCmd(cmd).
AddBoolFlag("all", "", false, "Update all plugins to its latest available version").
AddBoolFlag(constants.ArgAll, "", false, "Update all plugins to its latest available version").
AddBoolFlag(constants.ArgHelp, "h", false, "Help for plugin update")
return cmd
@@ -273,25 +273,14 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
exitCode = 1
}
}()
// args to 'plugin update' -- one or more plugins to install
// These can be simple names ('aws') for "standard" plugins, or
// full refs to the OCI image (us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0)
plugins := append([]string{}, args...)
if len(plugins) == 0 && !(cmdconfig.Viper().GetBool("all")) {
// args to 'plugin update' -- one or more plugins to update
// These can be simple names ('aws') for "standard" plugins,
// or full refs to the OCI image (us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0)
plugins, err := resolveUpdatePluginsFromArgs(args)
if err != nil {
fmt.Println()
utils.ShowError(fmt.Errorf("you need to provide at least one plugin to update or use the %s flag", constants.Bold("--all")))
fmt.Println()
cmd.Help()
fmt.Println()
exitCode = 2
return
}
if len(plugins) > 0 && cmdconfig.Viper().GetBool("all") {
// we can't allow update and install at the same time
fmt.Println()
utils.ShowError(fmt.Errorf("%s cannot be used when updating specific plugins", constants.Bold("`--all`")))
utils.ShowError(err)
fmt.Println()
cmd.Help()
fmt.Println()
@@ -320,7 +309,7 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
// a leading blank line - since we always output multiple lines
fmt.Println()
if cmdconfig.Viper().GetBool("all") {
if cmdconfig.Viper().GetBool(constants.ArgAll) {
for k, v := range versionData.Plugins {
ref := ociinstaller.NewSteampipeImageRef(k)
org, name, stream := ref.GetOrgNameAndStream()
@@ -421,6 +410,21 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
fmt.Println()
}
func resolveUpdatePluginsFromArgs(args []string) ([]string, error) {
plugins := append([]string{}, args...)
if len(plugins) == 0 && !(cmdconfig.Viper().GetBool("all")) {
// either plugin name(s) or "all" must be provided
return nil, fmt.Errorf("you need to provide at least one plugin to update or use the %s flag", constants.Bold("--all"))
}
if len(plugins) > 0 && cmdconfig.Viper().GetBool(constants.ArgAll) {
// we can't allow update and install at the same time
return nil, fmt.Errorf("%s cannot be used when updating specific plugins", constants.Bold("`--all`"))
}
return plugins, nil
}
// start service if necessary and refresh connections
func refreshConnectionsIfNecessary(ctx context.Context, reports []display.InstallReport, isUpdate bool) error {
// get count of skipped reports

3
cmd/root.go
View File

@@ -24,7 +24,7 @@ var exitCode int
// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
Use: "steampipe [--version] [--help] COMMAND [args]",
Version: version.String(),
Version: version.SteampipeVersion.String(),
PersistentPreRun: func(cmd *cobra.Command, args []string) {
utils.LogTime("cmd.root.PersistentPreRun start")
defer utils.LogTime("cmd.root.PersistentPreRun end")
@@ -172,6 +172,7 @@ func AddCommands() {
queryCmd(),
checkCmd(),
serviceCmd(),
modCmd(),
generateCompletionScriptsCmd(),
pluginManagerCmd(),
)

13
cmdconfig/builder.go
View File

@@ -14,7 +14,7 @@ type CmdBuilder struct {
bindings map[string]*pflag.Flag
}
// OnCmd :: starts a config builder wrapping over the provided *cobra.Command
// OnCmd starts a config builder wrapping over the provided *cobra.Command
func OnCmd(cmd *cobra.Command) *CmdBuilder {
cfg := new(CmdBuilder)
cfg.cmd = cmd
@@ -50,7 +50,7 @@ func OnCmd(cmd *cobra.Command) *CmdBuilder {
return cfg
}
// Helper function to add a string flag to a command
// AddStringFlag is a helper function to add a string flag to a command
func (c *CmdBuilder) AddStringFlag(name string, shorthand string, defaultValue string, desc string, opts ...flagOpt) *CmdBuilder {
c.cmd.Flags().StringP(name, shorthand, defaultValue, desc)
c.bindings[name] = c.cmd.Flags().Lookup(name)
@@ -61,7 +61,7 @@ func (c *CmdBuilder) AddStringFlag(name string, shorthand string, defaultValue s
return c
}
// Helper function to add an integer flag to a command
// AddIntFlag is a helper function to add an integer flag to a command
func (c *CmdBuilder) AddIntFlag(name, shorthand string, defaultValue int, desc string, opts ...flagOpt) *CmdBuilder {
c.cmd.Flags().IntP(name, shorthand, defaultValue, desc)
c.bindings[name] = c.cmd.Flags().Lookup(name)
@@ -71,7 +71,7 @@ func (c *CmdBuilder) AddIntFlag(name, shorthand string, defaultValue int, desc s
return c
}
// Helper function to add a boolean flag to a command
// AddBoolFlag ia s helper function to add a boolean flag to a command
func (c *CmdBuilder) AddBoolFlag(name, shorthand string, defaultValue bool, desc string, opts ...flagOpt) *CmdBuilder {
c.cmd.Flags().BoolP(name, shorthand, defaultValue, desc)
c.bindings[name] = c.cmd.Flags().Lookup(name)
@@ -81,7 +81,7 @@ func (c *CmdBuilder) AddBoolFlag(name, shorthand string, defaultValue bool, desc
return c
}
// Helper function to add a flag that accepts an array of strings
// AddStringSliceFlag is a helper function to add a flag that accepts an array of strings
func (c *CmdBuilder) AddStringSliceFlag(name, shorthand string, defaultValue []string, desc string, opts ...flagOpt) *CmdBuilder {
c.cmd.Flags().StringSliceP(name, shorthand, defaultValue, desc)
c.bindings[name] = c.cmd.Flags().Lookup(name)
@@ -91,6 +91,7 @@ func (c *CmdBuilder) AddStringSliceFlag(name, shorthand string, defaultValue []s
return c
}
// AddStringArrayFlag is a helper function to add a flag that accepts an array of strings
func (c *CmdBuilder) AddStringArrayFlag(name, shorthand string, defaultValue []string, desc string, opts ...flagOpt) *CmdBuilder {
c.cmd.Flags().StringArrayP(name, shorthand, defaultValue, desc)
c.bindings[name] = c.cmd.Flags().Lookup(name)
@@ -100,7 +101,7 @@ func (c *CmdBuilder) AddStringArrayFlag(name, shorthand string, defaultValue []s
return c
}
// Helper function to add a flag that accepts a map of strings
// AddStringMapStringFlag is a helper function to add a flag that accepts a map of strings
func (c *CmdBuilder) AddStringMapStringFlag(name, shorthand string, defaultValue map[string]string, desc string, opts ...flagOpt) *CmdBuilder {
c.cmd.Flags().StringToStringP(name, shorthand, defaultValue, desc)
c.bindings[name] = c.cmd.Flags().Lookup(name)

2
constants/args.go
View File

@@ -37,6 +37,8 @@ const (
ArgVarFile = "var-file"
ArgConnectionString = "connection-string"
ArgCheckDisplayWidth = "check-display-width"
ArgPrune = "prune"
ArgModInstall = "mod-install"
)
/// metaquery mode arguments

5
constants/file_paths.go
View File

@@ -43,11 +43,6 @@ func ConnectionStatePath() string {
return filepath.Join(InternalDir(), ConnectionsStateFileName)
}
// ModsDir returns the path to the mods directory (creates if missing)
func ModsDir() string {
return steampipeSubDir("mods")
}
// ConfigDir returns the path to the config directory (creates if missing)
func ConfigDir() string {
return steampipeSubDir("config")

15
constants/workspace.go
View File

@@ -2,6 +2,7 @@ package constants
import (
"path"
"path/filepath"
)
// mod related constants
@@ -10,15 +11,25 @@ const (
WorkspaceModDir = "mods"
WorkspaceConfigFileName = "workspace.spc"
WorkspaceIgnoreFile = ".steampipeignore"
WorkspaceDefaultModName = "local"
WorkspaceModFileName = "mod.sp"
ModFileName = "mod.sp"
DefaultVarsFileName = "steampipe.spvars"
WorkspaceLockFileName = ".mod.cache.json"
MaxControlRunAttempts = 2
)
func WorkspaceModPath(workspacePath string) string {
return path.Join(workspacePath, WorkspaceDataDir, WorkspaceModDir)
}
func WorkspaceLockPath(workspacePath string) string {
return path.Join(workspacePath, WorkspaceLockFileName)
}
func DefaultVarsFilePath(workspacePath string) string {
return path.Join(workspacePath, DefaultVarsFileName)
}
func ModFilePath(modFolder string) string {
modFilePath := filepath.Join(modFolder, ModFileName)
return modFilePath
}

2
control/controldisplay/csv.go
View File

@@ -57,7 +57,7 @@ func (r CSVRenderer) renderControl(run *controlexecute.ControlRun, group *contro
}
tags := make(map[string]string)
if run.Control.Tags != nil {
tags = *run.Control.Tags
tags = run.Control.Tags
}
for _, prop := range r.columns.TagColumns {
val := tags[prop]

2
control/controldisplay/formatter.go
View File

@@ -110,7 +110,7 @@ func (j *NullFormatter) FileExtension() string {
}
var formatterTemplateFuncMap template.FuncMap = template.FuncMap{
"steampipeversion": func() string { return version.String() },
"steampipeversion": func() string { return version.SteampipeVersion.String() },
"workingdir": func() string { wd, _ := os.Getwd(); return wd },
"asstr": func(i reflect.Value) string { return fmt.Sprintf("%v", i) },
"statusicon": func(status string) string {

103
control/controlexecute/control_run.go
View File

@@ -65,10 +65,15 @@ type ControlRun struct {
}
func NewControlRun(control *modconfig.Control, group *ResultGroup, executionTree *ExecutionTree) *ControlRun {
res := &ControlRun{
Control: control,
controlId := control.Name()
// only show qualified control names for controls from dependent mods
if control.Mod.Name() == executionTree.workspace.Mod.Name() {
controlId = control.UnqualifiedName
}
ControlId: control.Name(),
res := &ControlRun{
Control: control,
ControlId: controlId,
Description: typehelpers.SafeString(control.Description),
Severity: typehelpers.SafeString(control.Severity),
Title: typehelpers.SafeString(control.Title),
@@ -125,26 +130,6 @@ func (r *ControlRun) setSearchPath(ctx context.Context, session *db_common.Datab
return err
}
func (r *ControlRun) getCurrentSearchPath(ctx context.Context, session *db_common.DatabaseSession) ([]string, error) {
utils.LogTime("ControlRun.getCurrentSearchPath start")
defer utils.LogTime("ControlRun.getCurrentSearchPath end")
row := session.Connection.QueryRowContext(ctx, "show search_path")
pathAsString := ""
err := row.Scan(&pathAsString)
if err != nil {
return nil, err
}
currentSearchPath := strings.Split(pathAsString, ",")
// unescape search path
for idx, p := range currentSearchPath {
p = strings.Join(strings.Split(p, "\""), "")
p = strings.TrimSpace(p)
currentSearchPath[idx] = p
}
return currentSearchPath, nil
}
func (r *ControlRun) Execute(ctx context.Context, client db_common.Client) {
log.Printf("[TRACE] begin ControlRun.Start: %s\n", r.Control.Name())
defer log.Printf("[TRACE] end ControlRun.Start: %s\n", r.Control.Name())
@@ -246,6 +231,41 @@ func (r *ControlRun) Execute(ctx context.Context, client db_common.Client) {
log.Printf("[TRACE] finish result for, %s\n", control.Name())
}
func (r *ControlRun) SetError(err error) {
if err == nil {
return
}
r.runError = utils.TransformErrorToSteampipe(err)
// update error count
r.Summary.Error++
r.setRunStatus(ControlRunError)
}
func (r *ControlRun) GetError() error {
return r.runError
}
func (r *ControlRun) getCurrentSearchPath(ctx context.Context, session *db_common.DatabaseSession) ([]string, error) {
utils.LogTime("ControlRun.getCurrentSearchPath start")
defer utils.LogTime("ControlRun.getCurrentSearchPath end")
row := session.Connection.QueryRowContext(ctx, "show search_path")
pathAsString := ""
err := row.Scan(&pathAsString)
if err != nil {
return nil, err
}
currentSearchPath := strings.Split(pathAsString, ",")
// unescape search path
for idx, p := range currentSearchPath {
p = strings.Join(strings.Split(p, "\""), "")
p = strings.TrimSpace(p)
currentSearchPath[idx] = p
}
return currentSearchPath, nil
}
func (r *ControlRun) getControlQueryContext(ctx context.Context) context.Context {
// create a context with a deadline
shouldBeDoneBy := time.Now().Add(controlQueryTimeout)
@@ -254,6 +274,17 @@ func (r *ControlRun) getControlQueryContext(ctx context.Context) context.Context
return newCtx
}
func (r *ControlRun) GetRunStatus() ControlRunStatus {
r.stateLock.Lock()
defer r.stateLock.Unlock()
return r.runStatus
}
func (r *ControlRun) Finished() bool {
status := r.GetRunStatus()
return status == ControlRunComplete || status == ControlRunError
}
func (r *ControlRun) resolveControlQuery(control *modconfig.Control) (string, error) {
query, err := r.executionTree.workspace.ResolveControlQuery(control, nil)
if err != nil {
@@ -345,21 +376,6 @@ func (r *ControlRun) createdOrderedResultRows() {
}
}
func (r *ControlRun) SetError(err error) {
if err == nil {
return
}
r.runError = utils.TransformErrorToSteampipe(err)
// update error count
r.Summary.Error++
r.setRunStatus(ControlRunError)
}
func (r *ControlRun) GetError() error {
return r.runError
}
func (r *ControlRun) setRunStatus(status ControlRunStatus) {
r.stateLock.Lock()
r.runStatus = status
@@ -377,14 +393,3 @@ func (r *ControlRun) setRunStatus(status ControlRunStatus) {
r.doneChan <- true
}
}
func (r *ControlRun) GetRunStatus() ControlRunStatus {
r.stateLock.Lock()
defer r.stateLock.Unlock()
return r.runStatus
}
func (r *ControlRun) Finished() bool {
status := r.GetRunStatus()
return status == ControlRunComplete || status == ControlRunError
}

28
control/controlexecute/execution_tree.go
View File

@@ -49,13 +49,13 @@ func NewExecutionTree(ctx context.Context, workspace *workspace.Workspace, clien
}
// now identify the root item of the control list
rootItems, err := executionTree.getExecutionRootFromArg(arg)
rootItem, err := executionTree.getExecutionRootFromArg(arg)
if err != nil {
return nil, err
}
// build tree of result groups, starting with a synthetic 'root' node
executionTree.Root = NewRootResultGroup(executionTree, rootItems...)
executionTree.Root = NewRootResultGroup(executionTree, rootItem)
// after tree has built, ControlCount will be set - create progress rendered
executionTree.progress = NewControlProgressRenderer(len(executionTree.controlRuns))
@@ -192,16 +192,18 @@ func (e *ExecutionTree) ShouldIncludeControl(controlName string) bool {
// - if the arg is a benchmark name, the root will be the Benchmark with that name
// - if the arg is a mod name, the root will be the Mod with that name
// - if the arg is 'all' the root will be a node with all Mods as children
func (e *ExecutionTree) getExecutionRootFromArg(arg string) ([]modconfig.ModTreeItem, error) {
var res []modconfig.ModTreeItem
func (e *ExecutionTree) getExecutionRootFromArg(arg string) (modconfig.ModTreeItem, error) {
// special case handling for the string "all"
if arg == "all" {
//
// build list of all workspace mods - these will act as root items
for _, m := range e.workspace.Mods {
res = append(res, m)
// return the workspace mod as root
return e.workspace.Mod, nil
}
// if the arg is the name of one of the workjspace dependen
for _, mod := range e.workspace.Mods {
if mod.ShortName == arg {
return mod, nil
}
return res, nil
}
// what resource type is arg?
@@ -215,17 +217,17 @@ func (e *ExecutionTree) getExecutionRootFromArg(arg string) ([]modconfig.ModTree
case modconfig.BlockTypeControl:
// check whether the arg is a control name
if control, ok := e.workspace.Controls[arg]; ok {
return []modconfig.ModTreeItem{control}, nil
return control, nil
}
case modconfig.BlockTypeBenchmark:
// look in the workspace control group map for this control group
if benchmark, ok := e.workspace.Benchmarks[arg]; ok {
return []modconfig.ModTreeItem{benchmark}, nil
return benchmark, nil
}
case modconfig.BlockTypeMod:
// get all controls for the mod
if mod, ok := e.workspace.Mods[arg]; ok {
return []modconfig.ModTreeItem{mod}, nil
return mod, nil
}
}
return nil, fmt.Errorf("no controls found matching argument '%s'", arg)
@@ -280,7 +282,7 @@ func (e *ExecutionTree) GetAllTags() []string {
var tagColumns []string
for _, r := range e.controlRuns {
if r.Control.Tags != nil {
for tag := range *r.Control.Tags {
for tag := range r.Control.Tags {
if !tagColumnMap[tag] {
tagColumns = append(tagColumns, tag)
tagColumnMap[tag] = true

6
control/controlexecute/result_group.go
View File

@@ -72,6 +72,12 @@ func NewRootResultGroup(executionTree *ExecutionTree, rootItems ...modconfig.Mod
// NewResultGroup creates a result group from a ModTreeItem
func NewResultGroup(executionTree *ExecutionTree, treeItem modconfig.ModTreeItem, parent *ResultGroup) *ResultGroup {
// only show qualified group names for controls from dependent mods
groupId := treeItem.Name()
if mod := treeItem.GetMod(); mod != nil && mod.Name() == executionTree.workspace.Mod.Name() {
groupId = modconfig.UnqualifiedResourceName(groupId)
}
group := &ResultGroup{
GroupId: treeItem.Name(),
Title: treeItem.GetTitle(),

2
control/controlexecute/result_row.go
View File

@@ -19,7 +19,7 @@ type ResultRow struct {
Resource string `json:"resource" csv:"resource"`
Status string `json:"status" csv:"status"`
Dimensions []Dimension `json:"dimensions"`
Control *modconfig.Control `json:"-" csv:"control_id:FullName,control_title:Title,control_description:Description"`
Control *modconfig.Control `json:"-" csv:"control_id:UnqualifiedName,control_title:Title,control_description:Description"`
}
// AddDimension checks whether a column value is a scalar type, and if so adds it to the Dimensions map

3
db/db_client/db_client_session.go
View File

@@ -8,6 +8,7 @@ import (
"time"
"github.com/jackc/pgx/v4/stdlib"
"github.com/sethvargo/go-retry"
"github.com/turbot/steampipe/db/db_common"
"github.com/turbot/steampipe/utils"
@@ -144,5 +145,5 @@ func (c *DbClient) getSessionWithRetries(ctx context.Context) (*sql.Conn, uint32
return nil
})
return session, backendPid, nil
return session, uint32(backendPid), nil
}

2
go.mod
View File

@@ -4,6 +4,7 @@ go 1.17
require (
github.com/Machiel/slugify v1.0.1
github.com/Masterminds/semver v1.5.0
github.com/ahmetb/go-linq v3.0.0+incompatible
github.com/alecthomas/chroma v0.9.2
github.com/bgentry/speakeasy v0.1.0
@@ -45,6 +46,7 @@ require (
github.com/turbot/go-kit v0.3.0
github.com/turbot/steampipe-plugin-sdk v1.8.0
github.com/ulikunitz/xz v0.5.8
github.com/xlab/treeprint v1.1.0
github.com/zclconf/go-cty v1.8.2
github.com/zclconf/go-cty-yaml v1.0.2
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c

5
go.sum
View File

@@ -81,7 +81,9 @@ github.com/ChrisTrenkamp/goxpath v0.0.0-20190607011252-c5096ec8773d/go.mod h1:nu
github.com/Machiel/slugify v1.0.1 h1:EfWSlRWstMadsgzmiV7d0yVd2IFlagWH68Q+DcYCm4E=
github.com/Machiel/slugify v1.0.1/go.mod h1:fTFGn5uWEynW4CUMG7sWkYXOf1UgDxyTM3DbR6Qfg3k=
github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
@@ -729,7 +731,6 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
@@ -1015,6 +1016,8 @@ github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf
github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xlab/treeprint v0.0.0-20161029104018-1d6e34225557/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk=
github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

2
interactive/interactive_client.go
View File

@@ -115,7 +115,7 @@ func (c *InteractiveClient) InteractivePrompt() {
c.resultsStreamer.Close()
}()
fmt.Printf("Welcome to Steampipe v%s\n", version.String())
fmt.Printf("Welcome to Steampipe v%s\n", version.SteampipeVersion.String())
fmt.Printf("For more information, type %s\n", constants.Bold(".help"))
// run the prompt in a goroutine, so we can also detect async initialisation errors

68
mod_installer/git.go Normal file
View File

@@ -0,0 +1,68 @@
package mod_installer
import (
"fmt"
"sort"
"github.com/Masterminds/semver"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/storage/memory"
)
func getGitUrl(modName string) string {
return fmt.Sprintf("https://%s", modName)
}
func getTags(repo string) ([]string, error) {
// Create the remote with repository URL
rem := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
Name: "origin",
URLs: []string{repo},
})
// load remote references
refs, err := rem.List(&git.ListOptions{})
if err != nil {
return nil, err
}
// filters the references list and only keeps tags
var tags []string
for _, ref := range refs {
if ref.Name().IsTag() {
tags = append(tags, ref.Name().Short())
}
}
return tags, nil
}
func getTagVersionsFromGit(repo string, includePrerelease bool) (semver.Collection, error) {
tags, err := getTags(repo)
if err != nil {
return nil, err
}
versions := make(semver.Collection, len(tags))
// handle index manually as we may not add all tags - if we cannot parse them as a version
idx := 0
for _, raw := range tags {
v, err := semver.NewVersion(raw)
if err != nil {
continue
}
if !includePrerelease && v.Metadata() != "" || v.Prerelease() != "" {
continue
}
versions[idx] = v
idx++
}
// shrink slice
versions = versions[:idx]
// sort the versions in REVERSE order
sort.Sort(sort.Reverse(versions))
return versions, nil
}

16
mod_installer/helpers.go Normal file
View File

@@ -0,0 +1,16 @@
package mod_installer
import (
"github.com/Masterminds/semver"
"github.com/turbot/steampipe/version_helpers"
)
func getVersionSatisfyingConstraint(constraint *version_helpers.Constraints, availableVersions []*semver.Version) *semver.Version {
// search the reverse sorted versions, finding the highest version which satisfies ALL constraints
for _, version := range availableVersions {
if constraint.Check(version) {
return version
}
}
return nil
}

28
mod_installer/install.go Normal file
View File

@@ -0,0 +1,28 @@
package mod_installer
import (
"github.com/turbot/go-kit/helpers"
"github.com/turbot/steampipe/utils"
)
func InstallWorkspaceDependencies(opts *InstallOpts) (_ *InstallData, err error) {
utils.LogTime("cmd.InstallWorkspaceDependencies")
defer func() {
utils.LogTime("cmd.InstallWorkspaceDependencies end")
if r := recover(); r != nil {
err = helpers.ToError(r)
}
}()
// install workspace dependencies
installer, err := NewModInstaller(opts)
if err != nil {
return nil, err
}
if err := installer.InstallWorkspaceDependencies(); err != nil {
return nil, err
}
return installer.installData, nil
}

119
mod_installer/install_data.go Normal file
View File

@@ -0,0 +1,119 @@
package mod_installer
import (
"fmt"
"github.com/Masterminds/semver"
"github.com/turbot/steampipe/steampipeconfig/modconfig"
"github.com/turbot/steampipe/steampipeconfig/version_map"
"github.com/turbot/steampipe/version_helpers"
"github.com/xlab/treeprint"
)
type InstallData struct {
// record of the full dependency tree
Lock *version_map.WorkspaceLock
NewLock *version_map.WorkspaceLock
// ALL the available versions for each dependency mod(we populate this in a lazy fashion)
allAvailable version_map.VersionListMap
// list of dependencies installed by recent install operation
Installed version_map.DependencyVersionMap
// list of dependencies which have been upgraded
Upgraded version_map.DependencyVersionMap
// list of dependencies which have been downgraded
Downgraded version_map.DependencyVersionMap
// list of dependencies which have been uninstalled
Uninstalled version_map.DependencyVersionMap
WorkspaceMod *modconfig.Mod
}
func NewInstallData(workspaceLock *version_map.WorkspaceLock, workspaceMod *modconfig.Mod) *InstallData {
return &InstallData{
Lock: workspaceLock,
WorkspaceMod: workspaceMod,
NewLock: version_map.EmptyWorkspaceLock(workspaceLock),
allAvailable: make(version_map.VersionListMap),
Installed: make(version_map.DependencyVersionMap),
Upgraded: make(version_map.DependencyVersionMap),
Downgraded: make(version_map.DependencyVersionMap),
Uninstalled: make(version_map.DependencyVersionMap),
}
}
// GetAvailableUpdates returns a map of all installed mods which are not in the lock file
func (d *InstallData) GetAvailableUpdates() (version_map.DependencyVersionMap, error) {
res := make(version_map.DependencyVersionMap)
for parent, deps := range d.Lock.InstallCache {
for name, resolvedConstraint := range deps {
includePrerelease := resolvedConstraint.IsPrerelease()
availableVersions, err := d.getAvailableModVersions(name, includePrerelease)
if err != nil {
return nil, err
}
constraint, _ := version_helpers.NewConstraint(resolvedConstraint.Constraint)
var latestVersion = getVersionSatisfyingConstraint(constraint, availableVersions)
if latestVersion.GreaterThan(resolvedConstraint.Version) {
res.Add(name, latestVersion, constraint.Original, parent)
}
}
}
return res, nil
}
// onModInstalled is called when a dependency is satisfied by installing a mod version
func (d *InstallData) onModInstalled(dependency *ResolvedModRef, parent *modconfig.Mod) {
parentPath := parent.GetModDependencyPath()
// get the constraint from the parent (it must be there)
modVersion := parent.Require.GetModDependency(dependency.Name)
// update lock
d.NewLock.InstallCache.Add(dependency.Name, dependency.Version, modVersion.Constraint.Original, parentPath)
}
// addExisting is called when a dependency is satisfied by a mod which is already installed
func (d *InstallData) addExisting(name string, version *semver.Version, constraint *version_helpers.Constraints, parent *modconfig.Mod) {
// update lock
parentPath := parent.GetModDependencyPath()
d.NewLock.InstallCache.Add(name, version, constraint.Original, parentPath)
}
// retrieve all available mod versions from our cache, or from Git if not yet cached
func (d *InstallData) getAvailableModVersions(modName string, includePrerelease bool) ([]*semver.Version, error) {
// have we already loaded the versions for this mod
availableVersions, ok := d.allAvailable[modName]
if ok {
return availableVersions, nil
}
// so we have not cached this yet - retrieve from Git
var err error
availableVersions, err = getTagVersionsFromGit(getGitUrl(modName), includePrerelease)
if err != nil {
return nil, fmt.Errorf("could not retrieve version data from Git URL '%s'", modName)
}
// update our cache
d.allAvailable[modName] = availableVersions
return availableVersions, nil
}
// update the lock with the NewLock and dtermine if any mods have been uninstalled
func (d *InstallData) onInstallComplete() {
d.Installed = d.NewLock.InstallCache.GetMissingFromOther(d.Lock.InstallCache)
d.Uninstalled = d.Lock.InstallCache.GetMissingFromOther(d.NewLock.InstallCache)
d.Upgraded = d.Lock.InstallCache.GetUpgradedInOther(d.NewLock.InstallCache)
d.Downgraded = d.Lock.InstallCache.GetDowngradedInOther(d.NewLock.InstallCache)
d.Lock = d.NewLock
}
func (d *InstallData) GetUpdatedTree() treeprint.Tree {
return d.Upgraded.GetDependencyTree(d.WorkspaceMod.GetModDependencyPath())
}
func (d *InstallData) GetInstalledTree() treeprint.Tree {
return d.Installed.GetDependencyTree(d.WorkspaceMod.GetModDependencyPath())
}
func (d *InstallData) GetUninstalledTree() treeprint.Tree {
return d.Uninstalled.GetDependencyTree(d.WorkspaceMod.GetModDependencyPath())
}

8
mod_installer/install_opts.go Normal file
View File

@@ -0,0 +1,8 @@
package mod_installer
type InstallOpts struct {
WorkspacePath string
Command string
DryRun bool
ModArgs []string
}

554
mod_installer/mod_installer.go
View File

@@ -5,188 +5,404 @@ import (
"log"
"os"
"path/filepath"
"strings"
"github.com/turbot/steampipe/constants"
"github.com/Masterminds/semver"
git "github.com/go-git/go-git/v5"
"github.com/otiai10/copy"
"github.com/spf13/viper"
"github.com/turbot/steampipe/constants"
"github.com/turbot/steampipe/steampipeconfig/modconfig"
"github.com/turbot/steampipe/steampipeconfig/parse"
"github.com/turbot/steampipe/steampipeconfig/version_map"
"github.com/turbot/steampipe/utils"
)
/*
mog get
A user may install a mod with steampipe mod get modname[@version]
version may be:
- Not specified: steampipe mod get aws-cis
The latest version (highest version tag) will be installed.
A dependency is added to the requires block specifying the version that was downloaded
- A major version: steampipe mod get aws-cis@3
The latest release (highest version tag) of the specified major version will be installed.
A dependency is added to the requires block specifying the version that was downloaded
- A monotonic version tag: steampipe mod get aws-cis@v2.21
The specified version is downloaded and added as requires dependency.
- A branch name: steampipe mod get aws-cis@staging
The current version of the specified branch is downloaded.
The branch dependency is added to the requires list. Note that a branch is considered a distinct "major" release, it is not cached in the registry, and has no minor version.
Branch versions do not auto-update - you have to run steampipe mod update to get a newer version.
Branch versioning is meant to simplify development and testing - published mods should ONLY include version tag dependencies, NOT branch dependencies.
- A local file path: steampipe mod get "file:~/my_path/aws-core"
The mod from the local filesystem is added to the namespace, but nothing is downloaded.
The local dependency is added to the requires list. Note that a local mod is considered a distinct "major" release, it is not cached in the registry, and has no minor version.
Local versioning is meant to simplify development and testing - published mods should ONLY include version tag dependencies, NOT local dependencies.
Steampipe Version Dependency
If the installed version of Steampipe does not meet the dependency criteria, the user will be warned and the mod will NOT be installed.
Plugin Dependency5
If the mod specifies plugin versions that are not installed, or have no connections, the user will be warned but the mod will be installed. The user should be warned at installation time, and also when starting Steampipe in the workspace.
Detecting conflicts
mod 1 require a@1.0
mod 2 require a@file:/foo
-> how do we detect if the file version satisfied constrainst of a - this is for dev purposes so always pass?
mod 1 require a@1.0
mod 2 require a@<branch>
-> how do we detect if the file version satisfied constraints of a - check branch?
*/
type ModInstaller struct {
ModsDir string
InstalledDependencies []*ResolvedModRef
workspaceMod *modconfig.Mod
modsPath string
// temp location used to install dependencies
tmpPath string
workspacePath string
installData *InstallData
// what command is being run
command string
// are dependencies being added to the workspace
mods version_map.VersionConstraintMap
dryRun bool
}
func NewModInstaller(workspacePath string) *ModInstaller {
return &ModInstaller{
ModsDir: constants.WorkspaceModPath(workspacePath),
func NewModInstaller(opts *InstallOpts) (*ModInstaller, error) {
i := &ModInstaller{
workspacePath: opts.WorkspacePath,
command: opts.Command,
dryRun: opts.DryRun,
}
}
// InstallModDependencies installs all dependencies of the mod
func (i *ModInstaller) InstallModDependencies(mod *modconfig.Mod) error {
dependencyMap := make(map[string]*ResolvedModRef)
return i.installModDependenciesRecursively(mod, dependencyMap)
}
func (i *ModInstaller) installModDependenciesRecursively(mod *modconfig.Mod, dependencyMap map[string]*ResolvedModRef) error {
if mod.Requires == nil {
return nil
if err := i.setModsPath(); err != nil {
return nil, err
}
// first check our Steampipe version is sufficient
if err := mod.Requires.ValidateSteampipeVersion(mod.Name()); err != nil {
// load workspace mod, creating a default if needed
workspaceMod, err := i.loadModfile(i.workspacePath, true)
if err != nil {
return nil, err
}
i.workspaceMod = workspaceMod
// load lock file
workspaceLock, err := version_map.LoadWorkspaceLock(i.workspacePath)
if err != nil {
return nil, err
}
// create install data
i.installData = NewInstallData(workspaceLock, workspaceMod)
// parse args to get the required mod versions
requiredMods, err := i.GetRequiredModVersionsFromArgs(opts.ModArgs)
if err != nil {
return nil, err
}
i.mods = requiredMods
return i, nil
}
func (i *ModInstaller) setModsPath() error {
dir, err := os.MkdirTemp(os.TempDir(), "sp_dr_*")
if err != nil {
return err
}
i.tmpPath = dir
i.modsPath = constants.WorkspaceModPath(i.workspacePath)
return nil
}
func (i *ModInstaller) UninstallWorkspaceDependencies() error {
workspaceMod := i.workspaceMod
// remove required dependencies from the mod file
if len(i.mods) == 0 {
workspaceMod.RemoveAllModDependencies()
} else {
// verify all the mods specifed in the args exist in the modfile
workspaceMod.RemoveModDependencies(i.mods)
}
// uninstall by calling Install
if err := i.installMods(workspaceMod.Require.Mods, workspaceMod); err != nil {
return err
}
if workspaceMod.Require.Empty() {
workspaceMod.Require = nil
}
// if this is a dry run, return now
if i.dryRun {
log.Printf("[TRACE] UninstallWorkspaceDependencies - dry-run=true, returning before saving mod file and cache\n")
return nil
}
// write the lock file
if err := i.installData.Lock.Save(); err != nil {
return err
}
// now safe to save the mod file
if err := i.workspaceMod.Save(); err != nil {
return err
}
// tidy unused mods
if viper.GetBool(constants.ArgPrune) {
if _, err := i.Prune(); err != nil {
return err
}
}
return nil
}
// InstallWorkspaceDependencies installs all dependencies of the workspace mod
func (i *ModInstaller) InstallWorkspaceDependencies() (err error) {
workspaceMod := i.workspaceMod
defer func() {
// tidy unused mods
// (put in defer so it still gets called in case of errors)
if viper.GetBool(constants.ArgPrune) {
// be sure not to overwrite an existing return error
_, pruneErr := i.Prune()
if pruneErr != nil && err == nil {
err = pruneErr
}
}
}()
// first check our Steampipe version is sufficient
if err := workspaceMod.Require.ValidateSteampipeVersion(workspaceMod.Name()); err != nil {
return err
}
// if mod args have been provided, add them to the the workspace mod requires
// (this will replace any existing dependencies of same name)
if len(i.mods) > 0 {
workspaceMod.AddModDependencies(i.mods)
}
if err := i.installMods(workspaceMod.Require.Mods, workspaceMod); err != nil {
return err
}
// if this is a dry run, return now
if i.dryRun {
log.Printf("[TRACE] InstallWorkspaceDependencies - dry-run=true, returning before saving mod file and cache\n")
return nil
}
// write the lock file
if err := i.installData.Lock.Save(); err != nil {
return err
}
// now safe to save the mod file
if len(i.mods) > 0 {
if err := i.workspaceMod.Save(); err != nil {
return err
}
}
if !workspaceMod.HasDependentMods() {
// there are no dependencies - delete the cache
i.installData.Lock.Delete()
}
return nil
}
func (i *ModInstaller) GetModList() string {
return i.installData.Lock.GetModList(i.workspaceMod.GetModDependencyPath())
}
func (i *ModInstaller) installMods(mods []*modconfig.ModVersionConstraint, parent *modconfig.Mod) error {
// clean up the temp location
defer os.RemoveAll(i.tmpPath)
var errors []error
for _, modVersion := range mod.Requires.Mods {
// get a resolved mod ref for this mod version
resolvedRef, err := i.GetModRefForVersion(modVersion)
for _, requiredModVersion := range mods {
modToUse, err := i.getCurrentlyInstalledVersionToUse(requiredModVersion, parent, i.updating())
if err != nil {
return fmt.Errorf("dependency %s %s cannot be satisfied: %s", mod.Name(), modVersion.VersionString, err.Error())
errors = append(errors, err)
continue
}
// install this mod
// NOTE - this mutates dependency map
if err := i.installDependency(resolvedRef, dependencyMap); err != nil {
// if the mod is not installed or needs updating, pass shouldUpdate=true into installModDependencesRecursively
// this ensures that we update any dependencies which have updates available
shouldUpdate := modToUse == nil
if err := i.installModDependencesRecursively(requiredModVersion, modToUse, parent, shouldUpdate); err != nil {
errors = append(errors, err)
}
}
return utils.CombineErrorsWithPrefix(fmt.Sprintf("%d dependencies failed to install", len(errors)), errors...)
// update the lock to be the new lock, and record any uninstalled mods
i.installData.onInstallComplete()
return i.buildInstallError(errors)
}
func (i *ModInstaller) GetModRefForVersion(modVersion *modconfig.ModVersion) (*ResolvedModRef, error) {
// NOTE check whether the lock file contains this dependency and if so
// does the locked version satisy this version requirement
// return error if not
// NOTE check whether we are replacing this version
// if so does the locked version satisfy this version requirement
// return error if not
// so we need to resolve this mod version
// NOTE for now assume github
// get the most recent minor version fo rthis major version from github
return i.getLatestCompatibleVersionFromGithub(modVersion)
}
func (i *ModInstaller) getLatestCompatibleVersionFromGithub(modVersion *modconfig.ModVersion) (*ResolvedModRef, error) {
// NOTE for now assume the mod is specified with a full version
return NewResolvedModRef(modVersion)
}
func (i *ModInstaller) installDependency(dependency *ResolvedModRef, dependencyMap map[string]*ResolvedModRef) error {
// have we already installed a mod which satisfies this dependency
if modRef, ok := dependencyMap[dependency.Name]; ok {
if modRef.Version.GreaterThanOrEqual(dependency.Version) {
return nil
}
}
// add this dependency into the map (if we fail to install,m the whole installation process will terminate,
// so no need to check for errors
dependencyMap[dependency.Name] = dependency
var modPath string
if dependency.FilePath != "" {
// if there is a file path, verify it exists
if _, err := os.Stat(dependency.FilePath); os.IsNotExist(err) {
return fmt.Errorf("dependency %s file path %s does not exist", dependency.Name, dependency.FilePath)
}
modPath = dependency.FilePath
} else {
modPath = filepath.Join(i.ModsDir, dependency.FullName())
if err := i.installDependencyFromGit(dependency, modPath); err != nil {
return err
}
}
// no load the installed mod and install _its_ dependencies
if !parse.ModfileExists(modPath) {
log.Printf("[TRACE] dependency %s does not define a mod defintion - so there are no dependencies to install", dependency.Name)
func (i *ModInstaller) buildInstallError(errors []error) error {
if len(errors) == 0 {
return nil
}
mod, err := parse.ParseModDefinition(modPath)
if err != nil {
return err
}
err = i.installModDependenciesRecursively(mod, dependencyMap)
// if we succeeded, update our list
if err == nil {
i.InstalledDependencies = append(i.InstalledDependencies, dependency)
verb := "install"
if i.updating() {
verb = "update"
}
prefix := fmt.Sprintf("%d %s failed to %s", len(errors), utils.Pluralize("dependency", len(errors)), verb)
err := utils.CombineErrorsWithPrefix(prefix, errors...)
return err
}
func (i *ModInstaller) installDependencyFromGit(dependency *ResolvedModRef, installPath string) error {
// ensure mod directory exists - create if necessary
if err := os.MkdirAll(i.ModsDir, os.ModePerm); err != nil {
func (i *ModInstaller) installModDependencesRecursively(requiredModVersion *modconfig.ModVersionConstraint, dependencyMod *modconfig.Mod, parent *modconfig.Mod, shouldUpdate bool) error {
// get available versions for this mod
includePrerelease := requiredModVersion.Constraint.IsPrerelease()
availableVersions, err := i.installData.getAvailableModVersions(requiredModVersion.Name, includePrerelease)
if err != nil {
return err
}
// NOTE: we need to check existing installed mods
if dependencyMod == nil {
// so we ARE installing
// get a resolved mod ref that satisfies the version constraints
resolvedRef, err := i.getModRefSatisfyingConstraints(requiredModVersion, availableVersions)
if err != nil {
return err
}
// install the mod
dependencyMod, err = i.install(resolvedRef, parent)
if err != nil {
return err
}
} else {
// so we found an existing mod which will satisfy this requirement
// update the install data
i.installData.addExisting(requiredModVersion.Name, dependencyMod.Version, requiredModVersion.Constraint, parent)
log.Printf("[TRACE] not installing %s with version constraint %s as version %s is already installed", requiredModVersion.Name, requiredModVersion.Constraint.Original, dependencyMod.Version)
}
// to get here we have the dependency mod - either we installed it or it was already installed
// recursively install its dependencies
var errors []error
// now update the parent to dependency mod and install its child dependencies
parent = dependencyMod
for _, dep := range dependencyMod.Require.Mods {
childDependencyMod, err := i.getCurrentlyInstalledVersionToUse(dep, parent, shouldUpdate)
if err != nil {
errors = append(errors, err)
continue
}
if err := i.installModDependencesRecursively(dep, childDependencyMod, parent, shouldUpdate); err != nil {
errors = append(errors, err)
continue
}
}
return utils.CombineErrorsWithPrefix(fmt.Sprintf("%d child %s failed to install", len(errors), utils.Pluralize("dependency", len(errors))), errors...)
}
func (i *ModInstaller) getCurrentlyInstalledVersionToUse(requiredModVersion *modconfig.ModVersionConstraint, parent *modconfig.Mod, forceUpdate bool) (*modconfig.Mod, error) {
// do we have an installed version of this mod matching the required mod constraint
installedVersion, err := i.installData.Lock.GetLockedModVersion(requiredModVersion, parent)
if err != nil {
return nil, err
}
if installedVersion == nil {
return nil, nil
}
// can we update this
canUpdate, err := i.canUpdateMod(installedVersion, requiredModVersion, forceUpdate)
if err != nil {
return nil, err
}
if canUpdate {
// return nil mod to indicate we should update
return nil, nil
}
// load the existing mod and return
return i.loadDependencyMod(installedVersion)
}
// determine if we should update this mod, and if so whether there is an update available
func (i *ModInstaller) canUpdateMod(installedVersion *version_map.ResolvedVersionConstraint, requiredModVersion *modconfig.ModVersionConstraint, forceUpdate bool) (bool, error) {
// so should we update?
// if forceUpdate is set or if the required version constraint is different to the locked version constraint, update
// TODO check * vs latest - maybe need a custom equals?
if forceUpdate || installedVersion.Constraint != requiredModVersion.Constraint.Original {
// get available versions for this mod
includePrerelease := requiredModVersion.Constraint.IsPrerelease()
availableVersions, err := i.installData.getAvailableModVersions(requiredModVersion.Name, includePrerelease)
if err != nil {
return false, err
}
return i.updateAvailable(requiredModVersion, installedVersion.Version, availableVersions)
}
return false, nil
}
// determine whether there is a newer mod version avoilable which satisfies the dependency version constraint
func (i *ModInstaller) updateAvailable(requiredVersion *modconfig.ModVersionConstraint, currentVersion *semver.Version, availableVersions []*semver.Version) (bool, error) {
latestVersion, err := i.getModRefSatisfyingConstraints(requiredVersion, availableVersions)
if err != nil {
return false, err
}
if latestVersion.Version.GreaterThan(currentVersion) {
return true, nil
}
return false, nil
}
// get the most recent available mod version which satisfies the version constraint
func (i *ModInstaller) getModRefSatisfyingConstraints(modVersion *modconfig.ModVersionConstraint, availableVersions []*semver.Version) (*ResolvedModRef, error) {
// find a version which satisfies the version constraint
var version = getVersionSatisfyingConstraint(modVersion.Constraint, availableVersions)
if version == nil {
return nil, fmt.Errorf("no version of %s found satisfying verison constraint: %s", modVersion.Name, modVersion.Constraint.Original)
}
return NewResolvedModRef(modVersion, version)
}
// install a mod
func (i *ModInstaller) install(dependency *ResolvedModRef, parent *modconfig.Mod) (_ *modconfig.Mod, err error) {
// get the temp location to install the mod to
fullName := dependency.FullName()
tempDestPath := i.getDependencyTmpPath(fullName)
defer func() {
if err == nil {
i.installData.onModInstalled(dependency, parent)
}
}()
// if the target path exists, use the exiting file
// if it does not exist (the usual case), install it
if _, err := os.Stat(tempDestPath); os.IsNotExist(err) {
if err := i.installFromGit(dependency, tempDestPath); err != nil {
return nil, err
}
}
// now load the installed mod and return it
modDef, err := i.loadModfile(tempDestPath, false)
if err != nil {
return nil, err
}
if modDef == nil {
return nil, fmt.Errorf("'%s' has no mod definition file", dependency.FullName())
}
// hack set mod dependency path
if err := i.setModDependencyPath(modDef, i.tmpPath); err != nil {
return nil, err
}
// so we have successfully installed this dependency to the temp location, now copy to the mod location
if !i.dryRun {
destPath := i.getDependencyDestPath(fullName)
if err := i.copyModFromTempToModsFolder(tempDestPath, destPath); err != nil {
return nil, err
}
}
return modDef, nil
}
func (i *ModInstaller) copyModFromTempToModsFolder(tmpPath string, destPath string) error {
if err := os.RemoveAll(destPath); err != nil {
return err
}
if err := copy.Copy(tmpPath, destPath); err != nil {
return err
}
return nil
}
func (i *ModInstaller) installFromGit(dependency *ResolvedModRef, installPath string) error {
// get the mod from git
gitUrl := fmt.Sprintf("https://%s", dependency.Name)
gitUrl := getGitUrl(dependency.Name)
_, err := git.PlainClone(installPath,
false,
&git.CloneOptions{
URL: gitUrl,
//Progress: os.Stdout,
URL: gitUrl,
ReferenceName: dependency.GitReference,
Depth: 1,
SingleBranch: true,
@@ -195,14 +411,54 @@ func (i *ModInstaller) installDependencyFromGit(dependency *ResolvedModRef, inst
return err
}
func (i *ModInstaller) InstallReport() string {
if len(i.InstalledDependencies) == 0 {
return "No dependencies installed"
// build the path of the temp location to copy this depednency to
func (i *ModInstaller) getDependencyTmpPath(dependencyFullName string) string {
return filepath.Join(i.tmpPath, dependencyFullName)
}
// build the path of the temp location to copy this depednency to
func (i *ModInstaller) getDependencyDestPath(dependencyFullName string) string {
return filepath.Join(i.modsPath, dependencyFullName)
}
func (i *ModInstaller) loadDependencyMod(modVersion *version_map.ResolvedVersionConstraint) (*modconfig.Mod, error) {
modPath := i.getDependencyDestPath(modconfig.ModVersionFullName(modVersion.Name, modVersion.Version))
modDef, err := i.loadModfile(modPath, false)
if err != nil {
return nil, err
}
strs := make([]string, len(i.InstalledDependencies))
for idx, dep := range i.InstalledDependencies {
strs[idx] = dep.FullName()
if err := i.setModDependencyPath(modDef, modPath); err != nil {
return nil, err
}
return fmt.Sprintf("\nInstalled %d dependencies:\n - %s\n", len(i.InstalledDependencies), strings.Join(strs, "\n - "))
return modDef, nil
}
// HACK set the mod depdnency path
func (i *ModInstaller) setModDependencyPath(mod *modconfig.Mod, modPath string) (err error) {
mod.ModDependencyPath, err = filepath.Rel(i.modsPath, modPath)
return
}
func (i *ModInstaller) loadModfile(modPath string, createDefault bool) (*modconfig.Mod, error) {
if !parse.ModfileExists(modPath) {
if createDefault {
return modconfig.CreateDefaultMod(i.workspacePath), nil
}
return nil, nil
}
mod, err := parse.ParseModDefinition(modPath)
if err != nil {
return nil, err
}
return mod, nil
}
func (i *ModInstaller) updating() bool {
return i.command == "update"
}
func (i *ModInstaller) uninstalling() bool {
return i.command == "uninstall"
}

63
mod_installer/mod_installer_args.go Normal file
View File

@@ -0,0 +1,63 @@
package mod_installer
import (
"fmt"
"github.com/turbot/steampipe/steampipeconfig/modconfig"
"github.com/turbot/steampipe/steampipeconfig/version_map"
"github.com/turbot/steampipe/utils"
)
func (i *ModInstaller) GetRequiredModVersionsFromArgs(modsArgs []string) (version_map.VersionConstraintMap, error) {
var errors []error
mods := make(version_map.VersionConstraintMap, len(modsArgs))
for _, modArg := range modsArgs {
// create mod version from arg
modVersion, err := modconfig.NewModVersionConstraint(modArg)
if err != nil {
errors = append(errors, err)
continue
}
// if we are updating there are a few checks we need to make
if i.updating() {
modVersion, err = i.getUpdateVersion(modArg, modVersion)
if err != nil {
errors = append(errors, err)
continue
}
}
if i.uninstalling() {
// it is not valid to specify a mod version for uninstall
if modVersion.HasVersion() {
errors = append(errors, fmt.Errorf("invalid arg '%s' - cannot specify a version when uninstalling", modArg))
continue
}
}
mods[modVersion.Name] = modVersion
}
if len(errors) > 0 {
return nil, utils.CombineErrors(errors...)
}
return mods, nil
}
func (i *ModInstaller) getUpdateVersion(modArg string, modVersion *modconfig.ModVersionConstraint) (*modconfig.ModVersionConstraint, error) {
// verify the mod is already installed
if i.installData.Lock.GetMod(modVersion.Name, i.workspaceMod) == nil {
return nil, fmt.Errorf("cannot update '%s' as it is not installed", modArg)
}
// find the current dependency with this mod name
// - this is what we will be using, to ensure we keep the same version constraint
currentDependency := i.workspaceMod.GetModDependency(modVersion.Name)
if currentDependency == nil {
return nil, fmt.Errorf("cannot update '%s' as it is not a dependency of this workspace", modArg)
}
// it is not valid to specify a mod version - we will set the constraint from the modfile
if modVersion.HasVersion() {
return nil, fmt.Errorf("invalid arg '%s' - cannot specify a version when updating", modArg)
}
return currentDependency, nil
}

42
mod_installer/mod_installer_prune.go Normal file
View File

@@ -0,0 +1,42 @@
package mod_installer
import (
"os"
"path/filepath"
"github.com/turbot/steampipe/steampipeconfig/modconfig"
"github.com/turbot/steampipe/steampipeconfig/version_map"
)
func (i *ModInstaller) Prune() (version_map.VersionListMap, error) {
unusedMods := i.installData.Lock.GetUnreferencedMods()
// now delete any mod folders which are not in the lock file
for name, versions := range unusedMods {
for _, version := range versions {
depPath := i.getDependencyDestPath(modconfig.ModVersionFullName(name, version))
if err := i.deleteDependencyItem(depPath); err != nil {
return nil, err
}
}
}
return unusedMods, nil
}
func (i *ModInstaller) deleteDependencyItem(depPath string) error {
if err := os.RemoveAll(depPath); err != nil {
return err
}
return i.deleteEmptyFolderTree(filepath.Dir(depPath))
}
func (i *ModInstaller) deleteEmptyFolderTree(folderPath string) error {
// if the parent folder is empty, delete it
err := os.Remove(folderPath)
if err == nil {
parent := filepath.Dir(folderPath)
return i.deleteEmptyFolderTree(parent)
}
return nil
}

18
mod_installer/mod_installer_test.go Normal file
View File

@@ -0,0 +1,18 @@
package mod_installer
import (
"fmt"
"testing"
"github.com/Masterminds/semver"
)
func TestModInstaller(t *testing.T) {
cs, err := semver.NewConstraint("^3")
v, _ := semver.NewVersion("3.1")
res := cs.Check(v)
fmt.Println(res)
fmt.Println(cs)
fmt.Println(err)
}

53
mod_installer/mod_ref.go
View File

@@ -1,53 +0,0 @@
package mod_installer
import (
"fmt"
"strings"
goVersion "github.com/hashicorp/go-version"
)
// ModRef is a struct to represent an unresolved mod reference
type ModRef struct {
// the Git URL of the mod repo
Name string
// the version constraint of the mod
versionConstraint *goVersion.Version
// the branch to use
branch string
// the local file location to use
filePath string
// raw reference
raw string
}
func NewModRef(modRef string) (*ModRef, error) {
split := strings.Split(modRef, "@")
if len(split) > 2 {
return nil, fmt.Errorf("invalid mod ref %s", modRef)
}
res := &ModRef{
raw: modRef,
Name: split[0],
}
if len(split) == 2 {
res.setVersion(split[1])
}
return res, nil
}
func (r *ModRef) setVersion(versionString string) {
if strings.HasPrefix(versionString, "file:") {
r.filePath = versionString
return
}
// does the verison parse as a semver version
if v, err := goVersion.NewVersion(versionString); err == nil {
r.versionConstraint = v
return
}
// otherwise assume it is a branch
r.branch = versionString
}

60
mod_installer/resolved_mod_ref.go
View File

@@ -1,73 +1,49 @@
package mod_installer
import (
"fmt"
"github.com/Masterminds/semver"
"github.com/go-git/go-git/v5/plumbing"
goVersion "github.com/hashicorp/go-version"
"github.com/turbot/steampipe/steampipeconfig/modconfig"
"github.com/turbot/steampipe/version_helpers"
)
// ResolvedModRef is a struct to represent a resolved mod reference
// ResolvedModRef is a struct to represent a resolved mod git reference
type ResolvedModRef struct {
// the FQN of the mod - also the Git URL of the mod repo
Name string
// the mod version
Version *semver.Version
// the vestion constraint
Constraint *version_helpers.Constraints
// the Git branch/tag
GitReference plumbing.ReferenceName
// the monotonic version - may be unknown for local or branch
// although version will be monotonic, we can still use semver
Version *goVersion.Version
// the file path for local mods
FilePath string
}
func NewResolvedModRef(modVersion *modconfig.ModVersion) (*ResolvedModRef, error) {
func NewResolvedModRef(requiredModVersion *modconfig.ModVersionConstraint, version *semver.Version) (*ResolvedModRef, error) {
res := &ResolvedModRef{
Name: modVersion.Name,
Name: requiredModVersion.Name,
Version: version,
Constraint: requiredModVersion.Constraint,
// this may be empty strings
FilePath: modVersion.FilePath,
FilePath: requiredModVersion.FilePath,
}
if res.FilePath == "" {
// NOTE we currently only support explicit (i.e. minor) versions
// if the mod version has either a version constraint or branch, set the git ref
res.SetGitReference(modVersion)
res.setGitReference()
}
return res, nil
}
func (r *ResolvedModRef) SetGitReference(modVersion *modconfig.ModVersion) {
func (r *ResolvedModRef) setGitReference() {
// TODO handle branches
if modVersion.Branch != "" {
r.GitReference = plumbing.NewBranchReferenceName(modVersion.Branch)
// NOTE: we need to set version from branch
return
}
// so there is a version constraint
// NOTE: if it is just a major constraint, we need to find the latest version in the major
// for now assume it is a full version
// NOTE: we cannot just ToString the version as we need the 'v' at the beginning
r.GitReference = plumbing.NewTagReferenceName(modVersion.VersionString)
r.Version = modVersion.VersionConstraint
// NOTE: use the original version string - this will be the tag name
r.GitReference = plumbing.NewTagReferenceName(r.Version.Original())
}
// FullName returns name in the format <dependency name>@v<dependencyVersion>
func (r *ResolvedModRef) FullName() string {
segments := r.Version.Segments()
return fmt.Sprintf("%s@v%d.%d", r.Name, segments[0], segments[1])
}
// SatisfiesVersionConstraint return whether this resolved ref satisfies a version constraint
func (r *ResolvedModRef) SatisfiesVersionConstraint(versionConstraint *goVersion.Version) bool {
// if we do not have a version set, then we cannot satisfy a version constraint
// this may happen if we are a local file or unversioned branch
if r.Version == nil {
return false
}
return r.Version.GreaterThanOrEqual(versionConstraint)
return modconfig.ModVersionFullName(r.Name, r.Version)
}

97
mod_installer/summary_builder.go Normal file
View File

@@ -0,0 +1,97 @@
package mod_installer
import (
"fmt"
"github.com/spf13/viper"
"github.com/turbot/steampipe/constants"
"github.com/turbot/steampipe/steampipeconfig/version_map"
"github.com/turbot/steampipe/utils"
)
const (
VerbInstalled = "Installed"
VerbUninstalled = "Uninstalled"
VerbUpgraded = "Upgraded"
VerbDowngraded = "Downgraded"
VerbPruned = "Pruned"
)
var dryRunVerbs = map[string]string{
VerbInstalled: "Would install",
VerbUninstalled: "Would uninstall",
VerbUpgraded: "Would upgrade",
VerbDowngraded: "Would downgrade",
VerbPruned: "Would prune",
}
func getVerb(verb string) string {
if viper.GetBool(constants.ArgDryRun) {
verb = dryRunVerbs[verb]
}
return verb
}
func BuildInstallSummary(installData *InstallData) string {
// for now treat an install as update - we only install deps which are in the mod.sp but missing in the mod folder
modDependencyPath := installData.WorkspaceMod.GetModDependencyPath()
installCount, installedTreeString := getInstallationResultString(installData.Installed, modDependencyPath)
uninstallCount, uninstalledTreeString := getInstallationResultString(installData.Uninstalled, modDependencyPath)
upgradeCount, upgradeTreeString := getInstallationResultString(installData.Upgraded, modDependencyPath)
downgradeCount, downgradeTreeString := getInstallationResultString(installData.Downgraded, modDependencyPath)
var installString, upgradeString, downgradeString, uninstallString string
if installCount > 0 {
verb := getVerb(VerbInstalled)
installString = fmt.Sprintf("\n%s %d %s:\n\n%s\n", verb, installCount, utils.Pluralize("mod", installCount), installedTreeString)
}
if uninstallCount > 0 {
verb := getVerb(VerbUninstalled)
uninstallString = fmt.Sprintf("\n%s %d %s:\n\n%s\n", verb, uninstallCount, utils.Pluralize("mod", uninstallCount), uninstalledTreeString)
}
if upgradeCount > 0 {
verb := getVerb(VerbUpgraded)
upgradeString = fmt.Sprintf("\n%s %d %s:\n\n%s\n", verb, upgradeCount, utils.Pluralize("mod", upgradeCount), upgradeTreeString)
}
if downgradeCount > 0 {
verb := getVerb(VerbDowngraded)
downgradeString = fmt.Sprintf("\n%s %d %s:\n\n%s\n", verb, downgradeCount, utils.Pluralize("mod", downgradeCount), downgradeTreeString)
}
if installCount+uninstallCount+upgradeCount+downgradeCount == 0 {
if len(installData.Lock.InstallCache) == 0 {
return "No mods are installed"
}
return "All mods are up to date"
}
return fmt.Sprintf("%s%s%s%s", installString, upgradeString, downgradeString, uninstallString)
}
func getInstallationResultString(items version_map.DependencyVersionMap, modDependencyPath string) (int, string) {
var res string
count := len(items.FlatMap())
if count > 0 {
tree := items.GetDependencyTree(modDependencyPath)
res = tree.String()
}
return count, res
}
func BuildUninstallSummary(installData *InstallData) string {
// for now treat an install as update - we only install deps which are in the mod.sp but missing in the mod folder
uninstallCount := len(installData.Uninstalled.FlatMap())
if uninstallCount == 0 {
return "Nothing uninstalled"
}
uninstalledTree := installData.GetUninstalledTree()
verb := getVerb(VerbUninstalled)
return fmt.Sprintf("\n%s %d %s:\n\n%s", verb, uninstallCount, utils.Pluralize("mod", uninstallCount), uninstalledTree.String())
}
func BuildPruneSummary(pruned version_map.VersionListMap) string {
pruneCount := len(pruned.FlatMap())
verb := getVerb(VerbPruned)
return fmt.Sprintf("\n%s %d %s:\n", verb, pruneCount, utils.Pluralize("mod", pruneCount))
}

8
mod_installer/test_data/mods/dep1/mod.sp
View File

@@ -1,7 +1,7 @@
mod "dep1"{
requires {
mod "github.com/kaidaguerre/steampipe-mod-m2" {
version = "v1.0"
mod "dep1" {
require {
mod "github.com/turbot/steampipe-mod-aws-compliance" {
version = "0"
}
}
}

4
mod_installer/test_data/mods/dep2/control.sp Normal file
View File

@@ -0,0 +1,4 @@
control "c1"{
description = "control 1"
query = m2.query.m2_q1
}

7
mod_installer/test_data/mods/dep2/mod.sp Normal file
View File

@@ -0,0 +1,7 @@
mod "dep2" {
require {
mod "github.com/kaidaguerre/steampipe-mod-m2" {
version = "latest"
}
}
}

4
mod_installer/test_data/mods/dep3/control.sp Normal file
View File

@@ -0,0 +1,4 @@
control "c1"{
description = "control 1"
query = m2.query.m2_q1
}

10
mod_installer/test_data/mods/dep3/mod.sp Normal file
View File

@@ -0,0 +1,10 @@
mod "dep3"{
require {
mod "github.com/kaidaguerre/steampipe-mod-m1" {
version = "v1.*"
}
mod "github.com/kaidaguerre/steampipe-mod-m2" {
version = "v3.1"
}
}
}

4
mod_installer/test_data/mods/dep4/control.sp Normal file
View File

@@ -0,0 +1,4 @@
control "c1"{
description = "control 1"
query = m2.query.m2_q1
}

10
mod_installer/test_data/mods/dep4/mod.sp Normal file
View File

@@ -0,0 +1,10 @@
mod "dep4"{
require {
mod "github.com/kaidaguerre/steampipe-mod-m1" {
version = "v1.1"
}
mod "github.com/kaidaguerre/steampipe-mod-m2" {
version = "v3.0"
}
}
}

4
mod_installer/test_data/mods/dep5/control.sp Normal file
View File

@@ -0,0 +1,4 @@
control "c1"{
description = "control 1"
query = m2.query.m2_q1
}

10
mod_installer/test_data/mods/dep5/mod.sp Normal file
View File

@@ -0,0 +1,10 @@
mod "dep5"{
require {
mod "github.com/kaidaguerre/steampipe-mod-m1" {
version = "v1.*"
}
mod "github.com/kaidaguerre/steampipe-mod-m2" {
version = "v3.2"
}
}
}

4
mod_installer/test_data/mods/dep6_x/control.sp Normal file
View File

@@ -0,0 +1,4 @@
control "c1"{
description = "control 1"
query = m2.query.m2_q1
}

10
mod_installer/test_data/mods/dep6_x/mod.sp Normal file
View File

@@ -0,0 +1,10 @@
mod "dep6"{
require {
mod "github.com/kaidaguerre/steampipe-mod-m1" {
version = "v1.*"
}
mod "github.com/kaidaguerre/steampipe-mod-m2" {
version = "v3.3"
}
}
}

4
mod_installer/test_data/mods/dep7_x/control.sp Normal file
View File

@@ -0,0 +1,4 @@
control "c1"{
description = "control 1"
query = m2.query.m2_q1
}

10
mod_installer/test_data/mods/dep7_x/mod.sp Normal file
View File

@@ -0,0 +1,10 @@
mod "dep7"{
require {
mod "github.com/kaidaguerre/steampipe-mod-m1" {
version = "v2.*"
}
mod "github.com/kaidaguerre/steampipe-mod-m2" {
version = "v3.0"
}
}
}

4
mod_installer/test_data/mods/dep8_x/control.sp Normal file
View File

@@ -0,0 +1,4 @@
control "c1"{
description = "control 1"
query = m2.query.m2_q1
}

10
mod_installer/test_data/mods/dep8_x/mod.sp Normal file
View File

@@ -0,0 +1,10 @@
mod "dep8"{
require {
mod "github.com/kaidaguerre/steampipe-mod-m1" {
version = "v1.0"
}
mod "github.com/kaidaguerre/steampipe-mod-m2" {
version = "v3.2"
}
}
}

8
mod_installer/test_data/mods/dep_empty/c.sp
View File

@@ -1,8 +0,0 @@
benchmark "my_mod_public_resources" {
title = "Public Resources"
description = "Resources that are public."
children = [
aws_compliance.benchmark.cis_v140_1,
]
}

29
mod_installer/uninstall.go Normal file
View File

@@ -0,0 +1,29 @@
package mod_installer
import (
"github.com/turbot/go-kit/helpers"
"github.com/turbot/steampipe/utils"
)
func UninstallWorkspaceDependencies(opts *InstallOpts) (*InstallData, error) {
utils.LogTime("cmd.UninstallWorkspaceDependencies")
defer func() {
utils.LogTime("cmd.UninstallWorkspaceDependencies end")
if r := recover(); r != nil {
utils.ShowError(helpers.ToError(r))
}
}()
// uninstall workspace dependencies
installer, err := NewModInstaller(opts)
if err != nil {
return nil, err
}
if err := installer.UninstallWorkspaceDependencies(); err != nil {
return nil, err
}
return installer.installData, nil
}

2
steampipeconfig/connection_plugin.go
View File

@@ -34,7 +34,7 @@ type ConnectionPlugin struct {
}
// CreateConnectionPlugins instantiates plugins for specified connections, fetches schemas and sends connection config
func CreateConnectionPlugins(connections []*modconfig.Connection) (connectionPluginMap map[string]*ConnectionPlugin, err error) {
func CreateConnectionPlugins(connections ...*modconfig.Connection) (connectionPluginMap map[string]*ConnectionPlugin, err error) {
log.Printf("[TRACE] CreateConnectionPlugin creating %d connections", len(connections))
// build result map

4
steampipeconfig/connection_updates.go
View File

@@ -140,7 +140,7 @@ func (u *ConnectionUpdates) populateConnectionPlugins(alreadyCreatedConnectionPl
// - remove these from list of plugins to create
connectionsToCreate := removeConnectionsFromList(updateConnections, alreadyCreatedConnectionPlugins)
// now crerate them
connectionPlugins, err := CreateConnectionPlugins(connectionsToCreate)
connectionPlugins, err := CreateConnectionPlugins(connectionsToCreate...)
if err != nil {
return err
}
@@ -186,7 +186,7 @@ func getSchemaHashesForDynamicSchemas(requiredConnectionData ConnectionDataMap,
}
}
connectionsPluginsWithDynamicSchema, err := CreateConnectionPlugins(connectionsWithDynamicSchema.Connections())
connectionsPluginsWithDynamicSchema, err := CreateConnectionPlugins(connectionsWithDynamicSchema.Connections()...)
if err != nil {
return nil, nil, err
}

86
steampipeconfig/load_mod.go
View File

@@ -9,7 +9,7 @@ import (
"github.com/turbot/steampipe/utils"
goVersion "github.com/hashicorp/go-version"
"github.com/Masterminds/semver"
filehelpers "github.com/turbot/go-kit/files"
"github.com/turbot/go-kit/helpers"
@@ -23,13 +23,29 @@ import (
// if CreatePseudoResources flag is set, construct hcl resources for files with specific extensions
// NOTE: it is an error if there is more than 1 mod defined, however zero mods is acceptable
// - a default mod will be created assuming there are any resource files
func LoadMod(modPath string, runCtx *parse.RunContext) (mod *modconfig.Mod, err error) {
func LoadMod(modPath string, parentRunCtx *parse.RunContext) (mod *modconfig.Mod, err error) {
defer func() {
if r := recover(); r != nil {
err = helpers.ToError(r)
}
}()
runCtx := parentRunCtx
// if this it not the root mod, create child a run context for its evaluation
if modPath != parentRunCtx.RootEvalPath {
runCtx = parse.NewRunContext(
parentRunCtx.WorkspaceLock,
modPath,
parse.CreatePseudoResources,
&filehelpers.ListOptions{
// listFlag specifies whether to load files recursively
Flags: filehelpers.FilesRecursive,
// only load .sp files
Include: filehelpers.InclusionsFromExtensions([]string{constants.ModDataExtension}),
})
runCtx.BlockTypes = parentRunCtx.BlockTypes
}
// verify the mod folder exists
if _, err := os.Stat(modPath); os.IsNotExist(err) {
return nil, fmt.Errorf("mod folder %s does not exist", modPath)
@@ -87,21 +103,43 @@ func LoadMod(modPath string, runCtx *parse.RunContext) (mod *modconfig.Mod, err
return nil, err
}
// now add fully populated mod to the parent run context
if modPath != parentRunCtx.RootEvalPath {
parentRunCtx.CurrentMod = mod
parentRunCtx.AddMod(mod)
}
return mod, err
}
func loadModDependencies(mod *modconfig.Mod, runCtx *parse.RunContext) error {
var errors []error
if mod.Requires != nil {
for _, dependencyMod := range mod.Requires.Mods {
if mod.Require != nil {
// now ensure there is a lock file - if we have any mod dependnecies there MUST be a lock file -
// otherwise 'steampipe install' must be run
if err := runCtx.EnsureWorkspaceLock(mod); err != nil {
return err
}
for _, requiredModVersion := range mod.Require.Mods {
// if we have a locked version, update the required version to reflect this
lockedVersion, err := runCtx.WorkspaceLock.GetLockedModVersionConstraint(requiredModVersion, mod)
if err != nil {
errors = append(errors, err)
continue
}
if lockedVersion != nil {
requiredModVersion = lockedVersion
}
// have we already loaded a mod which satisfied this
if loadedMod, ok := runCtx.LoadedDependencyMods[dependencyMod.Name]; ok {
if loadedMod.Version.GreaterThanOrEqual(dependencyMod.VersionConstraint) {
if loadedMod, ok := runCtx.LoadedDependencyMods[requiredModVersion.Name]; ok {
if requiredModVersion.Constraint.Check(loadedMod.Version) {
continue
}
}
if err := loadModDependency(dependencyMod, runCtx); err != nil {
if err := loadModDependency(requiredModVersion, runCtx); err != nil {
errors = append(errors, err)
}
}
@@ -109,17 +147,16 @@ func loadModDependencies(mod *modconfig.Mod, runCtx *parse.RunContext) error {
return utils.CombineErrors(errors...)
}
func loadModDependency(modDependency *modconfig.ModVersion, runCtx *parse.RunContext) error {
func loadModDependency(modDependency *modconfig.ModVersionConstraint, runCtx *parse.RunContext) error {
// dependency mods are installed to <mod path>/<mod nam>@version
// for example workspace_folder/.steampipe/mods/github.com/turbot/steampipe-mod-aws-compliance@v1.0
// we need to list all mod folder in the parent folder: workspace_folder/.steampipe/mods/github.com/turbot/
// for each folder we parse the mod name and version and determine whether it meets the version constraint
// we need to iterate through all mods in the parent folder and find one that sarifies requirements
parentFolder := filepath.Dir(filepath.Join(runCtx.ModInstallationPath, modDependency.Name))
// get th elast segment of mod name
// we need to iterate through all mods in the parent folder and find one that satisfies requirements
parentFolder := filepath.Dir(filepath.Join(runCtx.WorkspaceLock.ModInstallationPath, modDependency.Name))
// get the last segment of mod name
dependencyPath, version, err := findInstalledDependency(modDependency, parentFolder)
if err != nil {
return err
@@ -146,13 +183,17 @@ func loadModDependency(modDependency *modconfig.ModVersion, runCtx *parse.RunCon
}
func findInstalledDependency(modDependency *modconfig.ModVersion, parentFolder string) (string, *goVersion.Version, error) {
func findInstalledDependency(modDependency *modconfig.ModVersionConstraint, parentFolder string) (string, *semver.Version, error) {
shortDepName := filepath.Base(modDependency.Name)
entries, err := os.ReadDir(parentFolder)
if err != nil {
return "", nil, fmt.Errorf("mod dependency %s is not installed", modDependency.Name)
return "", nil, fmt.Errorf("mod satisfying '%s' is not installed", modDependency)
}
// results vars
var dependencyPath string
var dependencyVersion *semver.Version
for _, entry := range entries {
split := strings.Split(entry.Name(), "@")
if len(split) != 2 {
@@ -162,19 +203,28 @@ func findInstalledDependency(modDependency *modconfig.ModVersion, parentFolder s
modName := split[0]
versionString := strings.TrimPrefix(split[1], "v")
if modName == shortDepName {
v, err := goVersion.NewVersion(versionString)
v, err := semver.NewVersion(versionString)
if err != nil {
// invalid format - ignore
continue
}
if v.GreaterThanOrEqual(modDependency.VersionConstraint) {
return filepath.Join(parentFolder, entry.Name()), v, nil
if modDependency.Constraint.Check(v) {
// if there is more than 1 mod which satisfied the dependency, fail (for now)
if dependencyVersion != nil {
return "", nil, fmt.Errorf("more than one mod found which satisfies dependency %s@%s", modDependency.Name, modDependency.VersionString)
}
dependencyPath = filepath.Join(parentFolder, entry.Name())
dependencyVersion = v
}
}
}
return "", nil, fmt.Errorf("mod dependency %s is not installed", modDependency.Name)
// did we find a result?
if dependencyVersion != nil {
return dependencyPath, dependencyVersion, nil
}
return "", nil, fmt.Errorf("mod satisfying '%s' is not installed", modDependency)
}
// LoadModResourceNames parses all hcl files in modPath and returns the names of all resources

10
steampipeconfig/load_mod_test.go
View File

@@ -428,9 +428,9 @@ Benchmarks:
expected: &modconfig.Mod{
ShortName: "m1",
FullName: "mod.m1",
Requires: &modconfig.Requires{
Require: &modconfig.Require{
SteampipeVersionString: "v0.8.0",
Mods: []*modconfig.ModVersion{
Mods: []*modconfig.ModVersionConstraint{
{
Name: "github.com/turbot/aws-core",
VersionString: "v1.0",
@@ -444,13 +444,13 @@ Benchmarks:
expected: &modconfig.Mod{
ShortName: "m1",
FullName: "mod.m1",
Requires: &modconfig.Requires{
Require: &modconfig.Require{
SteampipeVersionString: "v0.8.0",
Mods: []*modconfig.ModVersion{
Mods: []*modconfig.ModVersionConstraint{
{
Name: "github.com/turbot/aws-core",
VersionString: "v1.0",
Alias: utils.ToStringPointer("core"),
//Alias: utils.ToStringPointer("core"),
},
},
},

59
steampipeconfig/modconfig/benchmark.go
View File

@@ -24,11 +24,11 @@ type Benchmark struct {
ShortName string
FullName string `cty:"name"`
ChildNames *[]NamedItem `cty:"children" hcl:"children"`
Description *string `cty:"description" hcl:"description" column:"description,text"`
Documentation *string `cty:"documentation" hcl:"documentation" column:"documentation,text"`
Tags *map[string]string `cty:"tags" hcl:"tags" column:"tags,jsonb"`
Title *string `cty:"title" hcl:"title" column:"title,text"`
ChildNames []NamedItem `cty:"children" hcl:"children,optional"`
Description *string `cty:"description" hcl:"description" column:"description,text"`
Documentation *string `cty:"documentation" hcl:"documentation" column:"documentation,text"`
Tags map[string]string `cty:"tags" hcl:"tags,optional" column:"tags,jsonb"`
Title *string `cty:"title" hcl:"title" column:"title,text"`
// list of all block referenced by the resource
References []*ResourceReference
@@ -37,16 +37,18 @@ type Benchmark struct {
ChildNameStrings []string `column:"children,jsonb"`
DeclRange hcl.Range
parents []ModTreeItem
children []ModTreeItem
metadata *ResourceMetadata
parents []ModTreeItem
children []ModTreeItem
metadata *ResourceMetadata
UnqualifiedName string
}
func NewBenchmark(block *hcl.Block) *Benchmark {
return &Benchmark{
ShortName: block.Labels[0],
FullName: fmt.Sprintf("benchmark.%s", block.Labels[0]),
DeclRange: block.DefRange,
ShortName: block.Labels[0],
FullName: fmt.Sprintf("benchmark.%s", block.Labels[0]),
UnqualifiedName: fmt.Sprintf("benchmark.%s", block.Labels[0]),
DeclRange: block.DefRange,
}
}
@@ -60,20 +62,13 @@ func (b *Benchmark) Equals(other *Benchmark) bool {
return res
}
// tags
if b.Tags == nil {
if other.Tags != nil {
if len(b.Tags) != len(other.Tags) {
return false
}
for k, v := range b.Tags {
if otherVal := other.Tags[k]; v != otherVal {
return false
}
} else {
// we have tags
if other.Tags == nil {
return false
}
for k, v := range *b.Tags {
if otherVal, ok := (*other.Tags)[k]; !ok && v != otherVal {
return false
}
}
}
if len(b.ChildNameStrings) != len(other.ChildNameStrings) {
@@ -100,14 +95,14 @@ func (b *Benchmark) GetDeclRange() *hcl.Range {
// OnDecoded implements HclResource
func (b *Benchmark) OnDecoded(block *hcl.Block) hcl.Diagnostics {
var res hcl.Diagnostics
if b.ChildNames == nil || len(*b.ChildNames) == 0 {
if len(b.ChildNames) == 0 {
return nil
}
// validate each child name appears only once
nameMap := make(map[string]bool)
b.ChildNameStrings = make([]string, len(*b.ChildNames))
for i, n := range *b.ChildNames {
b.ChildNameStrings = make([]string, len(b.ChildNames))
for i, n := range b.ChildNames {
if nameMap[n.Name] {
res = append(res, &hcl.Diagnostic{
Severity: hcl.DiagError,
@@ -120,6 +115,7 @@ func (b *Benchmark) OnDecoded(block *hcl.Block) hcl.Diagnostics {
nameMap[n.Name] = true
}
// in order to populate th echildren in the order specified, we create an empty array and populate by index in AddChild
b.children = make([]ModTreeItem, len(b.ChildNameStrings))
return res
}
@@ -132,6 +128,8 @@ func (b *Benchmark) AddReference(ref *ResourceReference) {
// SetMod implements HclResource
func (b *Benchmark) SetMod(mod *Mod) {
b.Mod = mod
b.UnqualifiedName = b.FullName
b.FullName = fmt.Sprintf("%s.%s", mod.ShortName, b.FullName)
}
// GetMod implements HclResource
@@ -222,7 +220,7 @@ func (b *Benchmark) GetDescription() string {
// GetTags implements ModTreeItem
func (b *Benchmark) GetTags() map[string]string {
if b.Tags != nil {
return *b.Tags
return b.Tags
}
return map[string]string{}
}
@@ -244,7 +242,7 @@ func (b *Benchmark) GetPaths() []NodePath {
}
// Name implements ModTreeItem, HclResource, ResourceWithMetadata
// return name in format: 'control.<shortName>'
// return name in format: '<modname>.control.<shortName>'
func (b *Benchmark) Name() string {
return b.FullName
}
@@ -258,8 +256,3 @@ func (b *Benchmark) GetMetadata() *ResourceMetadata {
func (b *Benchmark) SetMetadata(metadata *ResourceMetadata) {
b.metadata = metadata
}
// QualifiedName returns the name in format: '<modName>.control.<shortName>'
func (b *Benchmark) QualifiedName() string {
return fmt.Sprintf("%s.%s", b.metadata.ModName, b.FullName)
}

58
steampipeconfig/modconfig/control.go
View File

@@ -14,15 +14,15 @@ import (
// Control is a struct representing the Control resource
type Control struct {
ShortName string
FullName string `cty:"name"`
Description *string `cty:"description" column:"description,text"`
Documentation *string `cty:"documentation" column:"documentation,text"`
SearchPath *string `cty:"search_path" column:"search_path,text"`
SearchPathPrefix *string `cty:"search_path_prefix" column:"search_path_prefix,text"`
Severity *string `cty:"severity" column:"severity,text"`
SQL *string `cty:"sql" column:"sql,text"`
Tags *map[string]string `cty:"tags" column:"tags,jsonb"`
Title *string `cty:"title" column:"title,text"`
FullName string `cty:"name"`
Description *string `cty:"description" column:"description,text"`
Documentation *string `cty:"documentation" column:"documentation,text"`
SearchPath *string `cty:"search_path" column:"search_path,text"`
SearchPathPrefix *string `cty:"search_path_prefix" column:"search_path_prefix,text"`
Severity *string `cty:"severity" column:"severity,text"`
SQL *string `cty:"sql" column:"sql,text"`
Tags map[string]string `cty:"tags" column:"tags,jsonb"`
Title *string `cty:"title" column:"title,text"`
Query *Query
// args
// arguments may be specified by either a map of named args or as a list of positional args
@@ -39,14 +39,16 @@ type Control struct {
parents []ModTreeItem
metadata *ResourceMetadata
PreparedStatementName string `column:"prepared_statement_name,text"`
UnqualifiedName string
}
func NewControl(block *hcl.Block) *Control {
control := &Control{
ShortName: block.Labels[0],
FullName: fmt.Sprintf("control.%s", block.Labels[0]),
DeclRange: block.DefRange,
Args: NewQueryArgs(),
ShortName: block.Labels[0],
FullName: fmt.Sprintf("control.%s", block.Labels[0]),
UnqualifiedName: fmt.Sprintf("control.%s", block.Labels[0]),
DeclRange: block.DefRange,
Args: NewQueryArgs(),
}
return control
}
@@ -64,21 +66,13 @@ func (c *Control) Equals(other *Control) bool {
if !res {
return res
}
// tags
if c.Tags == nil {
if other.Tags != nil {
if len(c.Tags) != len(other.Tags) {
return false
}
for k, v := range c.Tags {
if otherVal := other.Tags[k]; v != otherVal {
return false
}
} else {
// we have tags
if other.Tags == nil {
return false
}
for k, v := range *c.Tags {
if otherVal, ok := (*other.Tags)[k]; !ok && v != otherVal {
return false
}
}
}
// args
@@ -194,7 +188,7 @@ func (c *Control) GetDescription() string {
// GetTags implements ModTreeItem
func (c *Control) GetTags() map[string]string {
if c.Tags != nil {
return *c.Tags
return c.Tags
}
return map[string]string{}
}
@@ -210,9 +204,9 @@ func (c *Control) Name() string {
return c.FullName
}
// QualifiedName returns the name in format: '<modName>.control.<shortName>'
func (c *Control) QualifiedName() string {
return fmt.Sprintf("%s.%s", c.metadata.ModName, c.FullName)
// QualifiedNameWithVersion returns the name in format: '<modName>@version.control.<shortName>'
func (q *Control) QualifiedNameWithVersion() string {
return fmt.Sprintf("%s.%s", q.Mod.NameWithVersion(), q.FullName)
}
// GetPaths implements ModTreeItem
@@ -242,6 +236,8 @@ func (c *Control) AddReference(ref *ResourceReference) {
// SetMod implements HclResource
func (c *Control) SetMod(mod *Mod) {
c.Mod = mod
c.UnqualifiedName = c.FullName
c.FullName = fmt.Sprintf("%s.%s", mod.ShortName, c.FullName)
}
// GetMod implements HclResource
@@ -280,5 +276,5 @@ func (c *Control) GetPreparedStatementName() string {
// ModName implements QueryProvider
func (c *Control) ModName() string {
return c.Mod.ShortName
return c.Mod.NameWithVersion()
}

1
steampipeconfig/modconfig/interfaces.go
View File

@@ -29,6 +29,7 @@ type ModTreeItem interface {
GetTags() map[string]string
// GetPaths returns an array resource paths
GetPaths() []NodePath
GetMod() *Mod
}
// HclResource must be implemented by resources defined in HCL

1
steampipeconfig/modconfig/local.go
View File

@@ -51,6 +51,7 @@ func (l *Local) AddReference(*ResourceReference) {}
// SetMod implements HclResource
func (l *Local) SetMod(mod *Mod) {
l.Mod = mod
l.FullName = fmt.Sprintf("%s.%s", mod.ShortName, l.FullName)
}
// GetMod implements HclResource

489
steampipeconfig/modconfig/mod.go
View File

@@ -3,15 +3,18 @@ package modconfig
import (
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
goVersion "github.com/hashicorp/go-version"
"github.com/Masterminds/semver"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclwrite"
"github.com/turbot/go-kit/helpers"
"github.com/turbot/go-kit/types"
typehelpers "github.com/turbot/go-kit/types"
"github.com/turbot/steampipe/constants"
"github.com/zclconf/go-cty/cty"
)
@@ -30,22 +33,24 @@ type Mod struct {
ModDependencyPath string `cty:"mod_dependency_path"`
// attributes
Categories *[]string `cty:"categories" hcl:"categories" column:"categories,jsonb"`
Color *string `cty:"color" hcl:"color" column:"color,text"`
Description *string `cty:"description" hcl:"description" column:"description,text"`
Documentation *string `cty:"documentation" hcl:"documentation" column:"documentation,text"`
Icon *string `cty:"icon" hcl:"icon" column:"icon,text"`
Tags *map[string]string `cty:"tags" hcl:"tags" column:"tags,jsonb"`
Title *string `cty:"title" hcl:"title" column:"title,text"`
Categories []string `cty:"categories" hcl:"categories,optional" column:"categories,jsonb"`
Color *string `cty:"color" hcl:"color" column:"color,text"`
Description *string `cty:"description" hcl:"description" column:"description,text"`
Documentation *string `cty:"documentation" hcl:"documentation" column:"documentation,text"`
Icon *string `cty:"icon" hcl:"icon" column:"icon,text"`
Tags map[string]string `cty:"tags" hcl:"tags,optional" column:"tags,jsonb"`
Title *string `cty:"title" hcl:"title" column:"title,text"`
// list of all blocks referenced by the resource
References []*ResourceReference
// blocks
Requires *Requires `hcl:"requires,block"`
OpenGraph *OpenGraph `hcl:"opengraph,block" column:"open_graph,jsonb"`
Require *Require `hcl:"require,block"`
LegacyRequire *Require `hcl:"requires,block"`
OpenGraph *OpenGraph `hcl:"opengraph,block" column:"open_graph,jsonb"`
Version *goVersion.Version
VersionString string `cty:"version"`
Version *semver.Version
Queries map[string]*Query
Controls map[string]*Control
@@ -55,34 +60,49 @@ type Mod struct {
Variables map[string]*Variable
Locals map[string]*Local
// flat list of all resources
AllResources map[string]HclResource
// list of benchmark names, sorted alphabetically
benchmarksOrdered []string
// ModPath is the installation location of the mod
ModPath string
DeclRange hcl.Range
// all children as an array of hcl resources - built before the 'children' array
flatChildren []HclResource
// array of direct mod children - excluds resources which are children of othe rresources
children []ModTreeItem
metadata *ResourceMetadata
}
func NewMod(shortName, modPath string, defRange hcl.Range) *Mod {
return &Mod{
ShortName: shortName,
FullName: fmt.Sprintf("mod.%s", shortName),
Queries: make(map[string]*Query),
Controls: make(map[string]*Control),
Benchmarks: make(map[string]*Benchmark),
Reports: make(map[string]*Report),
Panels: make(map[string]*Panel),
Variables: make(map[string]*Variable),
Locals: make(map[string]*Local),
ModPath: modPath,
DeclRange: defRange,
AllResources: make(map[string]HclResource),
mod := &Mod{
ShortName: shortName,
FullName: fmt.Sprintf("mod.%s", shortName),
Queries: make(map[string]*Query),
Controls: make(map[string]*Control),
Benchmarks: make(map[string]*Benchmark),
Reports: make(map[string]*Report),
Panels: make(map[string]*Panel),
Variables: make(map[string]*Variable),
Locals: make(map[string]*Local),
ModPath: modPath,
DeclRange: defRange,
Require: newRequire(),
}
// try to derive mod version from the path
mod.setVersion()
return mod
}
func (m *Mod) setVersion() {
segments := strings.Split(m.ModPath, "@")
if len(segments) == 1 {
return
}
versionString := segments[len(segments)-1]
// try to set version, ignoring error
version, err := semver.NewVersion(versionString)
if err == nil {
m.Version = version
m.VersionString = fmt.Sprintf("%d.%d", version.Major(), version.Minor())
}
}
@@ -108,30 +128,23 @@ func (m *Mod) Equals(other *Mod) bool {
return false
}
if len(*m.Categories) != len(*other.Categories) {
if len(m.Categories) != len(other.Categories) {
return false
}
for i, c := range *m.Categories {
if (*other.Categories)[i] != c {
for i, c := range m.Categories {
if (other.Categories)[i] != c {
return false
}
}
}
// tags
if m.Tags == nil {
if other.Tags != nil {
if len(m.Tags) != len(other.Tags) {
return false
}
for k, v := range m.Tags {
if otherVal := other.Tags[k]; v != otherVal {
return false
}
} else {
// we have tags
if other.Tags == nil {
return false
}
for k, v := range *m.Tags {
if otherVal, ok := (*other.Tags)[k]; !ok && v != otherVal {
return false
}
}
}
// controls
@@ -266,22 +279,22 @@ func (m *Mod) String() string {
}
versionString := ""
if m.Version != nil {
versionString = fmt.Sprintf("\nVersion: %s", types.SafeString(m.Version))
if m.VersionString == "" {
versionString = fmt.Sprintf("\nVersion: v%s", m.VersionString)
}
var requiresStrings []string
var requiresString string
if m.Requires != nil {
if m.Requires.SteampipeVersionString != "" {
requiresStrings = append(requiresStrings, fmt.Sprintf("Steampipe %s", m.Requires.SteampipeVersionString))
if m.Require != nil {
if m.Require.SteampipeVersionString != "" {
requiresStrings = append(requiresStrings, fmt.Sprintf("Steampipe %s", m.Require.SteampipeVersionString))
}
for _, m := range m.Requires.Mods {
for _, m := range m.Require.Mods {
requiresStrings = append(requiresStrings, m.String())
}
for _, p := range m.Requires.Plugins {
for _, p := range m.Require.Plugins {
requiresStrings = append(requiresStrings, p.String())
}
requiresString = fmt.Sprintf("Requires: \n%s", strings.Join(requiresStrings, "\n"))
requiresString = fmt.Sprintf("Require: \n%s", strings.Join(requiresStrings, "\n"))
}
return fmt.Sprintf(`Name: %s
@@ -306,146 +319,14 @@ Benchmarks:
)
}
// BuildResourceTree builds the control tree structure by setting the parent property for each control and benchmar
// NOTE: this also builds the sorted benchmark list
func (m *Mod) BuildResourceTree() error {
// build sorted list of benchmarks
m.benchmarksOrdered = make([]string, len(m.Benchmarks))
idx := 0
for name, benchmark := range m.Benchmarks {
// save this benchmark name
m.benchmarksOrdered[idx] = name
idx++
// add benchmark into control tree
if err := m.addItemIntoResourceTree(benchmark); err != nil {
return err
}
func (m *Mod) NameWithVersion() string {
if m.VersionString == "" {
return m.ShortName
}
// now sort the benchmark names
sort.Strings(m.benchmarksOrdered)
for _, control := range m.Controls {
if err := m.addItemIntoResourceTree(control); err != nil {
return err
}
}
for _, panel := range m.Panels {
if err := m.addItemIntoResourceTree(panel); err != nil {
return err
}
}
for _, report := range m.Reports {
if err := m.addItemIntoResourceTree(report); err != nil {
return err
}
}
return nil
return fmt.Sprintf("%s@%s", m.ShortName, m.VersionString)
}
func (m *Mod) addItemIntoResourceTree(item ModTreeItem) error {
parents := m.getParents(item)
// so we have a result - add into tree
for _, p := range parents {
// TODO validity checking
//for _, parentPath := range p.GetPaths() {
// // check this item does not exist in the parent path
// if helpers.StringSliceContains(parentPath, item.Name()) {
// return fmt.Errorf("cyclical dependency adding '%s' into control tree - parent '%s'", item.Name(), p.Name())
// }
item.AddParent(p)
p.AddChild(item)
//}
}
return nil
}
func (m *Mod) AddResource(item HclResource) hcl.Diagnostics {
var diags hcl.Diagnostics
switch r := item.(type) {
case *Query:
name := r.Name()
// check for dupes
if _, ok := m.Queries[name]; ok {
diags = append(diags, duplicateResourceDiagnostics(item))
break
}
m.Queries[name] = r
case *Control:
name := r.Name()
// check for dupes
if _, ok := m.Controls[name]; ok {
diags = append(diags, duplicateResourceDiagnostics(item))
break
}
m.Controls[name] = r
case *Benchmark:
name := r.Name()
// check for dupes
if _, ok := m.Benchmarks[name]; ok {
diags = append(diags, duplicateResourceDiagnostics(item))
break
} else {
m.Benchmarks[name] = r
}
case *Panel:
name := r.Name()
// check for dupes
if _, ok := m.Panels[name]; ok {
diags = append(diags, duplicateResourceDiagnostics(item))
break
} else {
m.Panels[name] = r
}
case *Report:
name := r.Name()
// check for dupes
if _, ok := m.Reports[name]; ok {
diags = append(diags, duplicateResourceDiagnostics(item))
break
} else {
m.Reports[name] = r
}
case *Variable:
name := r.Name()
// check for dupes
if _, ok := m.Variables[name]; ok {
diags = append(diags, duplicateResourceDiagnostics(item))
break
} else {
m.Variables[name] = r
}
case *Local:
name := r.Name()
// check for dupes
if _, ok := m.Locals[name]; ok {
diags = append(diags, duplicateResourceDiagnostics(item))
break
} else {
m.Locals[name] = r
}
}
m.AllResources[item.Name()] = item
return diags
}
func duplicateResourceDiagnostics(item HclResource) *hcl.Diagnostic {
return &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("mod defines more than one resource named %s", item.Name()),
Subject: item.GetDeclRange(),
}
}
// AddChild implements ModTreeItem
// AddChild implements ModTreeItem
func (m *Mod) AddChild(child ModTreeItem) error {
m.children = append(m.children, child)
return nil
@@ -463,10 +344,15 @@ func (m *Mod) GetParents() []ModTreeItem {
// Name implements ModTreeItem, HclResource
func (m *Mod) Name() string {
if m.Version == nil {
return m.FullName
return m.FullName
}
// GetModDependencyPath ModDependencyPath if it is set. If not it returns NameWithVersion()
func (m *Mod) GetModDependencyPath() string {
if m.ModDependencyPath != "" {
return m.ModDependencyPath
}
return fmt.Sprintf("%s@%s", m.FullName, types.SafeString(m.Version))
return m.NameWithVersion()
}
// GetTitle implements ModTreeItem
@@ -482,7 +368,7 @@ func (m *Mod) GetDescription() string {
// GetTags implements ModTreeItem
func (m *Mod) GetTags() map[string]string {
if m.Tags != nil {
return *m.Tags
return m.Tags
}
return map[string]string{}
}
@@ -520,13 +406,31 @@ func (m *Mod) CtyValue() (cty.Value, error) {
}
// OnDecoded implements HclResource
func (m *Mod) OnDecoded(*hcl.Block) hcl.Diagnostics {
func (m *Mod) OnDecoded(block *hcl.Block) hcl.Diagnostics {
// if VersionString is set, set Version
if m.VersionString != "" && m.Version == nil {
m.Version, _ = semver.NewVersion(m.VersionString)
}
// build flat children
m.buildFlatChilden()
// initialise our Requires
if m.Requires == nil {
// handle legacy requires block
if m.LegacyRequire != nil && !m.Require.Empty() {
if m.Require != nil && !m.Require.Empty() {
return hcl.Diagnostics{&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Both 'require' and legacy 'requires' blocks are defined",
Subject: &block.DefRange,
}}
}
m.Require = m.LegacyRequire
}
// initialise our Require
if m.Require == nil {
return nil
}
return m.Requires.Initialise()
return m.Require.initialise()
}
// AddReference implements HclResource
@@ -558,15 +462,15 @@ func (m *Mod) SetMetadata(metadata *ResourceMetadata) {
}
// get the parent item for this ModTreeItem
// first check all benchmarks - if they do not have this as child, default to the mod
func (m *Mod) getParents(item ModTreeItem) []ModTreeItem {
var parents []ModTreeItem
for _, benchmark := range m.Benchmarks {
if benchmark.ChildNames == nil {
continue
}
// check all child names of this benchmark for a matching name
for _, childName := range *benchmark.ChildNames {
for _, childName := range benchmark.ChildNames {
if childName.Name == item.Name() {
parents = append(parents, benchmark)
}
@@ -581,6 +485,9 @@ func (m *Mod) getParents(item ModTreeItem) []ModTreeItem {
}
}
for _, panel := range m.Panels {
if panel.Name() == item.Name() {
parents = append(parents, m)
}
// check all child names of this benchmark for a matching name
for _, child := range panel.GetChildren() {
if child.Name() == item.Name() {
@@ -588,13 +495,24 @@ func (m *Mod) getParents(item ModTreeItem) []ModTreeItem {
}
}
}
if len(parents) == 0 {
// fall back on mod
// if this item has no parents and is a child of the mod, set the mod as parent
if len(parents) == 0 && m.hasChild(item) {
parents = []ModTreeItem{m}
}
return parents
}
// is the given item a child of the mod
func (m *Mod) hasChild(item ModTreeItem) bool {
for _, c := range m.flatChildren {
if c.Name() == item.Name() {
return true
}
}
return false
}
// GetChildControls return a flat list of controls underneath the mod
func (m *Mod) GetChildControls() []*Control {
var res []*Control
@@ -603,3 +521,168 @@ func (m *Mod) GetChildControls() []*Control {
}
return res
}
func (m *Mod) AddModDependencies(modVersions map[string]*ModVersionConstraint) {
m.Require.AddModDependencies(modVersions)
}
func (m *Mod) RemoveModDependencies(modVersions map[string]*ModVersionConstraint) {
m.Require.RemoveModDependencies(modVersions)
}
func (m *Mod) RemoveAllModDependencies() {
m.Require.RemoveAllModDependencies()
}
func (m *Mod) Save() error {
f := hclwrite.NewEmptyFile()
rootBody := f.Body()
modBody := rootBody.AppendNewBlock("mod", []string{m.ShortName}).Body()
if m.Title != nil {
modBody.SetAttributeValue("title", cty.StringVal(*m.Title))
}
if m.Description != nil {
modBody.SetAttributeValue("description", cty.StringVal(*m.Description))
}
if m.Color != nil {
modBody.SetAttributeValue("color", cty.StringVal(*m.Color))
}
if m.Documentation != nil {
modBody.SetAttributeValue("documentation", cty.StringVal(*m.Documentation))
}
if m.Icon != nil {
modBody.SetAttributeValue("icon", cty.StringVal(*m.Icon))
}
if len(m.Categories) > 0 {
categoryValues := make([]cty.Value, len(m.Categories))
for i, c := range m.Categories {
categoryValues[i] = cty.StringVal(typehelpers.SafeString(c))
}
modBody.SetAttributeValue("categories", cty.ListVal(categoryValues))
}
if len(m.Tags) > 0 {
tagMap := make(map[string]cty.Value, len(m.Tags))
for k, v := range m.Tags {
tagMap[k] = cty.StringVal(v)
}
modBody.SetAttributeValue("tags", cty.MapVal(tagMap))
}
// opengraph
if opengraph := m.OpenGraph; opengraph != nil {
opengraphBody := modBody.AppendNewBlock("opengraph", nil).Body()
if opengraph.Title != nil {
opengraphBody.SetAttributeValue("title", cty.StringVal(*opengraph.Title))
}
if opengraph.Description != nil {
opengraphBody.SetAttributeValue("description", cty.StringVal(*opengraph.Description))
}
if opengraph.Image != nil {
opengraphBody.SetAttributeValue("image", cty.StringVal(*opengraph.Image))
}
}
// require
if require := m.Require; require != nil && !m.Require.Empty() {
requiresBody := modBody.AppendNewBlock("require", nil).Body()
if require.SteampipeVersionString != "" {
requiresBody.SetAttributeValue("steampipe", cty.StringVal(require.SteampipeVersionString))
}
if len(require.Plugins) > 0 {
pluginValues := make([]cty.Value, len(require.Plugins))
for i, p := range require.Plugins {
pluginValues[i] = cty.StringVal(typehelpers.SafeString(p))
}
requiresBody.SetAttributeValue("plugins", cty.ListVal(pluginValues))
}
if len(require.Mods) > 0 {
for _, m := range require.Mods {
modBody := requiresBody.AppendNewBlock("mod", []string{m.Name}).Body()
modBody.SetAttributeValue("version", cty.StringVal(m.VersionString))
}
}
}
// load existing mod data and remove the mod definitions from it
nonModData, err := m.loadNonModDataInModFile()
if err != nil {
return err
}
modData := append(f.Bytes(), nonModData...)
return os.WriteFile(constants.ModFilePath(m.ModPath), modData, 0644)
}
func (m *Mod) HasDependentMods() bool {
return m.Require != nil && len(m.Require.Mods) > 0
}
func (m *Mod) GetModDependency(modName string) *ModVersionConstraint {
if m.Require == nil {
return nil
}
return m.Require.GetModDependency(modName)
}
func (m *Mod) buildFlatChilden() {
res := make([]HclResource, len(m.Queries)+len(m.Controls)+len(m.Benchmarks)+len(m.Reports)+len(m.Panels)+len(m.Variables)+len(m.Locals))
idx := 0
for _, r := range m.Queries {
res[idx] = r
idx++
}
for _, r := range m.Controls {
res[idx] = r
idx++
}
for _, r := range m.Benchmarks {
res[idx] = r
idx++
}
for _, r := range m.Reports {
res[idx] = r
idx++
}
for _, r := range m.Panels {
res[idx] = r
idx++
}
for _, r := range m.Variables {
res[idx] = r
idx++
}
for _, r := range m.Locals {
res[idx] = r
idx++
}
m.flatChildren = res
}
func (m *Mod) loadNonModDataInModFile() ([]byte, error) {
modFilePath := constants.ModFilePath(m.ModPath)
if !helpers.FileExists(modFilePath) {
return nil, nil
}
fileData, err := os.ReadFile(modFilePath)
if err != nil {
return nil, err
}
fileLines := strings.Split(string(fileData), "\n")
decl := m.DeclRange
// just use line positions
start := decl.Start.Line - 1
end := decl.End.Line - 1
var resLines []string
for i, line := range fileLines {
if (i < start || i > end) && line != "" {
resLines = append(resLines, line)
}
}
return []byte(strings.Join(resLines, "\n")), nil
}

2
steampipeconfig/modconfig/mod_map.go
View File

@@ -4,7 +4,7 @@ import (
"strings"
)
// ModMap is a map of mod name to mod-version
// ModMap is a map of mod name to mod
type ModMap map[string]*Mod
func (m ModMap) String() string {

42
steampipeconfig/modconfig/mod_name.go Normal file
View File

@@ -0,0 +1,42 @@
package modconfig
import (
"bytes"
"fmt"
"strings"
"github.com/Masterminds/semver"
)
func ModVersionFullName(name string, version *semver.Version) string {
if version == nil {
return name
}
versionString := GetMonotonicVersionString(version)
return fmt.Sprintf("%s@v%s", name, versionString)
}
func GetMonotonicVersionString(v *semver.Version) string {
var buf bytes.Buffer
fmt.Fprintf(&buf, "%d.%d", v.Major(), v.Minor())
if v.Metadata() != "" {
fmt.Fprintf(&buf, "+%s", v.Metadata())
}
return buf.String()
}
func ParseModFullName(fullName string) (modName string, modVersion *semver.Version, err error) {
// we expect modLongName to be of form github.com/turbot/steampipe-mod-m2@v1.0
// split to get the name and version
parts := strings.Split(fullName, "@")
if len(parts) != 2 {
err = fmt.Errorf("invalid mod full name %s", fullName)
return
}
modName = parts[0]
modVersion, err = semver.NewVersion(parts[1])
if err != nil {
err = fmt.Errorf("mod file %s has invalid version", fullName)
}
return
}

154
steampipeconfig/modconfig/mod_resource_tree.go Normal file
View File

@@ -0,0 +1,154 @@
package modconfig
import (
"fmt"
"github.com/hashicorp/hcl/v2"
)
// BuildResourceTree builds the control tree structure by setting the parent property for each control and benchmar
// NOTE: this also builds the sorted benchmark list
func (m *Mod) BuildResourceTree(loadedDependencyMods ModMap) error {
if err := m.addResourcesIntoTree(m); err != nil {
return err
}
defer m.validateResourceTree()
if !m.HasDependentMods() {
return nil
}
// add dependent mods into tree
for _, requiredMod := range m.Require.Mods {
// find this mod in installed dependency mods
depMod, ok := loadedDependencyMods[requiredMod.Name]
if !ok {
return fmt.Errorf("dependency mod %s is not loaded", requiredMod.Name)
}
if err := m.addResourcesIntoTree(depMod); err != nil {
return err
}
}
return nil
}
func (m *Mod) addResourcesIntoTree(sourceMod *Mod) error {
for _, benchmark := range sourceMod.Benchmarks {
// add benchmark into control tree
if err := m.addItemIntoResourceTree(benchmark); err != nil {
return err
}
}
for _, control := range sourceMod.Controls {
if err := m.addItemIntoResourceTree(control); err != nil {
return err
}
}
for _, panel := range sourceMod.Panels {
if err := m.addItemIntoResourceTree(panel); err != nil {
return err
}
}
for _, report := range sourceMod.Reports {
if err := m.addItemIntoResourceTree(report); err != nil {
return err
}
}
return nil
}
func (m *Mod) addItemIntoResourceTree(item ModTreeItem) error {
parents := append(m.getParents(item))
// so we have a result - add into tree
for _, p := range parents {
item.AddParent(p)
p.AddChild(item)
}
return nil
}
func (m *Mod) AddResource(item HclResource) hcl.Diagnostics {
var diags hcl.Diagnostics
switch r := item.(type) {
case *Query:
name := r.Name()
// check for dupes
if _, ok := m.Queries[name]; ok {
diags = append(diags, duplicateResourceDiagnostics(item))
break
}
m.Queries[name] = r
case *Control:
name := r.Name()
// check for dupes
if _, ok := m.Controls[name]; ok {
diags = append(diags, duplicateResourceDiagnostics(item))
break
}
m.Controls[name] = r
case *Benchmark:
name := r.Name()
// check for dupes
if _, ok := m.Benchmarks[name]; ok {
diags = append(diags, duplicateResourceDiagnostics(item))
break
} else {
m.Benchmarks[name] = r
}
case *Panel:
name := r.Name()
// check for dupes
if _, ok := m.Panels[name]; ok {
diags = append(diags, duplicateResourceDiagnostics(item))
break
} else {
m.Panels[name] = r
}
case *Report:
name := r.Name()
// check for dupes
if _, ok := m.Reports[name]; ok {
diags = append(diags, duplicateResourceDiagnostics(item))
break
} else {
m.Reports[name] = r
}
case *Variable:
// NOTE: add variable by unqualified name
name := r.UnqualifiedName
// check for dupes
if _, ok := m.Variables[name]; ok {
diags = append(diags, duplicateResourceDiagnostics(item))
break
} else {
m.Variables[name] = r
}
case *Local:
name := r.Name()
// check for dupes
if _, ok := m.Locals[name]; ok {
diags = append(diags, duplicateResourceDiagnostics(item))
break
} else {
m.Locals[name] = r
}
}
return diags
}
func duplicateResourceDiagnostics(item HclResource) *hcl.Diagnostic {
return &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("mod defines more than one resource named %s", item.Name()),
Subject: item.GetDeclRange(),
}
}

32
steampipeconfig/modconfig/mod_validate.go Normal file
View File

@@ -0,0 +1,32 @@
package modconfig
import (
"fmt"
"github.com/turbot/steampipe/utils"
)
// ensure we have resolved all children in the resource tree
func (m *Mod) validateResourceTree() error {
var errors []error
for _, child := range m.GetChildren() {
if err := m.validateChildren(child); err != nil {
errors = append(errors, err)
}
}
return utils.CombineErrorsWithPrefix(fmt.Sprintf("failed to resolve children for %d resources", len(errors)), errors...)
}
func (m *Mod) validateChildren(item ModTreeItem) error {
missing := 0
for _, child := range item.GetChildren() {
if child == nil {
missing++
}
}
if missing > 0 {
return fmt.Errorf("%s has %d unresolved children", item.Name(), missing)
}
return nil
}

69
steampipeconfig/modconfig/mod_version.go
View File

@@ -1,69 +0,0 @@
package modconfig
import (
"fmt"
"strings"
goVersion "github.com/hashicorp/go-version"
typehelpers "github.com/turbot/go-kit/types"
"github.com/hashicorp/hcl/v2"
"github.com/turbot/go-kit/helpers"
)
type ModVersion struct {
// the fully qualified mod name, e.g. github.com/turbot/mod1
Name string `cty:"name" hcl:"name,label"`
VersionString string `cty:"version" hcl:"version"`
Alias *string `cty:"alias" hcl:"alias,optional"`
// only one of VersionConstraint, Branch and FilePath will be set
VersionConstraint *goVersion.Version
// the branch to use
Branch string
// the local file location to use
FilePath string
DeclRange hcl.Range
}
func (m *ModVersion) FullName() string {
if m.HasVersion() {
return fmt.Sprintf("%s@%s", m.Name, m.VersionString)
}
return m.Name
}
// HasVersion returns whether the mod has a version specified, or is the latest
// if no version is specified, or the version is "latest", this is the latest version
func (m *ModVersion) HasVersion() bool {
return !helpers.StringSliceContains([]string{"", "latest"}, m.VersionString)
}
func (m *ModVersion) String() string {
if alias := typehelpers.SafeString(m.Alias); alias != "" {
return fmt.Sprintf("mod %s (%s)", m.FullName(), alias)
}
return fmt.Sprintf("mod %s", m.FullName())
}
// Initialise parses the version and name properties
func (m *ModVersion) Initialise() hcl.Diagnostics {
var diags hcl.Diagnostics
if strings.HasPrefix(m.VersionString, "file:") {
m.FilePath = m.VersionString
return diags
}
// does the version parse as a semver version
if v, err := goVersion.NewVersion(m.VersionString); err == nil {
m.VersionConstraint = v
return diags
}
// otherwise assume it is a branch
m.Branch = m.VersionString
return diags
}

25
steampipeconfig/modconfig/mod_version_constrain_collection.go Normal file
View File

@@ -0,0 +1,25 @@
package modconfig
// ModVersionConstraintCollection is a collection of ModVersionConstraint instances and implements the sort
// interface. See the sort package for more details.
// https://golang.org/pkg/sort/
type ModVersionConstraintCollection []*ModVersionConstraint
// Len returns the length of a collection. The number of Version instances
// on the slice.
func (c ModVersionConstraintCollection) Len() int {
return len(c)
}
// Less is needed for the sort interface to compare two Version objects on the
// slice. If checks if one is less than the other.
func (c ModVersionConstraintCollection) Less(i, j int) bool {
// sort by name
return c[i].Name < (c[j].Name)
}
// Swap is needed for the sort interface to replace the Version objects
// at two different positions in the slice.
func (c ModVersionConstraintCollection) Swap(i, j int) {
c[i], c[j] = c[j], c[i]
}

114
steampipeconfig/modconfig/mod_version_constraint.go Normal file
View File

@@ -0,0 +1,114 @@
package modconfig
import (
"fmt"
"strings"
"github.com/hashicorp/hcl/v2"
"github.com/turbot/go-kit/helpers"
"github.com/turbot/steampipe/version_helpers"
)
const filePrefix = "file:"
type VersionConstrainCollection []*ModVersionConstraint
type ModVersionConstraint struct {
// the fully qualified mod name, e.g. github.com/turbot/mod1
Name string `cty:"name" hcl:"name,label"`
VersionString string `cty:"version" hcl:"version"`
// only one of Constraint, Branch and FilePath will be set
Constraint *version_helpers.Constraints
// // NOTE: aliases will be supported in the future
//Alias string `cty:"alias" hcl:"alias"`
// the branch to use
Branch string
// the local file location to use
FilePath string
DeclRange hcl.Range
}
func NewModVersionConstraint(modFullName string) (*ModVersionConstraint, error) {
var m *ModVersionConstraint
// if name has `file:` prefix, just set the name and ignore version
if strings.HasPrefix(modFullName, filePrefix) {
m = &ModVersionConstraint{Name: modFullName}
} else {
// otherwise try to extract version from name
segments := strings.Split(modFullName, "@")
if len(segments) > 2 {
return nil, fmt.Errorf("invalid mod name %s", modFullName)
}
m = &ModVersionConstraint{Name: segments[0]}
if len(segments) == 2 {
m.VersionString = segments[1]
}
}
// try to convert version into a semver constraint
if err := m.Initialise(); err != nil {
return nil, err
}
return m, nil
}
func (m *ModVersionConstraint) FullName() string {
if m.HasVersion() {
return fmt.Sprintf("%s@%s", m.Name, m.VersionString)
}
return m.Name
}
// HasVersion returns whether the mod has a version specified, or is the latest
// if no version is specified, or the version is "latest", this is the latest version
func (m *ModVersionConstraint) HasVersion() bool {
return !helpers.StringSliceContains([]string{"", "latest", "*"}, m.VersionString)
}
func (m *ModVersionConstraint) String() string {
return fmt.Sprintf("%s", m.FullName())
}
// Initialise parses the version and name properties
func (m *ModVersionConstraint) Initialise() hcl.Diagnostics {
if strings.HasPrefix(m.Name, filePrefix) {
m.setFilePath()
return nil
}
var diags hcl.Diagnostics
if m.VersionString == "" {
m.Constraint, _ = version_helpers.NewConstraint("*")
m.VersionString = "latest"
return diags
}
if m.VersionString == "latest" {
m.Constraint, _ = version_helpers.NewConstraint("*")
return diags
}
// does the version parse as a semver version
if c, err := version_helpers.NewConstraint(m.VersionString); err == nil {
// no error
m.Constraint = c
return diags
}
// todo handle branch and commit hash
// so there was an error
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("invalid mod version %s", m.VersionString),
Subject: &m.DeclRange,
})
return diags
}
func (m *ModVersionConstraint) setFilePath() {
m.FilePath = strings.TrimPrefix(m.FilePath, filePrefix)
}
func (m *ModVersionConstraint) Equals(other *ModVersionConstraint) bool {
// just check the hcl properties
return m.Name == other.Name && m.VersionString == other.VersionString
}

4
steampipeconfig/modconfig/open_graph.go
View File

@@ -7,9 +7,9 @@ import (
// OpenGraph is a struct representing the OpenGraph group mod resource
type OpenGraph struct {
// The opengraph description (og:description) of the mod, for use in social media applications
Description string `cty:"description" hcl:"description" json:"description"`
Description *string `cty:"description" hcl:"description" json:"description"`
// The opengraph display title (og:title) of the mod, for use in social media applications.
Title string `cty:"title" hcl:"title" json:"title"`
Title *string `cty:"title" hcl:"title" json:"title"`
Image *string `cty:"image" hcl:"image" json:"image"`
DeclRange hcl.Range `json:"-"`
}

19
steampipeconfig/modconfig/panel.go
View File

@@ -30,15 +30,17 @@ type Panel struct {
DeclRange hcl.Range
Mod *Mod `cty:"mod"`
parents []ModTreeItem
metadata *ResourceMetadata
parents []ModTreeItem
metadata *ResourceMetadata
UnqualifiedName string
}
func NewPanel(block *hcl.Block) *Panel {
panel := &Panel{
ShortName: block.Labels[0],
FullName: fmt.Sprintf("panel.%s", block.Labels[0]),
DeclRange: block.DefRange,
ShortName: block.Labels[0],
FullName: fmt.Sprintf("panel.%s", block.Labels[0]),
UnqualifiedName: fmt.Sprintf("panel.%s", block.Labels[0]),
DeclRange: block.DefRange,
}
return panel
}
@@ -85,11 +87,6 @@ func (p *Panel) Name() string {
return p.FullName
}
// QualifiedName returns the name in format: '<modName>.panel.<shortName>'
func (p *Panel) QualifiedName() string {
return fmt.Sprintf("%s.%s", p.metadata.ModName, p.FullName)
}
// OnDecoded implements HclResource
func (p *Panel) OnDecoded(*hcl.Block) hcl.Diagnostics { return nil }
@@ -99,6 +96,8 @@ func (p *Panel) AddReference(*ResourceReference) {}
// SetMod implements HclResource
func (p *Panel) SetMod(mod *Mod) {
p.Mod = mod
p.UnqualifiedName = p.FullName
p.FullName = fmt.Sprintf("%s.%s", mod.ShortName, p.FullName)
}
// GetMod implements HclResource

11
steampipeconfig/modconfig/parse_names.go
View File

@@ -57,6 +57,17 @@ func ParseResourceName(fullName string) (res *ParsedResourceName, err error) {
return
}
// UnqualifiedResourceName removes the mod prefix from the given name
func UnqualifiedResourceName(fullName string) string {
parts := strings.Split(fullName, ".")
switch len(parts) {
case 3:
return strings.Join(parts[1:], ".")
default:
return fullName
}
}
func ParseResourcePropertyPath(propertyPath string) (res *ParsedPropertyPath, err error) {
res = &ParsedPropertyPath{}

6
steampipeconfig/modconfig/plugin_version.go
View File

@@ -4,7 +4,7 @@ import (
"fmt"
"strings"
goVersion "github.com/hashicorp/go-version"
"github.com/Masterminds/semver"
"github.com/hashicorp/hcl/v2"
"github.com/turbot/steampipe/ociinstaller"
)
@@ -14,7 +14,7 @@ type PluginVersion struct {
RawName string `cty:"name" hcl:"name,label"`
// the version STREAM, can be either a major or minor version stream i.e. 1 or 1.1
VersionString string `cty:"version" hcl:"version,optional"`
Version *goVersion.Version
Version *semver.Version
// the org and name which are parsed from the raw name
Org string
Name string
@@ -39,7 +39,7 @@ func (p *PluginVersion) String() string {
// Initialise parses the version and name properties
func (p *PluginVersion) Initialise() hcl.Diagnostics {
var diags hcl.Diagnostics
if version, err := goVersion.NewVersion(strings.TrimPrefix(p.VersionString, "v")); err != nil {
if version, err := semver.NewVersion(strings.TrimPrefix(p.VersionString, "v")); err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("invalid plugin version %s", p.VersionString),

3
steampipeconfig/modconfig/prepared_statement.go
View File

@@ -3,6 +3,7 @@ package modconfig
import (
"fmt"
"log"
"strings"
"github.com/turbot/steampipe/utils"
)
@@ -25,6 +26,8 @@ func GetPreparedStatementExecuteSQL(source QueryProvider, args *QueryArgs) (stri
func preparedStatementName(source QueryProvider) string {
var name, suffix string
prefix := fmt.Sprintf("%s_", source.ModName())
prefix = strings.Replace(prefix, ".", "_", -1)
prefix = strings.Replace(prefix, "@", "_", -1)
// build suffix using a char to indicate control or query, and the truncated hash
switch t := source.(type) {

10
steampipeconfig/modconfig/query.go
View File

@@ -35,6 +35,7 @@ type Query struct {
DeclRange hcl.Range
PreparedStatementName string `column:"prepared_statement_name,text"`
metadata *ResourceMetadata
UnqualifiedName string
}
func (q *Query) Equals(other *Query) bool {
@@ -151,11 +152,6 @@ func (q *Query) Name() string {
return q.FullName
}
// QualifiedName returns the name in format: '<modName>.control.<shortName>'
func (q *Query) QualifiedName() string {
return fmt.Sprintf("%s.%s", q.metadata.ModName, q.FullName)
}
// GetMetadata implements ResourceWithMetadata
func (q *Query) GetMetadata() *ResourceMetadata {
return q.metadata
@@ -177,6 +173,8 @@ func (q *Query) AddReference(ref *ResourceReference) {
// SetMod implements HclResource
func (q *Query) SetMod(mod *Mod) {
q.Mod = mod
q.UnqualifiedName = q.FullName
q.FullName = fmt.Sprintf("%s.%s", mod.ShortName, q.FullName)
}
// GetMod implements HclResource
@@ -205,5 +203,5 @@ func (q *Query) GetPreparedStatementName() string {
// ModName implements QueryProvider
func (q *Query) ModName() string {
return q.Mod.ShortName
return q.Mod.NameWithVersion()
}

19
steampipeconfig/modconfig/report.go
View File

@@ -21,15 +21,17 @@ type Report struct {
DeclRange hcl.Range
parents []ModTreeItem
metadata *ResourceMetadata
parents []ModTreeItem
metadata *ResourceMetadata
UnqualifiedName string
}
func NewReport(block *hcl.Block) *Report {
report := &Report{
ShortName: block.Labels[0],
FullName: fmt.Sprintf("report.%s", block.Labels[0]),
DeclRange: block.DefRange,
ShortName: block.Labels[0],
FullName: fmt.Sprintf("report.%s", block.Labels[0]),
UnqualifiedName: fmt.Sprintf("report.%s", block.Labels[0]),
DeclRange: block.DefRange,
}
return report
}
@@ -45,11 +47,6 @@ func (r *Report) Name() string {
return r.FullName
}
// QualifiedName returns the name in format: '<modName>.report.<shortName>'
func (r *Report) QualifiedName() string {
return fmt.Sprintf("%s.%s", r.metadata.ModName, r.FullName)
}
// OnDecoded implements HclResource
func (r *Report) OnDecoded(*hcl.Block) hcl.Diagnostics { return nil }
@@ -61,6 +58,8 @@ func (r *Report) AddReference(*ResourceReference) {
// SetMod implements HclResource
func (r *Report) SetMod(mod *Mod) {
r.Mod = mod
r.UnqualifiedName = r.FullName
r.FullName = fmt.Sprintf("%s.%s", mod.ShortName, r.FullName)
}
// GetMod implements HclResource

109
steampipeconfig/modconfig/requires.go
View File

@@ -2,36 +2,37 @@ package modconfig
import (
"fmt"
"sort"
"strings"
goVersion "github.com/hashicorp/go-version"
"github.com/Masterminds/semver"
"github.com/hashicorp/hcl/v2"
"github.com/turbot/steampipe/version"
)
// Requires is a struct representing mod dependencies
type Requires struct {
// Require is a struct representing mod dependencies
type Require struct {
SteampipeVersionString string `hcl:"steampipe,optional"`
SteampipeVersion *goVersion.Version
Plugins []*PluginVersion `hcl:"plugin,block"`
Mods []*ModVersion `hcl:"mod,block"`
DeclRange hcl.Range `json:"-"`
SteampipeVersion *semver.Version
Plugins []*PluginVersion `hcl:"plugin,block"`
Mods []*ModVersionConstraint `hcl:"mod,block"`
DeclRange hcl.Range `json:"-"`
// map keyed by name [and alias]
modMap map[string]*ModVersionConstraint
}
func (r *Requires) ValidateSteampipeVersion(modName string) error {
if r.SteampipeVersion != nil {
if version.Version.LessThan(r.SteampipeVersion) {
return fmt.Errorf("steampipe version %s does not satisfy %s which requires version %s", version.String(), modName, r.SteampipeVersion.String())
}
}
return nil
func newRequire() *Require {
r := &Require{}
r.initialise()
return r
}
func (r *Requires) Initialise() hcl.Diagnostics {
func (r *Require) initialise() hcl.Diagnostics {
var diags hcl.Diagnostics
r.modMap = make(map[string]*ModVersionConstraint)
if r.SteampipeVersionString != "" {
steampipeVersion, err := goVersion.NewVersion(strings.TrimPrefix(r.SteampipeVersionString, "v"))
steampipeVersion, err := semver.NewVersion(strings.TrimPrefix(r.SteampipeVersionString, "v"))
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
@@ -49,6 +50,82 @@ func (r *Requires) Initialise() hcl.Diagnostics {
for _, m := range r.Mods {
moreDiags := m.Initialise()
diags = append(diags, moreDiags...)
if !diags.HasErrors() {
// key map entry by name [and alias]
r.modMap[m.Name] = m
}
}
return diags
}
func (r *Require) ValidateSteampipeVersion(modName string) error {
if r.SteampipeVersion != nil {
if version.SteampipeVersion.LessThan(r.SteampipeVersion) {
return fmt.Errorf("steampipe version %s does not satisfy %s which requires version %s", version.SteampipeVersion.String(), modName, r.SteampipeVersion.String())
}
}
return nil
}
// AddModDependencies adds all the mod in newModVersions to our list of mods, using the following logic
// - if a mod with same name, [alias] and constraint exists, it is not added
// - if a mod with same name [and alias] and different constraint exist, it is replaced
func (r *Require) AddModDependencies(newModVersions map[string]*ModVersionConstraint) {
// rebuild the Mods array
// first rebuild the mod map
for name, newVersion := range newModVersions {
// todo take alias into account
r.modMap[name] = newVersion
}
// now update the mod array from the map
var newMods = make([]*ModVersionConstraint, len(r.modMap))
idx := 0
for _, requiredVersion := range r.modMap {
newMods[idx] = requiredVersion
idx++
}
// sort by name
sort.Sort(ModVersionConstraintCollection(newMods))
// write back
r.Mods = newMods
}
func (r *Require) RemoveModDependencies(versions map[string]*ModVersionConstraint) {
// first rebuild the mod map
for name := range versions {
// todo take alias into account
delete(r.modMap, name)
}
// now update the mod array from the map
var newMods = make([]*ModVersionConstraint, len(r.modMap))
idx := 0
for _, requiredVersion := range r.modMap {
newMods[idx] = requiredVersion
idx++
}
// sort by name
sort.Sort(ModVersionConstraintCollection(newMods))
// write back
r.Mods = newMods
}
func (r *Require) RemoveAllModDependencies() {
r.Mods = nil
}
func (r *Require) GetModDependency(name string /*,alias string*/) *ModVersionConstraint {
return r.modMap[name]
}
func (r *Require) ContainsMod(requiredModVersion *ModVersionConstraint) bool {
if c := r.GetModDependency(requiredModVersion.Name); c != nil {
return c.Equals(requiredModVersion)
}
return false
}
func (r *Require) Empty() bool {
return r.SteampipeVersion == nil && len(r.Mods) == 0 && len(r.Plugins) == 0
}

10
steampipeconfig/modconfig/variable.go
View File

@@ -29,7 +29,8 @@ type Variable struct {
ParsingMode var_config.VariableParsingMode
Mod *Mod
metadata *ResourceMetadata
metadata *ResourceMetadata
UnqualifiedName string
}
func NewVariable(v *var_config.Variable) *Variable {
@@ -58,11 +59,6 @@ func (v *Variable) Name() string {
return v.FullName
}
// QualifiedName returns the name in format: '<modName>.var.<shortName>'
func (v *Variable) QualifiedName() string {
return fmt.Sprintf("%s.%s", v.metadata.ModName, v.FullName)
}
// GetMetadata implements ResourceWithMetadata
func (v *Variable) GetMetadata() *ResourceMetadata {
return v.metadata
@@ -82,6 +78,8 @@ func (v *Variable) AddReference(*ResourceReference) {}
// SetMod implements HclResource
func (v *Variable) SetMod(mod *Mod) {
v.Mod = mod
v.UnqualifiedName = v.FullName
v.FullName = fmt.Sprintf("%s.%s", mod.ShortName, v.FullName)
}
// GetMod implements HclResource

14
steampipeconfig/parse/decode.go
View File

@@ -417,7 +417,6 @@ func decodeControl(block *hcl.Block, runCtx *RunContext) (*modconfig.Control, *d
}
return c, res
}
func decodeControlArgs(attr *hcl.Attribute, evalCtx *hcl.EvalContext, controlName string) (*modconfig.QueryArgs, hcl.Diagnostics) {
@@ -540,12 +539,6 @@ func decodeProperty(content *hcl.BodyContent, property string, dest interface{},
func handleDecodeResult(resource modconfig.HclResource, res *decodeResult, block *hcl.Block, runCtx *RunContext) hcl.Diagnostics {
var diags hcl.Diagnostics
if res.Success() {
// if resource supports metadata, save it
if resourceWithMetadata, ok := resource.(modconfig.ResourceWithMetadata); ok {
body := block.Body.(*hclsyntax.Body)
diags = addResourceMetadata(resourceWithMetadata, body.SrcRange, runCtx)
}
// if resource is NOT a mod, set mod pointer on hcl resource and add resource to current mod
if _, ok := resource.(*modconfig.Mod); !ok {
resource.SetMod(runCtx.CurrentMod)
@@ -553,6 +546,13 @@ func handleDecodeResult(resource modconfig.HclResource, res *decodeResult, block
moreDiags := runCtx.CurrentMod.AddResource(resource)
diags = append(diags, moreDiags...)
}
// if resource supports metadata, save it
if resourceWithMetadata, ok := resource.(modconfig.ResourceWithMetadata); ok {
body := block.Body.(*hclsyntax.Body)
diags = addResourceMetadata(resourceWithMetadata, body.SrcRange, runCtx)
}
// add resource into the run context
moreDiags := runCtx.AddResource(resource)
diags = append(diags, moreDiags...)

4
steampipeconfig/parse/installed_mod.go
View File

@@ -1,11 +1,11 @@
package parse
import (
goVersion "github.com/hashicorp/go-version"
"github.com/Masterminds/semver"
"github.com/turbot/steampipe/steampipeconfig/modconfig"
)
type InstalledMod struct {
Mod *modconfig.Mod
Version *goVersion.Version
Version *semver.Version
}

19
steampipeconfig/parse/parser.go
View File

@@ -7,10 +7,10 @@ import (
"os"
"path/filepath"
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/hashicorp/hcl/v2/hclparse"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/hashicorp/hcl/v2/json"
"github.com/turbot/steampipe-plugin-sdk/plugin"
"github.com/turbot/steampipe/constants"
@@ -75,12 +75,12 @@ func ModfileExists(modPath string) bool {
}
// ParseModDefinition parses the modfile only
// it is expected the callign code wil lhave verified the existence of the modfile by calling ModfileExists
// it is expected the calling code will have verified the existence of the modfile by calling ModfileExists
func ParseModDefinition(modPath string) (*modconfig.Mod, error) {
// TODO think about variables
// if there is no mod at this location, return error
modFilePath := filepath.Join(modPath, "mod.sp")
modFilePath := constants.ModFilePath(modPath)
if _, err := os.Stat(modFilePath); os.IsNotExist(err) {
return nil, fmt.Errorf("no mod file found in %s", modPath)
}
@@ -107,7 +107,11 @@ func ParseModDefinition(modPath string) (*modconfig.Mod, error) {
for _, block := range content.Blocks {
if block.Type == modconfig.BlockTypeMod {
mod := modconfig.NewMod(block.Labels[0], modPath, block.DefRange)
var defRange = block.DefRange
if hclBody, ok := block.Body.(*hclsyntax.Body); ok {
defRange = hclBody.SrcRange
}
mod := modconfig.NewMod(block.Labels[0], modPath, defRange)
diags := gohcl.DecodeBody(block.Body, evalCtx, mod)
if diags.HasErrors() {
return nil, plugin.DiagsToError("Failed to decode mod hcl file", diags)
@@ -156,7 +160,8 @@ func ParseMod(modPath string, fileData map[string][]byte, pseudoResources []modc
addPseudoResourcesToMod(pseudoResources, hclResources, mod)
// add this mod to run context - this it to ensure all pseudo resources get added
runCtx.AddMod(mod, content, fileData)
runCtx.SetDecodeContent(content, fileData)
runCtx.AddMod(mod)
// perform initial decode to get dependencies
// (if there are no dependencies, this is all that is needed)
@@ -180,7 +185,7 @@ func ParseMod(modPath string, fileData map[string][]byte, pseudoResources []modc
// now tell mod to build tree of controls.
// NOTE: this also builds the sorted benchmark list
if err := mod.BuildResourceTree(); err != nil {
if err := mod.BuildResourceTree(runCtx.LoadedDependencyMods); err != nil {
return nil, err
}

65
steampipeconfig/parse/run_context.go
View File

@@ -8,9 +8,9 @@ import (
"github.com/stevenle/topsort"
filehelpers "github.com/turbot/go-kit/files"
"github.com/turbot/go-kit/helpers"
"github.com/turbot/steampipe/constants"
"github.com/turbot/steampipe/steampipeconfig/hclhelpers"
"github.com/turbot/steampipe/steampipeconfig/modconfig"
"github.com/turbot/steampipe/steampipeconfig/version_map"
"github.com/zclconf/go-cty/cty"
)
@@ -35,7 +35,9 @@ type ReferenceTypeValueMap map[string]map[string]cty.Value
type RunContext struct {
// the mod which is currently being parsed
CurrentMod *modconfig.Mod
CurrentMod *modconfig.Mod
// the workspace lock data
WorkspaceLock *version_map.WorkspaceLock
UnresolvedBlocks map[string]*unresolvedBlock
FileData map[string][]byte
// the eval context used to decode references in HCL
@@ -44,8 +46,7 @@ type RunContext struct {
Flags ParseModFlag
ListOptions *filehelpers.ListOptions
LoadedDependencyMods modconfig.ModMap
WorkspacePath string
ModInstallationPath string
RootEvalPath string
// if set, only decode these blocks
BlockTypes []string
// if set, exclude these block types
@@ -59,11 +60,11 @@ type RunContext struct {
Variables map[string]*modconfig.Variable
}
func NewRunContext(workspacePath string, flags ParseModFlag, listOptions *filehelpers.ListOptions) *RunContext {
func NewRunContext(workspaceLock *version_map.WorkspaceLock, rootEvalPath string, flags ParseModFlag, listOptions *filehelpers.ListOptions) *RunContext {
c := &RunContext{
Flags: flags,
ModInstallationPath: constants.WorkspaceModPath(workspacePath),
WorkspacePath: workspacePath,
RootEvalPath: rootEvalPath,
WorkspaceLock: workspaceLock,
ListOptions: listOptions,
LoadedDependencyMods: make(modconfig.ModMap),
UnresolvedBlocks: make(map[string]*unresolvedBlock),
@@ -77,9 +78,20 @@ func NewRunContext(workspacePath string, flags ParseModFlag, listOptions *filehe
// add enums to the variables which may be referenced from within the hcl
c.addSteampipeEnums()
c.buildEvalContext()
return c
}
func (r *RunContext) EnsureWorkspaceLock(mod *modconfig.Mod) error {
// if the mod has dependencies, there must a workspace lock object in the run context
// (mod MUST be the workspace mod, not a dependency, as we would hit this error as soon as we parse it)
if mod.HasDependentMods() && (r.WorkspaceLock.Empty() || r.WorkspaceLock.Incomplete()) {
return fmt.Errorf("not all dependencies are installed - run 'steampipe mod install'")
}
return nil
}
// VariableValueMap converts a map of variables to a map of the underlying cty value
func VariableValueMap(variables map[string]*modconfig.Variable) map[string]cty.Value {
ret := make(map[string]cty.Value, len(variables))
@@ -101,15 +113,12 @@ func (r *RunContext) AddVariables(inputVariables map[string]*modconfig.Variable)
// AddMod is used to add a mod with any pseudo resources to the eval context
// - in practice this will be a shell mod with just pseudo resources - other resources will be added as they are parsed
func (r *RunContext) AddMod(mod *modconfig.Mod, content *hcl.BodyContent, fileData map[string][]byte) hcl.Diagnostics {
func (r *RunContext) AddMod(mod *modconfig.Mod) hcl.Diagnostics {
if len(r.UnresolvedBlocks) > 0 {
// should never happen
panic("calling SetContent on runContext but there are unresolved blocks from a previous parse")
}
r.FileData = fileData
r.blocks = content.Blocks
var diags hcl.Diagnostics
moreDiags := r.storeResourceInCtyMap(mod)
@@ -119,20 +128,24 @@ func (r *RunContext) AddMod(mod *modconfig.Mod, content *hcl.BodyContent, fileDa
moreDiags := r.storeResourceInCtyMap(q)
diags = append(diags, moreDiags...)
}
for _, q := range mod.Controls {
moreDiags := r.storeResourceInCtyMap(q)
for _, c := range mod.Controls {
moreDiags := r.storeResourceInCtyMap(c)
diags = append(diags, moreDiags...)
}
for _, q := range mod.Locals {
moreDiags := r.storeResourceInCtyMap(q)
for _, b := range mod.Benchmarks {
moreDiags := r.storeResourceInCtyMap(b)
diags = append(diags, moreDiags...)
}
for _, q := range mod.Reports {
moreDiags := r.storeResourceInCtyMap(q)
for _, l := range mod.Locals {
moreDiags := r.storeResourceInCtyMap(l)
diags = append(diags, moreDiags...)
}
for _, q := range mod.Panels {
moreDiags := r.storeResourceInCtyMap(q)
for _, rpt := range mod.Reports {
moreDiags := r.storeResourceInCtyMap(rpt)
diags = append(diags, moreDiags...)
}
for _, p := range mod.Panels {
moreDiags := r.storeResourceInCtyMap(p)
diags = append(diags, moreDiags...)
}
@@ -141,6 +154,11 @@ func (r *RunContext) AddMod(mod *modconfig.Mod, content *hcl.BodyContent, fileDa
return diags
}
func (r *RunContext) SetDecodeContent(content *hcl.BodyContent, fileData map[string][]byte) {
r.blocks = content.Blocks
r.FileData = fileData
}
func (r *RunContext) ShouldIncludeBlock(block *hcl.Block) bool {
if len(r.BlockTypes) > 0 && !helpers.StringSliceContains(r.BlockTypes, block.Type) {
return false
@@ -360,10 +378,11 @@ func (r *RunContext) buildEvalContext() {
variables[mod] = cty.ObjectVal(refTypeMap)
}
//create evaluation context
// create evaluation context
r.EvalCtx = &hcl.EvalContext{
Variables: variables,
Functions: ContextFunctions(r.WorkspacePath),
// use the mod path as the file root for functions
Functions: ContextFunctions(r.RootEvalPath),
}
}
@@ -411,11 +430,9 @@ func (r *RunContext) addReferenceValue(resource modconfig.HclResource, value cty
typeString := parsedName.ItemType
// the resource name will not have a mod - but the run context knows which mod we are parsing
mod := r.CurrentMod
modName := mod.ShortName
if mod.ModPath == r.WorkspacePath {
if mod.ModPath == r.RootEvalPath {
modName = "local"
}
variablesForMod, ok := r.referenceValues[modName]

2
steampipeconfig/test_data/demo/control_demo/mod.sp
View File

@@ -13,7 +13,7 @@ mod "m1" {
description = "CIS reports, queries, and actions for AWS. Open source CLI. No DB required."
}
//
// requires {
// require {
// steampipe ">0.3.0" {}
//
// plugin "aws" {}

2
steampipeconfig/test_data/dep_test2/mod.sp
View File

@@ -11,7 +11,7 @@ mod "m1" {
labels = ["public cloud", "aws"]
# dependencies
requires {
require {
steampipe = ">0.3.0"
plugin "aws" {}

103
steampipeconfig/version_map/dependency_version_map.go Normal file
View File

@@ -0,0 +1,103 @@
package version_map
import (
"github.com/Masterminds/semver"
"github.com/turbot/steampipe/steampipeconfig/modconfig"
"github.com/xlab/treeprint"
)
type DependencyVersionMap map[string]ResolvedVersionMap
// Add adds a dependency to the list of items installed for the given parent
func (m DependencyVersionMap) Add(dependencyName string, dependencyVersion *semver.Version, constraintString string, parentName string) {
// get the map for this parent
parentItems := m[parentName]
// create if needed
if parentItems == nil {
parentItems = make(ResolvedVersionMap)
}
// add the dependency
parentItems.Add(dependencyName, &ResolvedVersionConstraint{dependencyName, dependencyVersion, constraintString})
// save
m[parentName] = parentItems
}
// FlatMap converts the DependencyVersionMap into a ResolvedVersionMap, keyed by full name
func (m DependencyVersionMap) FlatMap() ResolvedVersionMap {
res := make(ResolvedVersionMap)
for _, deps := range m {
for _, dep := range deps {
res[modconfig.ModVersionFullName(dep.Name, dep.Version)] = dep
}
}
return res
}
func (m DependencyVersionMap) GetDependencyTree(rootName string) treeprint.Tree {
tree := treeprint.NewWithRoot(rootName)
m.buildTree(rootName, tree)
return tree
}
func (m DependencyVersionMap) buildTree(name string, tree treeprint.Tree) {
deps := m[name]
for name, version := range deps {
fullName := modconfig.ModVersionFullName(name, version.Version)
child := tree.AddBranch(fullName)
// if there are children add them
m.buildTree(fullName, child)
}
}
// GetMissingFromOther returns a map of dependencies which exit in this map but not 'other'
func (m DependencyVersionMap) GetMissingFromOther(other DependencyVersionMap) DependencyVersionMap {
res := make(DependencyVersionMap)
for parent, deps := range m {
otherDeps := other[parent]
if otherDeps == nil {
otherDeps = make(ResolvedVersionMap)
}
for name, dep := range deps {
if _, ok := otherDeps[name]; !ok {
res.Add(dep.Name, dep.Version, dep.Constraint, parent)
}
}
}
return res
}
func (m DependencyVersionMap) GetUpgradedInOther(other DependencyVersionMap) DependencyVersionMap {
res := make(DependencyVersionMap)
for parent, deps := range m {
otherDeps := other[parent]
if otherDeps == nil {
otherDeps = make(ResolvedVersionMap)
}
for name, dep := range deps {
if otherDep, ok := otherDeps[name]; ok {
if otherDep.Version.GreaterThan(dep.Version) {
res.Add(otherDep.Name, otherDep.Version, otherDep.Constraint, parent)
}
}
}
}
return res
}
func (m DependencyVersionMap) GetDowngradedInOther(other DependencyVersionMap) DependencyVersionMap {
res := make(DependencyVersionMap)
for parent, deps := range m {
otherDeps := other[parent]
if otherDeps == nil {
otherDeps = make(ResolvedVersionMap)
}
for name, dep := range deps {
if otherDep, ok := otherDeps[name]; ok {
if otherDep.Version.LessThan(dep.Version) {
res.Add(otherDep.Name, otherDep.Version, otherDep.Constraint, parent)
}
}
}
}
return res
}

20
steampipeconfig/version_map/resolved_version_constraint.go Normal file
View File

@@ -0,0 +1,20 @@
package version_map
import "github.com/Masterminds/semver"
type ResolvedVersionConstraint struct {
Name string
// Alias string
Version *semver.Version
Constraint string
}
func (c ResolvedVersionConstraint) Equals(other *ResolvedVersionConstraint) bool {
return c.Name == other.Name &&
c.Version.Equal(other.Version) &&
c.Constraint == other.Constraint
}
func (c ResolvedVersionConstraint) IsPrerelease() bool {
return c.Version.Prerelease() != "" || c.Version.Metadata() != ""
}

49
steampipeconfig/version_map/resolved_version_list_map.go Normal file
View File

@@ -0,0 +1,49 @@
package version_map
import (
"github.com/turbot/steampipe/steampipeconfig/modconfig"
)
// ResolvedVersionListMap represents a map of ResolvedVersionConstraint arrays, keyed by dependency name
type ResolvedVersionListMap map[string][]*ResolvedVersionConstraint
// Add appends the version constraint to the list for the given name
func (m ResolvedVersionListMap) Add(name string, versionConstraint *ResolvedVersionConstraint) {
// if there is already an entry for the same name, replace it
// TODO handle alias
m[name] = []*ResolvedVersionConstraint{versionConstraint}
}
// Remove removes the given version constraint from the list for the given name
func (m ResolvedVersionListMap) Remove(name string, constraint *ResolvedVersionConstraint) {
var res []*ResolvedVersionConstraint
for _, c := range m[name] {
if !c.Equals(constraint) {
res = append(res, c)
}
}
m[name] = res
}
// FlatMap converts the ResolvedVersionListMap map into a map keyed by the FULL dependency name (i.e. including version(
func (m ResolvedVersionListMap) FlatMap() map[string]*ResolvedVersionConstraint {
var res = make(map[string]*ResolvedVersionConstraint)
for name, versions := range m {
for _, version := range versions {
key := modconfig.ModVersionFullName(name, version.Version)
res[key] = version
}
}
return res
}
// FlatNames converts the ResolvedVersionListMap map into a string array of full names
func (m ResolvedVersionListMap) FlatNames() []string {
var res []string
for name, versions := range m {
for _, version := range versions {
res = append(res, modconfig.ModVersionFullName(name, version.Version))
}
}
return res
}

21
steampipeconfig/version_map/resolved_version_map.go Normal file
View File

@@ -0,0 +1,21 @@
package version_map
// ResolvedVersionMap represents a map of ResolvedVersionConstraint, keyed by dependency name
type ResolvedVersionMap map[string]*ResolvedVersionConstraint
func (m ResolvedVersionMap) Add(name string, constraint *ResolvedVersionConstraint) {
m[name] = constraint
}
func (m ResolvedVersionMap) Remove(name string) {
delete(m, name)
}
// ToVersionListMap converts this map into a ResolvedVersionListMap
func (m ResolvedVersionMap) ToVersionListMap() ResolvedVersionListMap {
res := make(ResolvedVersionListMap, len(m))
for k, v := range m {
res.Add(k, v)
}
return res
}

5
steampipeconfig/version_map/version_constraint_map.go Normal file
View File

@@ -0,0 +1,5 @@
package version_map
import "github.com/turbot/steampipe/steampipeconfig/modconfig"
type VersionConstraintMap map[string]*modconfig.ModVersionConstraint

31
steampipeconfig/version_map/version_list_map.go Normal file
View File

@@ -0,0 +1,31 @@
package version_map
import (
"sort"
"github.com/Masterminds/semver"
"github.com/turbot/steampipe/steampipeconfig/modconfig"
)
// VersionListMap is a map keyed by dependency name storing a list of versions for each dependency
type VersionListMap map[string]semver.Collection
func (i VersionListMap) Add(name string, version *semver.Version) {
versions := append(i[name], version)
// reverse sort the versions
sort.Sort(sort.Reverse(versions))
i[name] = versions
}
// FlatMap converts the VersionListMap map into a bool map keyed by qualified dependency name
func (m VersionListMap) FlatMap() map[string]bool {
var res = make(map[string]bool)
for name, versions := range m {
for _, version := range versions {
key := modconfig.ModVersionFullName(name, version)
res[key] = true
}
}
return res
}

8
steampipeconfig/version_map/version_map.go Normal file
View File

@@ -0,0 +1,8 @@
package version_map
import (
"github.com/Masterminds/semver"
)
// VersionMap represents a map of semver versions, keyed by dependency name
type VersionMap map[string]*semver.Version

294
steampipeconfig/version_map/workspace_lock.go Normal file
View File

@@ -0,0 +1,294 @@
package version_map
import (
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"github.com/Masterminds/semver"
filehelpers "github.com/turbot/go-kit/files"
"github.com/turbot/go-kit/helpers"
"github.com/turbot/steampipe/constants"
"github.com/turbot/steampipe/steampipeconfig/modconfig"
"github.com/turbot/steampipe/utils"
"github.com/turbot/steampipe/version_helpers"
)
// WorkspaceLock is a map of ModVersionMaps items keyed by the parent mod whose dependencies are installed
type WorkspaceLock struct {
WorkspacePath string
InstallCache DependencyVersionMap
MissingVersions DependencyVersionMap
ModInstallationPath string
installedMods VersionListMap
}
// EmptyWorkspaceLock creates a new empty workspace lock based,
// sharing workspace path and installedMods with 'existingLock'
func EmptyWorkspaceLock(existingLock *WorkspaceLock) *WorkspaceLock {
return &WorkspaceLock{
WorkspacePath: existingLock.WorkspacePath,
ModInstallationPath: constants.WorkspaceModPath(existingLock.WorkspacePath),
InstallCache: make(DependencyVersionMap),
MissingVersions: make(DependencyVersionMap),
installedMods: existingLock.installedMods,
}
}
func LoadWorkspaceLock(workspacePath string) (*WorkspaceLock, error) {
var installCache = make(DependencyVersionMap)
lockPath := constants.WorkspaceLockPath(workspacePath)
if helpers.FileExists(lockPath) {
fileContent, err := os.ReadFile(lockPath)
if err != nil {
log.Printf("[TRACE] error reading %s: %s\n", lockPath, err.Error())
return nil, err
}
err = json.Unmarshal(fileContent, &installCache)
if err != nil {
log.Printf("[TRACE] failed to unmarshal %s: %s\n", lockPath, err.Error())
return nil, nil
}
}
res := &WorkspaceLock{
WorkspacePath: workspacePath,
ModInstallationPath: constants.WorkspaceModPath(workspacePath),
InstallCache: installCache,
MissingVersions: make(DependencyVersionMap),
}
if err := res.getInstalledMods(); err != nil {
return nil, err
}
// populate the MissingVersions
// (this removes missing items from the install cache)
res.setMissing()
return res, nil
}
// populate MissingVersions and UnreferencedVersions
func (l *WorkspaceLock) validate() error {
return nil
}
// getInstalledMods returns a map installed mods, and the versions installed for each
func (l *WorkspaceLock) getInstalledMods() error {
// recursively search for all the mod.sp files under the .steampipe/mods folder, then build the mod name from the file path
modFiles, err := filehelpers.ListFiles(l.ModInstallationPath, &filehelpers.ListOptions{
Flags: filehelpers.FilesRecursive,
Include: []string{"**/mod.sp"},
})
if err != nil {
return err
}
// create result map - a list of version for each mod
installedMods := make(VersionListMap, len(modFiles))
// collect errors
var errors []error
for _, modfilePath := range modFiles {
// try to parse the mon name and version form the parent folder of the modfile
modName, version, err := l.parseModPath(modfilePath)
if err != nil {
// if we fail to parse, just ignore this modfile
// - it's parent is not a valid mod installation folder so it is probably a child folder of a mod
continue
}
// add this mod version to the map
installedMods.Add(modName, version)
}
if len(errors) > 0 {
return utils.CombineErrors(errors...)
}
l.installedMods = installedMods
return nil
}
// GetUnreferencedMods returns a map of all installed mods which are not in the lock file
func (l *WorkspaceLock) GetUnreferencedMods() VersionListMap {
var unreferencedVersions = make(VersionListMap)
for name, versions := range l.installedMods {
for _, version := range versions {
if !l.ContainsModVersion(name, version) {
unreferencedVersions.Add(name, version)
}
}
}
return unreferencedVersions
}
// identify mods which are in InstallCache but not installed
// move them from InstallCache into MissingVersions
func (l *WorkspaceLock) setMissing() {
// create a map of full modname to bool to allow simple checking
flatInstalled := l.installedMods.FlatMap()
for parent, deps := range l.InstallCache {
// deps is a map of dep name to resolved contraint list
// flatten and iterate
for name, resolvedConstraint := range deps {
fullName := modconfig.ModVersionFullName(name, resolvedConstraint.Version)
if !flatInstalled[fullName] {
// get the mod name from the constraint (fullName includes the version)
name := resolvedConstraint.Name
// remove this item from the install cache and add into missing
l.MissingVersions.Add(name, resolvedConstraint.Version, resolvedConstraint.Constraint, parent)
l.InstallCache[parent].Remove(name)
}
}
}
}
// extract the mod name and version from the modfile path
func (l *WorkspaceLock) parseModPath(modfilePath string) (modName string, modVersion *semver.Version, err error) {
modFullName, err := filepath.Rel(l.ModInstallationPath, filepath.Dir(modfilePath))
if err != nil {
return
}
return modconfig.ParseModFullName(modFullName)
}
func (l *WorkspaceLock) Save() error {
if len(l.InstallCache) == 0 {
// ignore error
l.Delete()
return nil
}
content, err := json.MarshalIndent(l.InstallCache, "", " ")
if err != nil {
return err
}
return os.WriteFile(constants.WorkspaceLockPath(l.WorkspacePath), content, 0644)
}
// Delete deletes the lock file
func (l *WorkspaceLock) Delete() error {
return os.Remove(constants.WorkspaceLockPath(l.WorkspacePath))
}
// DeleteMods removes mods from the lock file then, if it is empty, deletes the file
func (l *WorkspaceLock) DeleteMods(mods VersionConstraintMap, parent *modconfig.Mod) {
for modName := range mods {
if parentDependencies := l.InstallCache[parent.GetModDependencyPath()]; parentDependencies != nil {
parentDependencies.Remove(modName)
}
}
}
// GetMod looks for a lock file entry matching the given mod name
func (l *WorkspaceLock) GetMod(modName string, parent *modconfig.Mod) *ResolvedVersionConstraint {
if parentDependencies := l.InstallCache[parent.GetModDependencyPath()]; parentDependencies != nil {
// look for this mod in the lock file entries for this parent
return parentDependencies[modName]
}
return nil
}
// GetLockedModVersions builds a ResolvedVersionListMap with the resolved versions
// for each item of the given VersionConstraintMap found in the lock file
func (l *WorkspaceLock) GetLockedModVersions(mods VersionConstraintMap, parent *modconfig.Mod) (ResolvedVersionListMap, error) {
var res = make(ResolvedVersionListMap)
for name, constraint := range mods {
resolvedConstraint, err := l.GetLockedModVersion(constraint, parent)
if err != nil {
return nil, err
}
if resolvedConstraint != nil {
res.Add(name, resolvedConstraint)
}
}
return res, nil
}
// GetLockedModVersion looks for a lock file entry matching the required constraint and returns nil if not found
func (l *WorkspaceLock) GetLockedModVersion(requiredModVersion *modconfig.ModVersionConstraint, parent *modconfig.Mod) (*ResolvedVersionConstraint, error) {
lockedVersion := l.GetMod(requiredModVersion.Name, parent)
if lockedVersion == nil {
return nil, nil
}
// verify the locked version satisfies the version constraint
if !requiredModVersion.Constraint.Check(lockedVersion.Version) {
return nil, nil
}
return lockedVersion, nil
}
// EnsureLockedModVersion looks for a lock file entry matching the required mod name
func (l *WorkspaceLock) EnsureLockedModVersion(requiredModVersion *modconfig.ModVersionConstraint, parent *modconfig.Mod) (*ResolvedVersionConstraint, error) {
lockedVersion := l.GetMod(requiredModVersion.Name, parent)
if lockedVersion == nil {
return nil, nil
}
// verify the locked version satisfies the version constraint
if !requiredModVersion.Constraint.Check(lockedVersion.Version) {
return nil, fmt.Errorf("failed to resolvedependencies for %s - locked version %s does not meet the constraint %s", parent.GetModDependencyPath(), modconfig.ModVersionFullName(requiredModVersion.Name, lockedVersion.Version), requiredModVersion.Constraint.Original)
}
return lockedVersion, nil
}
// GetLockedModVersionConstraint looks for a lock file entry matching the required mod version and if found,
// returns it in the form of a ModVersionConstraint
func (l *WorkspaceLock) GetLockedModVersionConstraint(requiredModVersion *modconfig.ModVersionConstraint, parent *modconfig.Mod) (*modconfig.ModVersionConstraint, error) {
lockedVersion, err := l.EnsureLockedModVersion(requiredModVersion, parent)
if err != nil {
// EnsureLockedModVersion returns an error if the locked version does not satisfy the requirement
return nil, err
}
if lockedVersion == nil {
// EnsureLockedModVersion returns nil if no locked version is found
return nil, nil
}
// create a new ModVersionConstraint using the locked version
lockedVersionFullName := modconfig.ModVersionFullName(requiredModVersion.Name, lockedVersion.Version)
return modconfig.NewModVersionConstraint(lockedVersionFullName)
}
// ContainsModVersion returns whether the lockfile contains the given mod version
func (l *WorkspaceLock) ContainsModVersion(modName string, modVersion *semver.Version) bool {
for _, modVersionMap := range l.InstallCache {
for lockName, lockVersion := range modVersionMap {
// TODO consider handling of metadata
if lockName == modName && lockVersion.Version.Equal(modVersion) && lockVersion.Version.Metadata() == modVersion.Metadata() {
return true
}
}
}
return false
}
func (l *WorkspaceLock) ContainsModConstraint(modName string, constraint *version_helpers.Constraints) bool {
for _, modVersionMap := range l.InstallCache {
for lockName, lockVersion := range modVersionMap {
if lockName == modName && lockVersion.Constraint == constraint.Original {
return true
}
}
}
return false
}
// Incomplete returned whether there are any missing dependencies
// (i.e. they exist in the lock file but ate not installed)
func (l *WorkspaceLock) Incomplete() bool {
return len(l.MissingVersions) > 0
}
// Empty returns whether the install cache is empty
func (l *WorkspaceLock) Empty() bool {
return len(l.InstallCache) == 0
}

10
steampipeconfig/version_map/workspace_lock_list.go Normal file
View File

@@ -0,0 +1,10 @@
package version_map
func (l *WorkspaceLock) GetModList(rootName string) string {
if len(l.InstallCache) == 0 {
return "No mods installed"
}
tree := l.InstallCache.GetDependencyTree(rootName)
return tree.String()
}

4
task/version_checker.go
View File

@@ -8,8 +8,8 @@ import (
"net/url"
"os"
SemVer "github.com/Masterminds/semver"
"github.com/fatih/color"
SemVer "github.com/hashicorp/go-version"
"github.com/olekukonko/tablewriter"
"github.com/spf13/viper"
"github.com/turbot/steampipe/constants"
@@ -18,7 +18,7 @@ import (
)
// the current version of the Steampipe CLI application
var currentVersion = version.String()
var currentVersion = version.SteampipeVersion.String()
type versionCheckResponse struct {
NewVersion string `json:"latest_version,omitempty"` // `json:"current_version"`

1
tests/acceptance/test_data/mod_install/mod-install.txt Normal file
View File

@@ -0,0 +1 @@
This is a folder used for acceptance tests.

80
tests/acceptance/test_files/021.mod-install.bats Normal file
View File

@@ -0,0 +1,80 @@
load "$LIB_BATS_ASSERT/load.bash"
load "$LIB_BATS_SUPPORT/load.bash"
@test "list with no mods installed" {
run steampipe mod list
assert_output 'No mods installed'
}
@test "install latest" {
run steampipe mod install github.com/turbot/steampipe-mod-aws-compliance
assert_output --partial 'Installed 1 mod:
local
└── github.com/turbot/steampipe-mod-aws-compliance'
# need the check the version from mod.sp file as well
}
@test "install latest and then run install" {
steampipe mod install github.com/turbot/steampipe-mod-aws-compliance
run steampipe mod install
assert_output 'All mods are up to date'
}
@test "install mod and list" {
steampipe mod install github.com/turbot/steampipe-mod-aws-compliance@0.10
run steampipe mod list
assert_output '
local
└── github.com/turbot/steampipe-mod-aws-compliance@v0.10'
}
@test "install old version when latest already installed" {
steampipe mod install github.com/turbot/steampipe-mod-aws-compliance
run steampipe mod install github.com/turbot/steampipe-mod-aws-compliance@0.1
assert_output '
Downgraded 1 mod:
local
└── github.com/turbot/steampipe-mod-aws-compliance@v0.1'
}
@test "install mod version, remove .steampipe folder and then run install" {
# install particular mod version, remove .steampipe folder and run mod install
steampipe mod install github.com/turbot/steampipe-mod-aws-compliance@0.1
rm -rf .steampipe
run steampipe mod install
# should install the same cached version
# better message
assert_output '
Installed 1 mod:
local
└── github.com/turbot/steampipe-mod-aws-compliance@v0.1'
}
@test "install mod version, remove .cache file and then run install" {
# install particular mod version, remove .mod.cache.json file and run mod install
steampipe mod install github.com/turbot/steampipe-mod-aws-compliance@0.1
rm -rf .mod.cache.json
run steampipe mod install
# should install the same cached version
# better message
assert_output '
Installed 1 mod:
local
└── github.com/turbot/steampipe-mod-aws-compliance@v0.1'
}
function teardown() {
rm -rf .steampipe/
rm -rf .mod.cache.json
rm -rf mod.sp
}
function setup() {
cd $FILE_PATH/test_data/mod_install
}

4
utils/useragent.go
View File

@@ -17,13 +17,13 @@ import (
const httpTimeout = 5 * time.Second
func getUserAgent() string {
return fmt.Sprintf("Turbot Steampipe/%s (+https://steampipe.io)", version.String())
return fmt.Sprintf("Turbot Steampipe/%s (+https://steampipe.io)", version.SteampipeVersion.String())
}
// BuildRequestPayload :: merges the provided payload with the standard payload that needs to be sent
func BuildRequestPayload(signature string, payload map[string]interface{}) *bytes.Buffer {
requestPayload := map[string]interface{}{
"version": version.String(),
"version": version.SteampipeVersion.String(),
"os_platform": runtime.GOOS,
"arch": runtime.GOARCH,
"signature": signature,

14
version/version.go
View File

@@ -8,7 +8,7 @@ package version
import (
"fmt"
goVersion "github.com/hashicorp/go-version"
"github.com/Masterminds/semver"
)
/**
@@ -24,22 +24,18 @@ var steampipeVersion = "0.11.0"
// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
var prerelease = "dev.0"
var prerelease = "dev.6"
// Version is an instance of version.Version. This has the secondary
// SteampipeVersion is an instance of semver.Version. This has the secondary
// benefit of verifying during tests and init time that our version is a
// proper semantic version, which should always be the case.
var Version *goVersion.Version
var SteampipeVersion *semver.Version
func init() {
versionString := steampipeVersion
if prerelease != "" {
versionString = fmt.Sprintf("%s-%s", steampipeVersion, prerelease)
}
Version = goVersion.Must(goVersion.NewVersion(versionString))
}
SteampipeVersion = semver.MustParse(versionString)
// String returns the complete version string, including prerelease
func String() string {
return Version.String()
}

46
version_helpers/constraints.go Normal file
View File

@@ -0,0 +1,46 @@
package version_helpers
import (
"github.com/Masterminds/semver"
)
// Constraints wraps semver.Constraints type, adding the Original property
type Constraints struct {
constraint *semver.Constraints
Original string
}
func NewConstraint(c string) (*Constraints, error) {
constraints, err := semver.NewConstraint(c)
if err != nil {
return nil, err
}
return &Constraints{
constraint: constraints,
Original: c,
}, nil
}
// Check tests if a version satisfies the constraints.
func (c Constraints) Check(v *semver.Version) bool {
return c.constraint.Check(v)
}
// Validate checks if a version satisfies a constraint. If not a slice of
// reasons for the failure are returned in addition to a bool.
func (c Constraints) Validate(v *semver.Version) (bool, []error) {
return c.constraint.Validate(v)
}
func (c Constraints) Equals(other *Constraints) bool {
return c.Original == other.Original
}
// IsPrerelease determines whether the constraint parses as a specifc version with prerelease or metadata set
func (c Constraints) IsPrerelease() bool {
v, err := semver.NewVersion(c.Original)
if err != nil {
return false
}
return v.Prerelease() != "" || v.Metadata() != ""
}

86
workspace/workspace.go
View File

@@ -11,7 +11,6 @@ import (
"github.com/fsnotify/fsnotify"
filehelpers "github.com/turbot/go-kit/files"
"github.com/turbot/go-kit/types"
typehelpers "github.com/turbot/go-kit/types"
"github.com/turbot/steampipe/constants"
"github.com/turbot/steampipe/db/db_common"
@@ -19,6 +18,7 @@ import (
"github.com/turbot/steampipe/steampipeconfig"
"github.com/turbot/steampipe/steampipeconfig/modconfig"
"github.com/turbot/steampipe/steampipeconfig/parse"
"github.com/turbot/steampipe/steampipeconfig/version_map"
"github.com/turbot/steampipe/utils"
)
@@ -140,7 +140,7 @@ func (w *Workspace) SetOnFileWatcherEventMessages(f func()) {
}
// access functions
// NOTE: all access functions lock 'loadLock' - this is to avoid conflicts with th efile watcher
// NOTE: all access functions lock 'loadLock' - this is to avoid conflicts with the file watcher
func (w *Workspace) Close() {
if w.watcher != nil {
@@ -219,33 +219,6 @@ func (w *Workspace) GetResourceMaps() *modconfig.WorkspaceResourceMaps {
return workspaceMap
}
// GetMod attempts to return the mod with a name matching 'modName'
// It first checks the workspace mod, then checks all mod dependencies
func (w *Workspace) GetMod(modName string) *modconfig.Mod {
// is it the workspace mod?
if modName == w.Mod.Name() {
return w.Mod
}
// try workspace mod dependencies
return w.Mods[modName]
}
// ModList returns a flat list of all mods - the workspace mod and depenfency mods
func (w *Workspace) ModList() []*modconfig.Mod {
var res = []*modconfig.Mod{w.Mod}
for _, m := range w.Mods {
res = append(res, m)
}
return res
}
// SaveWorkspaceMod searialises the workspace mode to <workspace path?.mod.sp
func (w *Workspace) SaveWorkspaceMod() error {
// TODO
return nil
}
// clear all resource maps
func (w *Workspace) reset() {
w.Queries = make(map[string]*modconfig.Query)
@@ -259,7 +232,7 @@ func (w *Workspace) reset() {
// check whether the workspace contains a modfile
// this will determine whether we load files recursively, and create pseudo resources for sql files
func (w *Workspace) setModfileExists() {
modFilePath := filepath.Join(w.Path, constants.WorkspaceModFileName)
modFilePath := constants.ModFilePath(w.Path)
_, err := os.Stat(modFilePath)
modFileExists := err == nil
@@ -285,7 +258,10 @@ func (w *Workspace) loadWorkspaceMod() error {
}
// build run context which we use to load the workspace
runCtx := w.getRunContext()
runCtx, err := w.getRunContext()
if err != nil {
return err
}
// add variables to runContext
runCtx.AddVariables(inputVariables)
@@ -314,12 +290,19 @@ func (w *Workspace) loadWorkspaceMod() error {
// build options used to load workspace
// set flags to create pseudo resources and a default mod if needed
func (w *Workspace) getRunContext() *parse.RunContext {
func (w *Workspace) getRunContext() (*parse.RunContext, error) {
parseFlag := parse.CreateDefaultMod
if w.loadPseudoResources {
parseFlag |= parse.CreatePseudoResources
}
return parse.NewRunContext(
// load the workspace lock
workspaceLock, err := version_map.LoadWorkspaceLock(w.Path)
if err != nil {
return nil, fmt.Errorf("failed to load installation cache from %s: %s", w.Path, err)
}
runCtx := parse.NewRunContext(
workspaceLock,
w.Path,
parseFlag,
&filehelpers.ListOptions{
@@ -329,13 +312,18 @@ func (w *Workspace) getRunContext() *parse.RunContext {
// only load .sp files
Include: filehelpers.InclusionsFromExtensions([]string{constants.ModDataExtension}),
})
return runCtx, nil
}
func (w *Workspace) loadWorkspaceResourceName() (*modconfig.WorkspaceResources, error) {
// build options used to load workspace
opts := w.getRunContext()
runCtx, err := w.getRunContext()
if err != nil {
return nil, err
}
workspaceResourceNames, err := steampipeconfig.LoadModResourceNames(w.Path, opts)
workspaceResourceNames, err := steampipeconfig.LoadModResourceNames(w.Path, runCtx)
if err != nil {
return nil, err
}
@@ -358,14 +346,15 @@ func (w *Workspace) buildQueryMap(modMap modconfig.ModMap) map[string]*modconfig
for _, q := range w.Mod.Queries {
// add 'local' alias
res[q.Name()] = q
longName := fmt.Sprintf("%s.query.%s", types.SafeString(w.Mod.ShortName), q.ShortName)
res[longName] = q
res[q.UnqualifiedName] = q
}
// for mod dependencies, add resources keyed by long name only
for _, mod := range modMap {
for _, q := range mod.Queries {
res[q.QualifiedName()] = q
// if this mod is a direct dependency of the workspace mod, add it to the map _without_ a verison
res[q.Name()] = q
}
}
return res
@@ -378,13 +367,13 @@ func (w *Workspace) buildControlMap(modMap modconfig.ModMap) map[string]*modconf
// for LOCAL resources, add map entries keyed by both short name: control.<shortName> and long name: <modName>.control.<shortName?
for _, c := range w.Mod.Controls {
res[c.Name()] = c
res[c.QualifiedName()] = c
res[c.UnqualifiedName] = c
}
// for mode dependencies, add resources keyed by long name only
for _, mod := range modMap {
for _, c := range mod.Controls {
res[c.QualifiedName()] = c
res[c.Name()] = c
}
}
return res
@@ -394,16 +383,16 @@ func (w *Workspace) buildBenchmarkMap(modMap modconfig.ModMap) map[string]*modco
// build a list of long and short names for these queries
var res = make(map[string]*modconfig.Benchmark)
// for LOCAL resources, add map entries keyed by both short name: benchmark.<shortName> and long name: <modName>.benchmark.<shortName?
// for LOCAL resources, add map entries keyed by both unqualified name: benchmark.<shortName> and full name: <modName>.benchmark.<shortName?
for _, b := range w.Mod.Benchmarks {
res[b.UnqualifiedName] = b
res[b.Name()] = b
res[b.QualifiedName()] = b
}
// for mod dependencies, add resources keyed by long name only
for _, mod := range modMap {
for _, c := range mod.Benchmarks {
res[c.QualifiedName()] = c
res[c.Name()] = c
}
}
return res
@@ -416,13 +405,13 @@ func (w *Workspace) buildReportMap(modMap modconfig.ModMap) map[string]*modconfi
// for LOCAL resources, add map entries keyed by both short name: benchmark.<shortName> and long name: <modName>.benchmark.<shortName?
for _, r := range w.Mod.Reports {
res[r.Name()] = r
res[r.QualifiedName()] = r
res[r.UnqualifiedName] = r
}
// for mod dependencies, add resources keyed by long name only
for _, mod := range modMap {
for _, r := range mod.Reports {
res[r.QualifiedName()] = r
res[r.Name()] = r
}
}
return res
@@ -436,13 +425,13 @@ func (w *Workspace) buildPanelMap(modMap modconfig.ModMap) map[string]*modconfig
for _, p := range w.Mod.Panels {
res[fmt.Sprintf("local.%s", p.Name())] = p
res[p.Name()] = p
res[p.QualifiedName()] = p
res[p.UnqualifiedName] = p
}
// for mod dependencies, add resources keyed by long name only
for _, mod := range modMap {
for _, p := range mod.Panels {
res[p.QualifiedName()] = p
res[p.Name()] = p
}
}
return res
@@ -532,7 +521,8 @@ func (w *Workspace) GetQueriesFromArgs(args []string) ([]string, *modconfig.Work
// ResolveQueryAndArgs attempts to resolve 'arg' to a query and query args
func (w *Workspace) ResolveQueryAndArgs(sqlString string) (string, modconfig.QueryProvider, error) {
var args *modconfig.QueryArgs
var args = &modconfig.QueryArgs{}
var err error
// if this looks like a named query or named control invocation, parse the sql string for arguments

23
workspace/workspace_requires.go → workspace/workspace_require.go
View File

@@ -6,7 +6,10 @@ import (
"sort"
"strings"
"github.com/hashicorp/go-version"
"github.com/turbot/steampipe/steampipeconfig/version_map"
"github.com/Masterminds/semver"
"github.com/turbot/steampipe/constants"
"github.com/turbot/steampipe/ociinstaller"
"github.com/turbot/steampipe/plugin"
@@ -48,10 +51,10 @@ func (w *Workspace) CheckRequiredPluginsInstalled() error {
return nil
}
func (w *Workspace) getRequiredPlugins() map[string]*version.Version {
if w.Mod.Requires != nil {
requiredPluginVersions := w.Mod.Requires.Plugins
requiredVersion := make(map[string]*version.Version)
func (w *Workspace) getRequiredPlugins() map[string]*semver.Version {
if w.Mod.Require != nil {
requiredPluginVersions := w.Mod.Require.Plugins
requiredVersion := make(version_map.VersionMap)
for _, pluginVersion := range requiredPluginVersions {
requiredVersion[pluginVersion.ShortName()] = pluginVersion.Version
}
@@ -60,12 +63,12 @@ func (w *Workspace) getRequiredPlugins() map[string]*version.Version {
return nil
}
func (w *Workspace) getInstalledPlugins() (map[string]*version.Version, error) {
installedPlugins := make(map[string]*version.Version)
func (w *Workspace) getInstalledPlugins() (version_map.VersionMap, error) {
installedPlugins := make(version_map.VersionMap)
installedPluginsData, _ := plugin.List(nil)
for _, plugin := range installedPluginsData {
org, name, _ := ociinstaller.NewSteampipeImageRef(plugin.Name).GetOrgNameAndStream()
semverVersion, err := version.NewVersion(plugin.Version)
semverVersion, err := semver.NewVersion(plugin.Version)
if err != nil {
continue
}
@@ -81,7 +84,7 @@ type requiredPluginVersion struct {
installedVersion string
}
func (v *requiredPluginVersion) SetRequiredVersion(requiredVersion *version.Version) {
func (v *requiredPluginVersion) SetRequiredVersion(requiredVersion *semver.Version) {
requiredVersionString := requiredVersion.String()
// if no required version was specified, the version will be 0.0.0
if requiredVersionString == "0.0.0" {
@@ -91,7 +94,7 @@ func (v *requiredPluginVersion) SetRequiredVersion(requiredVersion *version.Vers
}
}
func (v *requiredPluginVersion) SetInstalledVersion(installedVersion *version.Version) {
func (v *requiredPluginVersion) SetInstalledVersion(installedVersion *semver.Version) {
v.installedVersion = installedVersion.String()
}

2
workspace/workspace_test.go
View File

@@ -28,7 +28,7 @@ var testCasesLoadWorkspace = map[string]loadWorkspaceTest{
Mod: &modconfig.Mod{
ShortName: "w_1",
Title: toStringPointer("workspace 1"),
//ModDepends: []*modconfig.ModVersion{
//ModDepends: []*modconfig.ModVersionConstraint{
// {ShortName: "github.com/turbot/m1", Version: "0.0.0"},
// {ShortName: "github.com/turbot/m2", Version: "0.0.0"},
//},

5
workspace/workspace_variables.go
View File

@@ -16,7 +16,10 @@ import (
func (w *Workspace) getAllVariables() (map[string]*modconfig.Variable, error) {
// build options used to load workspace
runCtx := w.getRunContext()
runCtx, err := w.getRunContext()
if err != nil {
return nil, err
}
// only load variables blocks
runCtx.BlockTypes = []string{modconfig.BlockTypeVariable}
mod, err := steampipeconfig.LoadMod(w.Path, runCtx)