From 33f55e584f3acf7bc308bb950b7ef4ea550d45ca Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Tue, 21 Dec 2021 14:10:00 +0000 Subject: [PATCH] Add support for mod management commands. Closes #442. Closes #443 --- cmd/check.go | 14 +- cmd/mod.go | 258 ++++++++ cmd/plugin.go | 42 +- cmd/root.go | 3 +- cmdconfig/builder.go | 13 +- constants/args.go | 2 + constants/file_paths.go | 5 - constants/workspace.go | 15 +- control/controldisplay/csv.go | 2 +- control/controldisplay/formatter.go | 2 +- control/controlexecute/control_run.go | 103 ++-- control/controlexecute/execution_tree.go | 28 +- control/controlexecute/result_group.go | 6 + control/controlexecute/result_row.go | 2 +- db/db_client/db_client_session.go | 3 +- go.mod | 2 + go.sum | 5 +- interactive/interactive_client.go | 2 +- mod_installer/git.go | 68 +++ mod_installer/helpers.go | 16 + mod_installer/install.go | 28 + mod_installer/install_data.go | 119 ++++ mod_installer/install_opts.go | 8 + mod_installer/mod_installer.go | 554 +++++++++++++----- mod_installer/mod_installer_args.go | 63 ++ mod_installer/mod_installer_prune.go | 42 ++ mod_installer/mod_installer_test.go | 18 + mod_installer/mod_ref.go | 53 -- mod_installer/resolved_mod_ref.go | 60 +- mod_installer/summary_builder.go | 97 +++ mod_installer/test_data/mods/dep1/mod.sp | 8 +- mod_installer/test_data/mods/dep2/control.sp | 4 + mod_installer/test_data/mods/dep2/mod.sp | 7 + mod_installer/test_data/mods/dep3/control.sp | 4 + mod_installer/test_data/mods/dep3/mod.sp | 10 + mod_installer/test_data/mods/dep4/control.sp | 4 + mod_installer/test_data/mods/dep4/mod.sp | 10 + mod_installer/test_data/mods/dep5/control.sp | 4 + mod_installer/test_data/mods/dep5/mod.sp | 10 + .../test_data/mods/dep6_x/control.sp | 4 + mod_installer/test_data/mods/dep6_x/mod.sp | 10 + .../test_data/mods/dep7_x/control.sp | 4 + mod_installer/test_data/mods/dep7_x/mod.sp | 10 + .../test_data/mods/dep8_x/control.sp | 4 + mod_installer/test_data/mods/dep8_x/mod.sp | 10 + mod_installer/test_data/mods/dep_empty/c.sp | 8 - mod_installer/uninstall.go | 29 + steampipeconfig/connection_plugin.go | 2 +- steampipeconfig/connection_updates.go | 4 +- steampipeconfig/load_mod.go | 86 ++- steampipeconfig/load_mod_test.go | 10 +- steampipeconfig/modconfig/benchmark.go | 59 +- steampipeconfig/modconfig/control.go | 58 +- steampipeconfig/modconfig/interfaces.go | 1 + steampipeconfig/modconfig/local.go | 1 + steampipeconfig/modconfig/mod.go | 489 +++++++++------- steampipeconfig/modconfig/mod_map.go | 2 +- steampipeconfig/modconfig/mod_name.go | 42 ++ .../modconfig/mod_resource_tree.go | 154 +++++ steampipeconfig/modconfig/mod_validate.go | 32 + steampipeconfig/modconfig/mod_version.go | 69 --- .../mod_version_constrain_collection.go | 25 + .../modconfig/mod_version_constraint.go | 114 ++++ steampipeconfig/modconfig/open_graph.go | 4 +- steampipeconfig/modconfig/panel.go | 19 +- steampipeconfig/modconfig/parse_names.go | 11 + steampipeconfig/modconfig/plugin_version.go | 6 +- .../modconfig/prepared_statement.go | 3 + steampipeconfig/modconfig/query.go | 10 +- steampipeconfig/modconfig/report.go | 19 +- steampipeconfig/modconfig/requires.go | 109 +++- steampipeconfig/modconfig/variable.go | 10 +- steampipeconfig/parse/decode.go | 14 +- steampipeconfig/parse/installed_mod.go | 4 +- steampipeconfig/parse/parser.go | 19 +- steampipeconfig/parse/run_context.go | 65 +- .../test_data/demo/control_demo/mod.sp | 2 +- steampipeconfig/test_data/dep_test2/mod.sp | 2 +- .../version_map/dependency_version_map.go | 103 ++++ .../resolved_version_constraint.go | 20 + .../version_map/resolved_version_list_map.go | 49 ++ .../version_map/resolved_version_map.go | 21 + .../version_map/version_constraint_map.go | 5 + .../version_map/version_list_map.go | 31 + steampipeconfig/version_map/version_map.go | 8 + steampipeconfig/version_map/workspace_lock.go | 294 ++++++++++ .../version_map/workspace_lock_list.go | 10 + task/version_checker.go | 4 +- .../test_data/mod_install/mod-install.txt | 1 + .../test_files/021.mod-install.bats | 80 +++ utils/useragent.go | 4 +- version/version.go | 14 +- version_helpers/constraints.go | 46 ++ workspace/workspace.go | 86 ++- ...space_requires.go => workspace_require.go} | 23 +- workspace/workspace_test.go | 2 +- workspace/workspace_variables.go | 5 +- 97 files changed, 3135 insertions(+), 889 deletions(-) create mode 100644 cmd/mod.go create mode 100644 mod_installer/git.go create mode 100644 mod_installer/helpers.go create mode 100644 mod_installer/install.go create mode 100644 mod_installer/install_data.go create mode 100644 mod_installer/install_opts.go create mode 100644 mod_installer/mod_installer_args.go create mode 100644 mod_installer/mod_installer_prune.go create mode 100644 mod_installer/mod_installer_test.go delete mode 100644 mod_installer/mod_ref.go create mode 100644 mod_installer/summary_builder.go create mode 100644 mod_installer/test_data/mods/dep2/control.sp create mode 100644 mod_installer/test_data/mods/dep2/mod.sp create mode 100644 mod_installer/test_data/mods/dep3/control.sp create mode 100644 mod_installer/test_data/mods/dep3/mod.sp create mode 100644 mod_installer/test_data/mods/dep4/control.sp create mode 100644 mod_installer/test_data/mods/dep4/mod.sp create mode 100644 mod_installer/test_data/mods/dep5/control.sp create mode 100644 mod_installer/test_data/mods/dep5/mod.sp create mode 100644 mod_installer/test_data/mods/dep6_x/control.sp create mode 100644 mod_installer/test_data/mods/dep6_x/mod.sp create mode 100644 mod_installer/test_data/mods/dep7_x/control.sp create mode 100644 mod_installer/test_data/mods/dep7_x/mod.sp create mode 100644 mod_installer/test_data/mods/dep8_x/control.sp create mode 100644 mod_installer/test_data/mods/dep8_x/mod.sp delete mode 100644 mod_installer/test_data/mods/dep_empty/c.sp create mode 100644 mod_installer/uninstall.go create mode 100644 steampipeconfig/modconfig/mod_name.go create mode 100644 steampipeconfig/modconfig/mod_resource_tree.go create mode 100644 steampipeconfig/modconfig/mod_validate.go delete mode 100644 steampipeconfig/modconfig/mod_version.go create mode 100644 steampipeconfig/modconfig/mod_version_constrain_collection.go create mode 100644 steampipeconfig/modconfig/mod_version_constraint.go create mode 100644 steampipeconfig/version_map/dependency_version_map.go create mode 100644 steampipeconfig/version_map/resolved_version_constraint.go create mode 100644 steampipeconfig/version_map/resolved_version_list_map.go create mode 100644 steampipeconfig/version_map/resolved_version_map.go create mode 100644 steampipeconfig/version_map/version_constraint_map.go create mode 100644 steampipeconfig/version_map/version_list_map.go create mode 100644 steampipeconfig/version_map/version_map.go create mode 100644 steampipeconfig/version_map/workspace_lock.go create mode 100644 steampipeconfig/version_map/workspace_lock_list.go create mode 100644 tests/acceptance/test_data/mod_install/mod-install.txt create mode 100644 tests/acceptance/test_files/021.mod-install.bats create mode 100644 version_helpers/constraints.go rename workspace/{workspace_requires.go => workspace_require.go} (87%) diff --git a/cmd/check.go b/cmd/check.go index 570ef1193..fe43a37ac 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -10,6 +10,8 @@ import ( "sync" "time" + "github.com/turbot/steampipe/mod_installer" + "github.com/briandowns/spinner" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -95,7 +97,8 @@ You may specify one or more benchmarks or controls to run (separated by a space) // where args passed to StringArrayFlag are not parsed and used raw AddStringArrayFlag(constants.ArgVariable, "", nil, "Specify the value of a variable"). AddStringFlag(constants.ArgWhere, "", "", "SQL 'where' clause, or named query, used to filter controls (cannot be used with '--tag')"). - AddIntFlag(constants.ArgMaxParallel, "", constants.DefaultMaxConnections, "The maximum number of parallel executions", cmdconfig.FlagOptions.Hidden()) + AddIntFlag(constants.ArgMaxParallel, "", constants.DefaultMaxConnections, "The maximum number of parallel executions", cmdconfig.FlagOptions.Hidden()). + AddBoolFlag(constants.ArgModInstall, "", true, "Specify whether to install mod depdencies before runnign the check") return cmd } @@ -198,6 +201,15 @@ func initialiseCheck(ctx context.Context, spinner *spinner.Spinner) *checkInitDa result: &db_common.InitResult{}, } + if viper.GetBool(constants.ArgModInstall) { + opts := &mod_installer.InstallOpts{WorkspacePath: viper.GetString(constants.ArgWorkspaceChDir)} + _, err := mod_installer.InstallWorkspaceDependencies(opts) + if err != nil { + initData.result.Error = err + return initData + } + } + cmdconfig.Viper().Set(constants.ConfigKeyShowInteractiveOutput, false) err := validateOutputFormat() diff --git a/cmd/mod.go b/cmd/mod.go new file mode 100644 index 000000000..2f6f36509 --- /dev/null +++ b/cmd/mod.go @@ -0,0 +1,258 @@ +package cmd + +import ( + "fmt" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/turbot/go-kit/helpers" + "github.com/turbot/steampipe/cmdconfig" + "github.com/turbot/steampipe/constants" + "github.com/turbot/steampipe/mod_installer" + "github.com/turbot/steampipe/steampipeconfig/modconfig" + "github.com/turbot/steampipe/steampipeconfig/parse" + "github.com/turbot/steampipe/utils" +) + +// mod management commands +func modCmd() *cobra.Command { + + var cmd = &cobra.Command{ + Use: "mod [command]", + Args: cobra.NoArgs, + Short: "Steampipe mod management", + Long: `Steampipe mod management.`, + } + + cmd.AddCommand(modInstallCmd()) + cmd.AddCommand(modUninstallCmd()) + cmd.AddCommand(modUpdateCmd()) + cmd.AddCommand(modPruneCmd()) + cmd.AddCommand(modListCmd()) + cmd.AddCommand(modInitCmd()) + + return cmd +} + +// install +func modInstallCmd() *cobra.Command { + var cmd = &cobra.Command{ + Use: "install", + Run: runModInstallCmd, + Short: "Install mod dependencies", + Long: `Install mod dependencies. +`, + } + + cmdconfig.OnCmd(cmd). + AddBoolFlag(constants.ArgPrune, "", true, "Remove unreferenced mods after installation"). + AddBoolFlag(constants.ArgDryRun, "", false, "Show which mods would be installed or uninstalled without performing the installation") + return cmd +} + +func runModInstallCmd(cmd *cobra.Command, args []string) { + utils.LogTime("cmd.runModInstallCmd") + defer func() { + utils.LogTime("cmd.runModInstallCmd end") + if r := recover(); r != nil { + utils.ShowError(helpers.ToError(r)) + exitCode = 1 + } + }() + + // if any mod names were passed as args, convert into formed mod names + opts := newInstallOpts(cmd, args...) + installData, err := mod_installer.InstallWorkspaceDependencies(opts) + utils.FailOnError(err) + + fmt.Println(mod_installer.BuildInstallSummary(installData)) +} + +// uninstall +func modUninstallCmd() *cobra.Command { + var cmd = &cobra.Command{ + Use: "uninstall", + Run: runModUninstallCmd, + Short: "Uninstall mod dependencies", + Long: `Uninstall mod dependencies. +`, + } + + cmdconfig.OnCmd(cmd). + AddBoolFlag(constants.ArgPrune, "", true, "Remove unreferenced mods after uninstallation"). + AddBoolFlag(constants.ArgDryRun, "", false, "Show which mods would be uninstalled without removing them") + + return cmd +} + +func runModUninstallCmd(cmd *cobra.Command, args []string) { + utils.LogTime("cmd.runModInstallCmd") + defer func() { + utils.LogTime("cmd.runModInstallCmd end") + if r := recover(); r != nil { + utils.ShowError(helpers.ToError(r)) + exitCode = 1 + } + }() + + opts := newInstallOpts(cmd, args...) + installData, err := mod_installer.UninstallWorkspaceDependencies(opts) + utils.FailOnError(err) + + fmt.Println(mod_installer.BuildUninstallSummary(installData)) +} + +// update +func modUpdateCmd() *cobra.Command { + var cmd = &cobra.Command{ + Use: "update", + Run: runModUpdateCmd, + Short: "Update workspace dependencies", + Long: `Update workspace dependencies. +`, + } + + cmdconfig.OnCmd(cmd). + AddBoolFlag(constants.ArgPrune, "", true, "Remove unreferenced mods after installation"). + AddBoolFlag(constants.ArgDryRun, "", false, "Show which mods would be updated without updating them") + + return cmd +} + +func runModUpdateCmd(cmd *cobra.Command, args []string) { + utils.LogTime("cmd.runModUpdateCmd") + defer func() { + utils.LogTime("cmd.runModUpdateCmd end") + if r := recover(); r != nil { + utils.ShowError(helpers.ToError(r)) + exitCode = 1 + } + }() + + opts := newInstallOpts(cmd, args...) + + installData, err := mod_installer.InstallWorkspaceDependencies(opts) + utils.FailOnError(err) + + fmt.Println(mod_installer.BuildInstallSummary(installData)) +} + +// list +func modListCmd() *cobra.Command { + var cmd = &cobra.Command{ + Use: "list", + Run: runModListCmd, + Short: "List mod dependencies", + Long: `List mod dependencies. +`, + } + + cmdconfig.OnCmd(cmd) + return cmd +} + +func runModListCmd(cmd *cobra.Command, _ []string) { + utils.LogTime("cmd.runModListCmd") + defer func() { + utils.LogTime("cmd.runModListCmd end") + if r := recover(); r != nil { + utils.ShowError(helpers.ToError(r)) + exitCode = 1 + } + }() + opts := newInstallOpts(cmd) + installer, err := mod_installer.NewModInstaller(opts) + utils.FailOnError(err) + + treeString := installer.GetModList() + if len(strings.Split(treeString, "\n")) > 1 { + fmt.Println() + } + fmt.Println(treeString) +} + +// prune +func modPruneCmd() *cobra.Command { + var cmd = &cobra.Command{ + Use: "prune", + Run: runModPruneCmd, + Short: "Prune mod dependencies", + Long: `Prune mod dependencies. +`, + } + + cmdconfig.OnCmd(cmd) + return cmd +} + +func runModPruneCmd(cmd *cobra.Command, args []string) { + utils.LogTime("cmd.runModPruneCmd") + defer func() { + utils.LogTime("cmd.runModPruneCmd end") + if r := recover(); r != nil { + utils.ShowError(helpers.ToError(r)) + exitCode = 1 + } + }() + + opts := &mod_installer.InstallOpts{ + WorkspacePath: viper.GetString(constants.ArgWorkspaceChDir), + DryRun: viper.GetBool(constants.ArgDryRun), + } + + // install workspace dependencies + installer, err := mod_installer.NewModInstaller(opts) + utils.FailOnError(err) + + unusedMods, err := installer.Prune() + utils.FailOnError(err) + + if count := len(unusedMods.FlatMap()); count > 0 { + fmt.Println(mod_installer.BuildPruneSummary(unusedMods)) + } +} + +// init +func modInitCmd() *cobra.Command { + var cmd = &cobra.Command{ + Use: "init", + Run: runModInitCmd, + Short: "Create a modfile in the current directory", + Long: `Create a modfile in the current directory`, + } + + cmdconfig.OnCmd(cmd) + return cmd +} + +func runModInitCmd(cmd *cobra.Command, args []string) { + utils.LogTime("cmd.runModInitCmd") + defer func() { + utils.LogTime("cmd.runModInitCmd end") + if r := recover(); r != nil { + utils.ShowError(helpers.ToError(r)) + exitCode = 1 + } + }() + workspacePath := viper.GetString(constants.ArgWorkspaceChDir) + if parse.ModfileExists(workspacePath) { + fmt.Println("Working folder already contains a mod definition file") + return + } + mod := modconfig.CreateDefaultMod(workspacePath) + utils.FailOnError(mod.Save()) + fmt.Printf("Created mod definition file '%s'\n", constants.ModFilePath(workspacePath)) +} + +// helpers + +func newInstallOpts(cmd *cobra.Command, args ...string) *mod_installer.InstallOpts { + opts := &mod_installer.InstallOpts{ + WorkspacePath: viper.GetString(constants.ArgWorkspaceChDir), + DryRun: viper.GetBool(constants.ArgDryRun), + ModArgs: args, + Command: cmd.Name(), + } + return opts +} diff --git a/cmd/plugin.go b/cmd/plugin.go index a21b933da..7cdd80180 100644 --- a/cmd/plugin.go +++ b/cmd/plugin.go @@ -112,7 +112,7 @@ Examples: cmdconfig. OnCmd(cmd). - AddBoolFlag("all", "", false, "Update all plugins to its latest available version"). + AddBoolFlag(constants.ArgAll, "", false, "Update all plugins to its latest available version"). AddBoolFlag(constants.ArgHelp, "h", false, "Help for plugin update") return cmd @@ -273,25 +273,14 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) { exitCode = 1 } }() - // args to 'plugin update' -- one or more plugins to install - // These can be simple names ('aws') for "standard" plugins, or - // full refs to the OCI image (us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0) - plugins := append([]string{}, args...) - if len(plugins) == 0 && !(cmdconfig.Viper().GetBool("all")) { + // args to 'plugin update' -- one or more plugins to update + // These can be simple names ('aws') for "standard" plugins, + // or full refs to the OCI image (us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0) + plugins, err := resolveUpdatePluginsFromArgs(args) + if err != nil { fmt.Println() - utils.ShowError(fmt.Errorf("you need to provide at least one plugin to update or use the %s flag", constants.Bold("--all"))) - fmt.Println() - cmd.Help() - fmt.Println() - exitCode = 2 - return - } - - if len(plugins) > 0 && cmdconfig.Viper().GetBool("all") { - // we can't allow update and install at the same time - fmt.Println() - utils.ShowError(fmt.Errorf("%s cannot be used when updating specific plugins", constants.Bold("`--all`"))) + utils.ShowError(err) fmt.Println() cmd.Help() fmt.Println() @@ -320,7 +309,7 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) { // a leading blank line - since we always output multiple lines fmt.Println() - if cmdconfig.Viper().GetBool("all") { + if cmdconfig.Viper().GetBool(constants.ArgAll) { for k, v := range versionData.Plugins { ref := ociinstaller.NewSteampipeImageRef(k) org, name, stream := ref.GetOrgNameAndStream() @@ -421,6 +410,21 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) { fmt.Println() } +func resolveUpdatePluginsFromArgs(args []string) ([]string, error) { + plugins := append([]string{}, args...) + + if len(plugins) == 0 && !(cmdconfig.Viper().GetBool("all")) { + // either plugin name(s) or "all" must be provided + return nil, fmt.Errorf("you need to provide at least one plugin to update or use the %s flag", constants.Bold("--all")) + } + + if len(plugins) > 0 && cmdconfig.Viper().GetBool(constants.ArgAll) { + // we can't allow update and install at the same time + return nil, fmt.Errorf("%s cannot be used when updating specific plugins", constants.Bold("`--all`")) + } + return plugins, nil +} + // start service if necessary and refresh connections func refreshConnectionsIfNecessary(ctx context.Context, reports []display.InstallReport, isUpdate bool) error { // get count of skipped reports diff --git a/cmd/root.go b/cmd/root.go index 90c978f23..17a00a03b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -24,7 +24,7 @@ var exitCode int // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "steampipe [--version] [--help] COMMAND [args]", - Version: version.String(), + Version: version.SteampipeVersion.String(), PersistentPreRun: func(cmd *cobra.Command, args []string) { utils.LogTime("cmd.root.PersistentPreRun start") defer utils.LogTime("cmd.root.PersistentPreRun end") @@ -172,6 +172,7 @@ func AddCommands() { queryCmd(), checkCmd(), serviceCmd(), + modCmd(), generateCompletionScriptsCmd(), pluginManagerCmd(), ) diff --git a/cmdconfig/builder.go b/cmdconfig/builder.go index 5cadd7e3b..3707443f4 100644 --- a/cmdconfig/builder.go +++ b/cmdconfig/builder.go @@ -14,7 +14,7 @@ type CmdBuilder struct { bindings map[string]*pflag.Flag } -// OnCmd :: starts a config builder wrapping over the provided *cobra.Command +// OnCmd starts a config builder wrapping over the provided *cobra.Command func OnCmd(cmd *cobra.Command) *CmdBuilder { cfg := new(CmdBuilder) cfg.cmd = cmd @@ -50,7 +50,7 @@ func OnCmd(cmd *cobra.Command) *CmdBuilder { return cfg } -// Helper function to add a string flag to a command +// AddStringFlag is a helper function to add a string flag to a command func (c *CmdBuilder) AddStringFlag(name string, shorthand string, defaultValue string, desc string, opts ...flagOpt) *CmdBuilder { c.cmd.Flags().StringP(name, shorthand, defaultValue, desc) c.bindings[name] = c.cmd.Flags().Lookup(name) @@ -61,7 +61,7 @@ func (c *CmdBuilder) AddStringFlag(name string, shorthand string, defaultValue s return c } -// Helper function to add an integer flag to a command +// AddIntFlag is a helper function to add an integer flag to a command func (c *CmdBuilder) AddIntFlag(name, shorthand string, defaultValue int, desc string, opts ...flagOpt) *CmdBuilder { c.cmd.Flags().IntP(name, shorthand, defaultValue, desc) c.bindings[name] = c.cmd.Flags().Lookup(name) @@ -71,7 +71,7 @@ func (c *CmdBuilder) AddIntFlag(name, shorthand string, defaultValue int, desc s return c } -// Helper function to add a boolean flag to a command +// AddBoolFlag ia s helper function to add a boolean flag to a command func (c *CmdBuilder) AddBoolFlag(name, shorthand string, defaultValue bool, desc string, opts ...flagOpt) *CmdBuilder { c.cmd.Flags().BoolP(name, shorthand, defaultValue, desc) c.bindings[name] = c.cmd.Flags().Lookup(name) @@ -81,7 +81,7 @@ func (c *CmdBuilder) AddBoolFlag(name, shorthand string, defaultValue bool, desc return c } -// Helper function to add a flag that accepts an array of strings +// AddStringSliceFlag is a helper function to add a flag that accepts an array of strings func (c *CmdBuilder) AddStringSliceFlag(name, shorthand string, defaultValue []string, desc string, opts ...flagOpt) *CmdBuilder { c.cmd.Flags().StringSliceP(name, shorthand, defaultValue, desc) c.bindings[name] = c.cmd.Flags().Lookup(name) @@ -91,6 +91,7 @@ func (c *CmdBuilder) AddStringSliceFlag(name, shorthand string, defaultValue []s return c } +// AddStringArrayFlag is a helper function to add a flag that accepts an array of strings func (c *CmdBuilder) AddStringArrayFlag(name, shorthand string, defaultValue []string, desc string, opts ...flagOpt) *CmdBuilder { c.cmd.Flags().StringArrayP(name, shorthand, defaultValue, desc) c.bindings[name] = c.cmd.Flags().Lookup(name) @@ -100,7 +101,7 @@ func (c *CmdBuilder) AddStringArrayFlag(name, shorthand string, defaultValue []s return c } -// Helper function to add a flag that accepts a map of strings +// AddStringMapStringFlag is a helper function to add a flag that accepts a map of strings func (c *CmdBuilder) AddStringMapStringFlag(name, shorthand string, defaultValue map[string]string, desc string, opts ...flagOpt) *CmdBuilder { c.cmd.Flags().StringToStringP(name, shorthand, defaultValue, desc) c.bindings[name] = c.cmd.Flags().Lookup(name) diff --git a/constants/args.go b/constants/args.go index 50adc9e7e..f9977d34b 100644 --- a/constants/args.go +++ b/constants/args.go @@ -37,6 +37,8 @@ const ( ArgVarFile = "var-file" ArgConnectionString = "connection-string" ArgCheckDisplayWidth = "check-display-width" + ArgPrune = "prune" + ArgModInstall = "mod-install" ) /// metaquery mode arguments diff --git a/constants/file_paths.go b/constants/file_paths.go index 6c12fd17d..c92590649 100644 --- a/constants/file_paths.go +++ b/constants/file_paths.go @@ -43,11 +43,6 @@ func ConnectionStatePath() string { return filepath.Join(InternalDir(), ConnectionsStateFileName) } -// ModsDir returns the path to the mods directory (creates if missing) -func ModsDir() string { - return steampipeSubDir("mods") -} - // ConfigDir returns the path to the config directory (creates if missing) func ConfigDir() string { return steampipeSubDir("config") diff --git a/constants/workspace.go b/constants/workspace.go index 3e94c923d..900d694a8 100644 --- a/constants/workspace.go +++ b/constants/workspace.go @@ -2,6 +2,7 @@ package constants import ( "path" + "path/filepath" ) // mod related constants @@ -10,15 +11,25 @@ const ( WorkspaceModDir = "mods" WorkspaceConfigFileName = "workspace.spc" WorkspaceIgnoreFile = ".steampipeignore" - WorkspaceDefaultModName = "local" - WorkspaceModFileName = "mod.sp" + ModFileName = "mod.sp" DefaultVarsFileName = "steampipe.spvars" + WorkspaceLockFileName = ".mod.cache.json" MaxControlRunAttempts = 2 ) func WorkspaceModPath(workspacePath string) string { return path.Join(workspacePath, WorkspaceDataDir, WorkspaceModDir) } + +func WorkspaceLockPath(workspacePath string) string { + return path.Join(workspacePath, WorkspaceLockFileName) +} + func DefaultVarsFilePath(workspacePath string) string { return path.Join(workspacePath, DefaultVarsFileName) } + +func ModFilePath(modFolder string) string { + modFilePath := filepath.Join(modFolder, ModFileName) + return modFilePath +} diff --git a/control/controldisplay/csv.go b/control/controldisplay/csv.go index dac6cc5a8..1593dc97d 100644 --- a/control/controldisplay/csv.go +++ b/control/controldisplay/csv.go @@ -57,7 +57,7 @@ func (r CSVRenderer) renderControl(run *controlexecute.ControlRun, group *contro } tags := make(map[string]string) if run.Control.Tags != nil { - tags = *run.Control.Tags + tags = run.Control.Tags } for _, prop := range r.columns.TagColumns { val := tags[prop] diff --git a/control/controldisplay/formatter.go b/control/controldisplay/formatter.go index 2e74e0c86..f0f63a17a 100644 --- a/control/controldisplay/formatter.go +++ b/control/controldisplay/formatter.go @@ -110,7 +110,7 @@ func (j *NullFormatter) FileExtension() string { } var formatterTemplateFuncMap template.FuncMap = template.FuncMap{ - "steampipeversion": func() string { return version.String() }, + "steampipeversion": func() string { return version.SteampipeVersion.String() }, "workingdir": func() string { wd, _ := os.Getwd(); return wd }, "asstr": func(i reflect.Value) string { return fmt.Sprintf("%v", i) }, "statusicon": func(status string) string { diff --git a/control/controlexecute/control_run.go b/control/controlexecute/control_run.go index 9e47d2169..e38a3d6a8 100644 --- a/control/controlexecute/control_run.go +++ b/control/controlexecute/control_run.go @@ -65,10 +65,15 @@ type ControlRun struct { } func NewControlRun(control *modconfig.Control, group *ResultGroup, executionTree *ExecutionTree) *ControlRun { - res := &ControlRun{ - Control: control, + controlId := control.Name() + // only show qualified control names for controls from dependent mods + if control.Mod.Name() == executionTree.workspace.Mod.Name() { + controlId = control.UnqualifiedName + } - ControlId: control.Name(), + res := &ControlRun{ + Control: control, + ControlId: controlId, Description: typehelpers.SafeString(control.Description), Severity: typehelpers.SafeString(control.Severity), Title: typehelpers.SafeString(control.Title), @@ -125,26 +130,6 @@ func (r *ControlRun) setSearchPath(ctx context.Context, session *db_common.Datab return err } -func (r *ControlRun) getCurrentSearchPath(ctx context.Context, session *db_common.DatabaseSession) ([]string, error) { - utils.LogTime("ControlRun.getCurrentSearchPath start") - defer utils.LogTime("ControlRun.getCurrentSearchPath end") - - row := session.Connection.QueryRowContext(ctx, "show search_path") - pathAsString := "" - err := row.Scan(&pathAsString) - if err != nil { - return nil, err - } - currentSearchPath := strings.Split(pathAsString, ",") - // unescape search path - for idx, p := range currentSearchPath { - p = strings.Join(strings.Split(p, "\""), "") - p = strings.TrimSpace(p) - currentSearchPath[idx] = p - } - return currentSearchPath, nil -} - func (r *ControlRun) Execute(ctx context.Context, client db_common.Client) { log.Printf("[TRACE] begin ControlRun.Start: %s\n", r.Control.Name()) defer log.Printf("[TRACE] end ControlRun.Start: %s\n", r.Control.Name()) @@ -246,6 +231,41 @@ func (r *ControlRun) Execute(ctx context.Context, client db_common.Client) { log.Printf("[TRACE] finish result for, %s\n", control.Name()) } +func (r *ControlRun) SetError(err error) { + if err == nil { + return + } + r.runError = utils.TransformErrorToSteampipe(err) + + // update error count + r.Summary.Error++ + r.setRunStatus(ControlRunError) +} + +func (r *ControlRun) GetError() error { + return r.runError +} + +func (r *ControlRun) getCurrentSearchPath(ctx context.Context, session *db_common.DatabaseSession) ([]string, error) { + utils.LogTime("ControlRun.getCurrentSearchPath start") + defer utils.LogTime("ControlRun.getCurrentSearchPath end") + + row := session.Connection.QueryRowContext(ctx, "show search_path") + pathAsString := "" + err := row.Scan(&pathAsString) + if err != nil { + return nil, err + } + currentSearchPath := strings.Split(pathAsString, ",") + // unescape search path + for idx, p := range currentSearchPath { + p = strings.Join(strings.Split(p, "\""), "") + p = strings.TrimSpace(p) + currentSearchPath[idx] = p + } + return currentSearchPath, nil +} + func (r *ControlRun) getControlQueryContext(ctx context.Context) context.Context { // create a context with a deadline shouldBeDoneBy := time.Now().Add(controlQueryTimeout) @@ -254,6 +274,17 @@ func (r *ControlRun) getControlQueryContext(ctx context.Context) context.Context return newCtx } +func (r *ControlRun) GetRunStatus() ControlRunStatus { + r.stateLock.Lock() + defer r.stateLock.Unlock() + return r.runStatus +} + +func (r *ControlRun) Finished() bool { + status := r.GetRunStatus() + return status == ControlRunComplete || status == ControlRunError +} + func (r *ControlRun) resolveControlQuery(control *modconfig.Control) (string, error) { query, err := r.executionTree.workspace.ResolveControlQuery(control, nil) if err != nil { @@ -345,21 +376,6 @@ func (r *ControlRun) createdOrderedResultRows() { } } -func (r *ControlRun) SetError(err error) { - if err == nil { - return - } - r.runError = utils.TransformErrorToSteampipe(err) - - // update error count - r.Summary.Error++ - r.setRunStatus(ControlRunError) -} - -func (r *ControlRun) GetError() error { - return r.runError -} - func (r *ControlRun) setRunStatus(status ControlRunStatus) { r.stateLock.Lock() r.runStatus = status @@ -377,14 +393,3 @@ func (r *ControlRun) setRunStatus(status ControlRunStatus) { r.doneChan <- true } } - -func (r *ControlRun) GetRunStatus() ControlRunStatus { - r.stateLock.Lock() - defer r.stateLock.Unlock() - return r.runStatus -} - -func (r *ControlRun) Finished() bool { - status := r.GetRunStatus() - return status == ControlRunComplete || status == ControlRunError -} diff --git a/control/controlexecute/execution_tree.go b/control/controlexecute/execution_tree.go index 3f03e4d7f..65d9491bc 100644 --- a/control/controlexecute/execution_tree.go +++ b/control/controlexecute/execution_tree.go @@ -49,13 +49,13 @@ func NewExecutionTree(ctx context.Context, workspace *workspace.Workspace, clien } // now identify the root item of the control list - rootItems, err := executionTree.getExecutionRootFromArg(arg) + rootItem, err := executionTree.getExecutionRootFromArg(arg) if err != nil { return nil, err } // build tree of result groups, starting with a synthetic 'root' node - executionTree.Root = NewRootResultGroup(executionTree, rootItems...) + executionTree.Root = NewRootResultGroup(executionTree, rootItem) // after tree has built, ControlCount will be set - create progress rendered executionTree.progress = NewControlProgressRenderer(len(executionTree.controlRuns)) @@ -192,16 +192,18 @@ func (e *ExecutionTree) ShouldIncludeControl(controlName string) bool { // - if the arg is a benchmark name, the root will be the Benchmark with that name // - if the arg is a mod name, the root will be the Mod with that name // - if the arg is 'all' the root will be a node with all Mods as children -func (e *ExecutionTree) getExecutionRootFromArg(arg string) ([]modconfig.ModTreeItem, error) { - var res []modconfig.ModTreeItem +func (e *ExecutionTree) getExecutionRootFromArg(arg string) (modconfig.ModTreeItem, error) { // special case handling for the string "all" if arg == "all" { - // - // build list of all workspace mods - these will act as root items - for _, m := range e.workspace.Mods { - res = append(res, m) + // return the workspace mod as root + return e.workspace.Mod, nil + } + + // if the arg is the name of one of the workjspace dependen + for _, mod := range e.workspace.Mods { + if mod.ShortName == arg { + return mod, nil } - return res, nil } // what resource type is arg? @@ -215,17 +217,17 @@ func (e *ExecutionTree) getExecutionRootFromArg(arg string) ([]modconfig.ModTree case modconfig.BlockTypeControl: // check whether the arg is a control name if control, ok := e.workspace.Controls[arg]; ok { - return []modconfig.ModTreeItem{control}, nil + return control, nil } case modconfig.BlockTypeBenchmark: // look in the workspace control group map for this control group if benchmark, ok := e.workspace.Benchmarks[arg]; ok { - return []modconfig.ModTreeItem{benchmark}, nil + return benchmark, nil } case modconfig.BlockTypeMod: // get all controls for the mod if mod, ok := e.workspace.Mods[arg]; ok { - return []modconfig.ModTreeItem{mod}, nil + return mod, nil } } return nil, fmt.Errorf("no controls found matching argument '%s'", arg) @@ -280,7 +282,7 @@ func (e *ExecutionTree) GetAllTags() []string { var tagColumns []string for _, r := range e.controlRuns { if r.Control.Tags != nil { - for tag := range *r.Control.Tags { + for tag := range r.Control.Tags { if !tagColumnMap[tag] { tagColumns = append(tagColumns, tag) tagColumnMap[tag] = true diff --git a/control/controlexecute/result_group.go b/control/controlexecute/result_group.go index 8e9831757..840506a6c 100644 --- a/control/controlexecute/result_group.go +++ b/control/controlexecute/result_group.go @@ -72,6 +72,12 @@ func NewRootResultGroup(executionTree *ExecutionTree, rootItems ...modconfig.Mod // NewResultGroup creates a result group from a ModTreeItem func NewResultGroup(executionTree *ExecutionTree, treeItem modconfig.ModTreeItem, parent *ResultGroup) *ResultGroup { + // only show qualified group names for controls from dependent mods + groupId := treeItem.Name() + if mod := treeItem.GetMod(); mod != nil && mod.Name() == executionTree.workspace.Mod.Name() { + groupId = modconfig.UnqualifiedResourceName(groupId) + } + group := &ResultGroup{ GroupId: treeItem.Name(), Title: treeItem.GetTitle(), diff --git a/control/controlexecute/result_row.go b/control/controlexecute/result_row.go index 8dbced854..ab3135591 100644 --- a/control/controlexecute/result_row.go +++ b/control/controlexecute/result_row.go @@ -19,7 +19,7 @@ type ResultRow struct { Resource string `json:"resource" csv:"resource"` Status string `json:"status" csv:"status"` Dimensions []Dimension `json:"dimensions"` - Control *modconfig.Control `json:"-" csv:"control_id:FullName,control_title:Title,control_description:Description"` + Control *modconfig.Control `json:"-" csv:"control_id:UnqualifiedName,control_title:Title,control_description:Description"` } // AddDimension checks whether a column value is a scalar type, and if so adds it to the Dimensions map diff --git a/db/db_client/db_client_session.go b/db/db_client/db_client_session.go index 05f0e1b1a..fc19cbd7a 100644 --- a/db/db_client/db_client_session.go +++ b/db/db_client/db_client_session.go @@ -8,6 +8,7 @@ import ( "time" "github.com/jackc/pgx/v4/stdlib" + "github.com/sethvargo/go-retry" "github.com/turbot/steampipe/db/db_common" "github.com/turbot/steampipe/utils" @@ -144,5 +145,5 @@ func (c *DbClient) getSessionWithRetries(ctx context.Context) (*sql.Conn, uint32 return nil }) - return session, backendPid, nil + return session, uint32(backendPid), nil } diff --git a/go.mod b/go.mod index ed6ab21b3..bb8ce3271 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.17 require ( github.com/Machiel/slugify v1.0.1 + github.com/Masterminds/semver v1.5.0 github.com/ahmetb/go-linq v3.0.0+incompatible github.com/alecthomas/chroma v0.9.2 github.com/bgentry/speakeasy v0.1.0 @@ -45,6 +46,7 @@ require ( github.com/turbot/go-kit v0.3.0 github.com/turbot/steampipe-plugin-sdk v1.8.0 github.com/ulikunitz/xz v0.5.8 + github.com/xlab/treeprint v1.1.0 github.com/zclconf/go-cty v1.8.2 github.com/zclconf/go-cty-yaml v1.0.2 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c diff --git a/go.sum b/go.sum index 7440c1d59..a5a9017ee 100644 --- a/go.sum +++ b/go.sum @@ -81,7 +81,9 @@ github.com/ChrisTrenkamp/goxpath v0.0.0-20190607011252-c5096ec8773d/go.mod h1:nu github.com/Machiel/slugify v1.0.1 h1:EfWSlRWstMadsgzmiV7d0yVd2IFlagWH68Q+DcYCm4E= github.com/Machiel/slugify v1.0.1/go.mod h1:fTFGn5uWEynW4CUMG7sWkYXOf1UgDxyTM3DbR6Qfg3k= github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= @@ -729,7 +731,6 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -1015,6 +1016,8 @@ github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20161029104018-1d6e34225557/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk= +github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/interactive/interactive_client.go b/interactive/interactive_client.go index 6cbdf829b..03c283f35 100644 --- a/interactive/interactive_client.go +++ b/interactive/interactive_client.go @@ -115,7 +115,7 @@ func (c *InteractiveClient) InteractivePrompt() { c.resultsStreamer.Close() }() - fmt.Printf("Welcome to Steampipe v%s\n", version.String()) + fmt.Printf("Welcome to Steampipe v%s\n", version.SteampipeVersion.String()) fmt.Printf("For more information, type %s\n", constants.Bold(".help")) // run the prompt in a goroutine, so we can also detect async initialisation errors diff --git a/mod_installer/git.go b/mod_installer/git.go new file mode 100644 index 000000000..4ba4f170d --- /dev/null +++ b/mod_installer/git.go @@ -0,0 +1,68 @@ +package mod_installer + +import ( + "fmt" + "sort" + + "github.com/Masterminds/semver" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/storage/memory" +) + +func getGitUrl(modName string) string { + return fmt.Sprintf("https://%s", modName) +} + +func getTags(repo string) ([]string, error) { + // Create the remote with repository URL + rem := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{ + Name: "origin", + URLs: []string{repo}, + }) + + // load remote references + refs, err := rem.List(&git.ListOptions{}) + if err != nil { + return nil, err + } + + // filters the references list and only keeps tags + var tags []string + for _, ref := range refs { + if ref.Name().IsTag() { + tags = append(tags, ref.Name().Short()) + } + } + + return tags, nil +} + +func getTagVersionsFromGit(repo string, includePrerelease bool) (semver.Collection, error) { + tags, err := getTags(repo) + if err != nil { + return nil, err + } + + versions := make(semver.Collection, len(tags)) + // handle index manually as we may not add all tags - if we cannot parse them as a version + idx := 0 + for _, raw := range tags { + v, err := semver.NewVersion(raw) + if err != nil { + continue + } + + if !includePrerelease && v.Metadata() != "" || v.Prerelease() != "" { + continue + } + versions[idx] = v + idx++ + } + // shrink slice + versions = versions[:idx] + + // sort the versions in REVERSE order + sort.Sort(sort.Reverse(versions)) + return versions, nil +} diff --git a/mod_installer/helpers.go b/mod_installer/helpers.go new file mode 100644 index 000000000..e0d5c6cd9 --- /dev/null +++ b/mod_installer/helpers.go @@ -0,0 +1,16 @@ +package mod_installer + +import ( + "github.com/Masterminds/semver" + "github.com/turbot/steampipe/version_helpers" +) + +func getVersionSatisfyingConstraint(constraint *version_helpers.Constraints, availableVersions []*semver.Version) *semver.Version { + // search the reverse sorted versions, finding the highest version which satisfies ALL constraints + for _, version := range availableVersions { + if constraint.Check(version) { + return version + } + } + return nil +} diff --git a/mod_installer/install.go b/mod_installer/install.go new file mode 100644 index 000000000..b7328815e --- /dev/null +++ b/mod_installer/install.go @@ -0,0 +1,28 @@ +package mod_installer + +import ( + "github.com/turbot/go-kit/helpers" + "github.com/turbot/steampipe/utils" +) + +func InstallWorkspaceDependencies(opts *InstallOpts) (_ *InstallData, err error) { + utils.LogTime("cmd.InstallWorkspaceDependencies") + defer func() { + utils.LogTime("cmd.InstallWorkspaceDependencies end") + if r := recover(); r != nil { + err = helpers.ToError(r) + } + }() + + // install workspace dependencies + installer, err := NewModInstaller(opts) + if err != nil { + return nil, err + } + + if err := installer.InstallWorkspaceDependencies(); err != nil { + return nil, err + } + + return installer.installData, nil +} diff --git a/mod_installer/install_data.go b/mod_installer/install_data.go new file mode 100644 index 000000000..ca0e9a4bc --- /dev/null +++ b/mod_installer/install_data.go @@ -0,0 +1,119 @@ +package mod_installer + +import ( + "fmt" + + "github.com/Masterminds/semver" + "github.com/turbot/steampipe/steampipeconfig/modconfig" + "github.com/turbot/steampipe/steampipeconfig/version_map" + "github.com/turbot/steampipe/version_helpers" + "github.com/xlab/treeprint" +) + +type InstallData struct { + // record of the full dependency tree + Lock *version_map.WorkspaceLock + NewLock *version_map.WorkspaceLock + + // ALL the available versions for each dependency mod(we populate this in a lazy fashion) + allAvailable version_map.VersionListMap + + // list of dependencies installed by recent install operation + Installed version_map.DependencyVersionMap + // list of dependencies which have been upgraded + Upgraded version_map.DependencyVersionMap + // list of dependencies which have been downgraded + Downgraded version_map.DependencyVersionMap + // list of dependencies which have been uninstalled + Uninstalled version_map.DependencyVersionMap + WorkspaceMod *modconfig.Mod +} + +func NewInstallData(workspaceLock *version_map.WorkspaceLock, workspaceMod *modconfig.Mod) *InstallData { + return &InstallData{ + Lock: workspaceLock, + WorkspaceMod: workspaceMod, + NewLock: version_map.EmptyWorkspaceLock(workspaceLock), + allAvailable: make(version_map.VersionListMap), + Installed: make(version_map.DependencyVersionMap), + Upgraded: make(version_map.DependencyVersionMap), + Downgraded: make(version_map.DependencyVersionMap), + Uninstalled: make(version_map.DependencyVersionMap), + } +} + +// GetAvailableUpdates returns a map of all installed mods which are not in the lock file +func (d *InstallData) GetAvailableUpdates() (version_map.DependencyVersionMap, error) { + res := make(version_map.DependencyVersionMap) + for parent, deps := range d.Lock.InstallCache { + for name, resolvedConstraint := range deps { + includePrerelease := resolvedConstraint.IsPrerelease() + availableVersions, err := d.getAvailableModVersions(name, includePrerelease) + if err != nil { + return nil, err + } + constraint, _ := version_helpers.NewConstraint(resolvedConstraint.Constraint) + var latestVersion = getVersionSatisfyingConstraint(constraint, availableVersions) + if latestVersion.GreaterThan(resolvedConstraint.Version) { + res.Add(name, latestVersion, constraint.Original, parent) + } + } + } + return res, nil +} + +// onModInstalled is called when a dependency is satisfied by installing a mod version +func (d *InstallData) onModInstalled(dependency *ResolvedModRef, parent *modconfig.Mod) { + parentPath := parent.GetModDependencyPath() + // get the constraint from the parent (it must be there) + modVersion := parent.Require.GetModDependency(dependency.Name) + // update lock + d.NewLock.InstallCache.Add(dependency.Name, dependency.Version, modVersion.Constraint.Original, parentPath) +} + +// addExisting is called when a dependency is satisfied by a mod which is already installed +func (d *InstallData) addExisting(name string, version *semver.Version, constraint *version_helpers.Constraints, parent *modconfig.Mod) { + // update lock + parentPath := parent.GetModDependencyPath() + d.NewLock.InstallCache.Add(name, version, constraint.Original, parentPath) +} + +// retrieve all available mod versions from our cache, or from Git if not yet cached +func (d *InstallData) getAvailableModVersions(modName string, includePrerelease bool) ([]*semver.Version, error) { + // have we already loaded the versions for this mod + availableVersions, ok := d.allAvailable[modName] + if ok { + return availableVersions, nil + } + // so we have not cached this yet - retrieve from Git + var err error + availableVersions, err = getTagVersionsFromGit(getGitUrl(modName), includePrerelease) + if err != nil { + return nil, fmt.Errorf("could not retrieve version data from Git URL '%s'", modName) + } + // update our cache + d.allAvailable[modName] = availableVersions + + return availableVersions, nil +} + +// update the lock with the NewLock and dtermine if any mods have been uninstalled +func (d *InstallData) onInstallComplete() { + d.Installed = d.NewLock.InstallCache.GetMissingFromOther(d.Lock.InstallCache) + d.Uninstalled = d.Lock.InstallCache.GetMissingFromOther(d.NewLock.InstallCache) + d.Upgraded = d.Lock.InstallCache.GetUpgradedInOther(d.NewLock.InstallCache) + d.Downgraded = d.Lock.InstallCache.GetDowngradedInOther(d.NewLock.InstallCache) + d.Lock = d.NewLock +} + +func (d *InstallData) GetUpdatedTree() treeprint.Tree { + return d.Upgraded.GetDependencyTree(d.WorkspaceMod.GetModDependencyPath()) +} + +func (d *InstallData) GetInstalledTree() treeprint.Tree { + return d.Installed.GetDependencyTree(d.WorkspaceMod.GetModDependencyPath()) +} + +func (d *InstallData) GetUninstalledTree() treeprint.Tree { + return d.Uninstalled.GetDependencyTree(d.WorkspaceMod.GetModDependencyPath()) +} diff --git a/mod_installer/install_opts.go b/mod_installer/install_opts.go new file mode 100644 index 000000000..e1fb12b9e --- /dev/null +++ b/mod_installer/install_opts.go @@ -0,0 +1,8 @@ +package mod_installer + +type InstallOpts struct { + WorkspacePath string + Command string + DryRun bool + ModArgs []string +} diff --git a/mod_installer/mod_installer.go b/mod_installer/mod_installer.go index e652d41f7..1b7ea1dc5 100644 --- a/mod_installer/mod_installer.go +++ b/mod_installer/mod_installer.go @@ -5,188 +5,404 @@ import ( "log" "os" "path/filepath" - "strings" - - "github.com/turbot/steampipe/constants" + "github.com/Masterminds/semver" git "github.com/go-git/go-git/v5" + "github.com/otiai10/copy" + "github.com/spf13/viper" + "github.com/turbot/steampipe/constants" "github.com/turbot/steampipe/steampipeconfig/modconfig" "github.com/turbot/steampipe/steampipeconfig/parse" + "github.com/turbot/steampipe/steampipeconfig/version_map" "github.com/turbot/steampipe/utils" ) -/* -mog get - -A user may install a mod with steampipe mod get modname[@version] - -version may be: - -- Not specified: steampipe mod get aws-cis - The latest version (highest version tag) will be installed. - A dependency is added to the requires block specifying the version that was downloaded -- A major version: steampipe mod get aws-cis@3 - The latest release (highest version tag) of the specified major version will be installed. - A dependency is added to the requires block specifying the version that was downloaded -- A monotonic version tag: steampipe mod get aws-cis@v2.21 - The specified version is downloaded and added as requires dependency. -- A branch name: steampipe mod get aws-cis@staging - The current version of the specified branch is downloaded. - The branch dependency is added to the requires list. Note that a branch is considered a distinct "major" release, it is not cached in the registry, and has no minor version. - Branch versions do not auto-update - you have to run steampipe mod update to get a newer version. - Branch versioning is meant to simplify development and testing - published mods should ONLY include version tag dependencies, NOT branch dependencies. -- A local file path: steampipe mod get "file:~/my_path/aws-core" - The mod from the local filesystem is added to the namespace, but nothing is downloaded. - The local dependency is added to the requires list. Note that a local mod is considered a distinct "major" release, it is not cached in the registry, and has no minor version. - Local versioning is meant to simplify development and testing - published mods should ONLY include version tag dependencies, NOT local dependencies. - - -Steampipe Version Dependency -If the installed version of Steampipe does not meet the dependency criteria, the user will be warned and the mod will NOT be installed. - -Plugin Dependency5 -If the mod specifies plugin versions that are not installed, or have no connections, the user will be warned but the mod will be installed. The user should be warned at installation time, and also when starting Steampipe in the workspace. - - -Detecting conflicts -mod 1 require a@1.0 -mod 2 require a@file:/foo - --> how do we detect if the file version satisfied constrainst of a - this is for dev purposes so always pass? - -mod 1 require a@1.0 -mod 2 require a@ - --> how do we detect if the file version satisfied constraints of a - check branch? - -*/ - type ModInstaller struct { - ModsDir string - InstalledDependencies []*ResolvedModRef + workspaceMod *modconfig.Mod + modsPath string + // temp location used to install dependencies + tmpPath string + workspacePath string + + installData *InstallData + + // what command is being run + command string + // are dependencies being added to the workspace + mods version_map.VersionConstraintMap + dryRun bool } -func NewModInstaller(workspacePath string) *ModInstaller { - return &ModInstaller{ - ModsDir: constants.WorkspaceModPath(workspacePath), +func NewModInstaller(opts *InstallOpts) (*ModInstaller, error) { + i := &ModInstaller{ + workspacePath: opts.WorkspacePath, + command: opts.Command, + dryRun: opts.DryRun, } -} - -// InstallModDependencies installs all dependencies of the mod -func (i *ModInstaller) InstallModDependencies(mod *modconfig.Mod) error { - dependencyMap := make(map[string]*ResolvedModRef) - return i.installModDependenciesRecursively(mod, dependencyMap) -} - -func (i *ModInstaller) installModDependenciesRecursively(mod *modconfig.Mod, dependencyMap map[string]*ResolvedModRef) error { - if mod.Requires == nil { - return nil + if err := i.setModsPath(); err != nil { + return nil, err } - // first check our Steampipe version is sufficient - if err := mod.Requires.ValidateSteampipeVersion(mod.Name()); err != nil { + // load workspace mod, creating a default if needed + workspaceMod, err := i.loadModfile(i.workspacePath, true) + if err != nil { + return nil, err + } + i.workspaceMod = workspaceMod + + // load lock file + workspaceLock, err := version_map.LoadWorkspaceLock(i.workspacePath) + if err != nil { + return nil, err + } + + // create install data + i.installData = NewInstallData(workspaceLock, workspaceMod) + + // parse args to get the required mod versions + requiredMods, err := i.GetRequiredModVersionsFromArgs(opts.ModArgs) + if err != nil { + return nil, err + } + i.mods = requiredMods + + return i, nil +} + +func (i *ModInstaller) setModsPath() error { + dir, err := os.MkdirTemp(os.TempDir(), "sp_dr_*") + if err != nil { + return err + } + i.tmpPath = dir + i.modsPath = constants.WorkspaceModPath(i.workspacePath) + return nil +} + +func (i *ModInstaller) UninstallWorkspaceDependencies() error { + workspaceMod := i.workspaceMod + + // remove required dependencies from the mod file + if len(i.mods) == 0 { + workspaceMod.RemoveAllModDependencies() + + } else { + // verify all the mods specifed in the args exist in the modfile + workspaceMod.RemoveModDependencies(i.mods) + } + + // uninstall by calling Install + if err := i.installMods(workspaceMod.Require.Mods, workspaceMod); err != nil { return err } + if workspaceMod.Require.Empty() { + workspaceMod.Require = nil + } + + // if this is a dry run, return now + if i.dryRun { + + log.Printf("[TRACE] UninstallWorkspaceDependencies - dry-run=true, returning before saving mod file and cache\n") + return nil + } + + // write the lock file + if err := i.installData.Lock.Save(); err != nil { + return err + } + + // now safe to save the mod file + if err := i.workspaceMod.Save(); err != nil { + return err + } + + // tidy unused mods + if viper.GetBool(constants.ArgPrune) { + if _, err := i.Prune(); err != nil { + return err + } + } + return nil +} + +// InstallWorkspaceDependencies installs all dependencies of the workspace mod +func (i *ModInstaller) InstallWorkspaceDependencies() (err error) { + workspaceMod := i.workspaceMod + defer func() { + // tidy unused mods + // (put in defer so it still gets called in case of errors) + if viper.GetBool(constants.ArgPrune) { + // be sure not to overwrite an existing return error + _, pruneErr := i.Prune() + if pruneErr != nil && err == nil { + err = pruneErr + } + } + }() + + // first check our Steampipe version is sufficient + if err := workspaceMod.Require.ValidateSteampipeVersion(workspaceMod.Name()); err != nil { + return err + } + + // if mod args have been provided, add them to the the workspace mod requires + // (this will replace any existing dependencies of same name) + if len(i.mods) > 0 { + workspaceMod.AddModDependencies(i.mods) + } + + if err := i.installMods(workspaceMod.Require.Mods, workspaceMod); err != nil { + return err + } + + // if this is a dry run, return now + if i.dryRun { + log.Printf("[TRACE] InstallWorkspaceDependencies - dry-run=true, returning before saving mod file and cache\n") + return nil + } + + // write the lock file + if err := i.installData.Lock.Save(); err != nil { + return err + } + + // now safe to save the mod file + if len(i.mods) > 0 { + if err := i.workspaceMod.Save(); err != nil { + return err + } + } + + if !workspaceMod.HasDependentMods() { + // there are no dependencies - delete the cache + i.installData.Lock.Delete() + } + return nil +} + +func (i *ModInstaller) GetModList() string { + return i.installData.Lock.GetModList(i.workspaceMod.GetModDependencyPath()) +} + +func (i *ModInstaller) installMods(mods []*modconfig.ModVersionConstraint, parent *modconfig.Mod) error { + // clean up the temp location + defer os.RemoveAll(i.tmpPath) + var errors []error - for _, modVersion := range mod.Requires.Mods { - // get a resolved mod ref for this mod version - resolvedRef, err := i.GetModRefForVersion(modVersion) + for _, requiredModVersion := range mods { + modToUse, err := i.getCurrentlyInstalledVersionToUse(requiredModVersion, parent, i.updating()) if err != nil { - return fmt.Errorf("dependency %s %s cannot be satisfied: %s", mod.Name(), modVersion.VersionString, err.Error()) + errors = append(errors, err) + continue } - // install this mod - // NOTE - this mutates dependency map - if err := i.installDependency(resolvedRef, dependencyMap); err != nil { + // if the mod is not installed or needs updating, pass shouldUpdate=true into installModDependencesRecursively + // this ensures that we update any dependencies which have updates available + shouldUpdate := modToUse == nil + if err := i.installModDependencesRecursively(requiredModVersion, modToUse, parent, shouldUpdate); err != nil { errors = append(errors, err) } } - return utils.CombineErrorsWithPrefix(fmt.Sprintf("%d dependencies failed to install", len(errors)), errors...) + // update the lock to be the new lock, and record any uninstalled mods + i.installData.onInstallComplete() + + return i.buildInstallError(errors) } -func (i *ModInstaller) GetModRefForVersion(modVersion *modconfig.ModVersion) (*ResolvedModRef, error) { - - // NOTE check whether the lock file contains this dependency and if so - // does the locked version satisy this version requirement - // return error if not - - // NOTE check whether we are replacing this version - // if so does the locked version satisfy this version requirement - // return error if not - - // so we need to resolve this mod version - - // NOTE for now assume github - // get the most recent minor version fo rthis major version from github - return i.getLatestCompatibleVersionFromGithub(modVersion) -} - -func (i *ModInstaller) getLatestCompatibleVersionFromGithub(modVersion *modconfig.ModVersion) (*ResolvedModRef, error) { - // NOTE for now assume the mod is specified with a full version - return NewResolvedModRef(modVersion) -} - -func (i *ModInstaller) installDependency(dependency *ResolvedModRef, dependencyMap map[string]*ResolvedModRef) error { - // have we already installed a mod which satisfies this dependency - if modRef, ok := dependencyMap[dependency.Name]; ok { - if modRef.Version.GreaterThanOrEqual(dependency.Version) { - return nil - } - } - - // add this dependency into the map (if we fail to install,m the whole installation process will terminate, - // so no need to check for errors - dependencyMap[dependency.Name] = dependency - - var modPath string - if dependency.FilePath != "" { - // if there is a file path, verify it exists - if _, err := os.Stat(dependency.FilePath); os.IsNotExist(err) { - return fmt.Errorf("dependency %s file path %s does not exist", dependency.Name, dependency.FilePath) - } - modPath = dependency.FilePath - } else { - modPath = filepath.Join(i.ModsDir, dependency.FullName()) - if err := i.installDependencyFromGit(dependency, modPath); err != nil { - return err - } - } - // no load the installed mod and install _its_ dependencies - if !parse.ModfileExists(modPath) { - log.Printf("[TRACE] dependency %s does not define a mod defintion - so there are no dependencies to install", dependency.Name) +func (i *ModInstaller) buildInstallError(errors []error) error { + if len(errors) == 0 { return nil } - - mod, err := parse.ParseModDefinition(modPath) - if err != nil { - return err - } - err = i.installModDependenciesRecursively(mod, dependencyMap) - // if we succeeded, update our list - if err == nil { - i.InstalledDependencies = append(i.InstalledDependencies, dependency) + verb := "install" + if i.updating() { + verb = "update" } + prefix := fmt.Sprintf("%d %s failed to %s", len(errors), utils.Pluralize("dependency", len(errors)), verb) + err := utils.CombineErrorsWithPrefix(prefix, errors...) return err } -func (i *ModInstaller) installDependencyFromGit(dependency *ResolvedModRef, installPath string) error { - // ensure mod directory exists - create if necessary - if err := os.MkdirAll(i.ModsDir, os.ModePerm); err != nil { +func (i *ModInstaller) installModDependencesRecursively(requiredModVersion *modconfig.ModVersionConstraint, dependencyMod *modconfig.Mod, parent *modconfig.Mod, shouldUpdate bool) error { + // get available versions for this mod + includePrerelease := requiredModVersion.Constraint.IsPrerelease() + availableVersions, err := i.installData.getAvailableModVersions(requiredModVersion.Name, includePrerelease) + + if err != nil { return err } - // NOTE: we need to check existing installed mods + if dependencyMod == nil { + // so we ARE installing + // get a resolved mod ref that satisfies the version constraints + resolvedRef, err := i.getModRefSatisfyingConstraints(requiredModVersion, availableVersions) + if err != nil { + return err + } + + // install the mod + dependencyMod, err = i.install(resolvedRef, parent) + if err != nil { + return err + } + } else { + // so we found an existing mod which will satisfy this requirement + + // update the install data + i.installData.addExisting(requiredModVersion.Name, dependencyMod.Version, requiredModVersion.Constraint, parent) + log.Printf("[TRACE] not installing %s with version constraint %s as version %s is already installed", requiredModVersion.Name, requiredModVersion.Constraint.Original, dependencyMod.Version) + } + + // to get here we have the dependency mod - either we installed it or it was already installed + // recursively install its dependencies + var errors []error + // now update the parent to dependency mod and install its child dependencies + parent = dependencyMod + for _, dep := range dependencyMod.Require.Mods { + childDependencyMod, err := i.getCurrentlyInstalledVersionToUse(dep, parent, shouldUpdate) + if err != nil { + errors = append(errors, err) + continue + } + if err := i.installModDependencesRecursively(dep, childDependencyMod, parent, shouldUpdate); err != nil { + errors = append(errors, err) + continue + } + } + + return utils.CombineErrorsWithPrefix(fmt.Sprintf("%d child %s failed to install", len(errors), utils.Pluralize("dependency", len(errors))), errors...) +} + +func (i *ModInstaller) getCurrentlyInstalledVersionToUse(requiredModVersion *modconfig.ModVersionConstraint, parent *modconfig.Mod, forceUpdate bool) (*modconfig.Mod, error) { + // do we have an installed version of this mod matching the required mod constraint + installedVersion, err := i.installData.Lock.GetLockedModVersion(requiredModVersion, parent) + if err != nil { + return nil, err + } + if installedVersion == nil { + return nil, nil + } + + // can we update this + canUpdate, err := i.canUpdateMod(installedVersion, requiredModVersion, forceUpdate) + if err != nil { + return nil, err + + } + if canUpdate { + // return nil mod to indicate we should update + return nil, nil + } + + // load the existing mod and return + return i.loadDependencyMod(installedVersion) +} + +// determine if we should update this mod, and if so whether there is an update available +func (i *ModInstaller) canUpdateMod(installedVersion *version_map.ResolvedVersionConstraint, requiredModVersion *modconfig.ModVersionConstraint, forceUpdate bool) (bool, error) { + // so should we update? + // if forceUpdate is set or if the required version constraint is different to the locked version constraint, update + // TODO check * vs latest - maybe need a custom equals? + if forceUpdate || installedVersion.Constraint != requiredModVersion.Constraint.Original { + // get available versions for this mod + includePrerelease := requiredModVersion.Constraint.IsPrerelease() + availableVersions, err := i.installData.getAvailableModVersions(requiredModVersion.Name, includePrerelease) + if err != nil { + return false, err + } + + return i.updateAvailable(requiredModVersion, installedVersion.Version, availableVersions) + } + return false, nil + +} + +// determine whether there is a newer mod version avoilable which satisfies the dependency version constraint +func (i *ModInstaller) updateAvailable(requiredVersion *modconfig.ModVersionConstraint, currentVersion *semver.Version, availableVersions []*semver.Version) (bool, error) { + latestVersion, err := i.getModRefSatisfyingConstraints(requiredVersion, availableVersions) + if err != nil { + return false, err + } + if latestVersion.Version.GreaterThan(currentVersion) { + return true, nil + } + return false, nil +} + +// get the most recent available mod version which satisfies the version constraint +func (i *ModInstaller) getModRefSatisfyingConstraints(modVersion *modconfig.ModVersionConstraint, availableVersions []*semver.Version) (*ResolvedModRef, error) { + // find a version which satisfies the version constraint + var version = getVersionSatisfyingConstraint(modVersion.Constraint, availableVersions) + if version == nil { + return nil, fmt.Errorf("no version of %s found satisfying verison constraint: %s", modVersion.Name, modVersion.Constraint.Original) + } + + return NewResolvedModRef(modVersion, version) +} + +// install a mod +func (i *ModInstaller) install(dependency *ResolvedModRef, parent *modconfig.Mod) (_ *modconfig.Mod, err error) { + // get the temp location to install the mod to + fullName := dependency.FullName() + tempDestPath := i.getDependencyTmpPath(fullName) + + defer func() { + if err == nil { + i.installData.onModInstalled(dependency, parent) + } + }() + // if the target path exists, use the exiting file + // if it does not exist (the usual case), install it + if _, err := os.Stat(tempDestPath); os.IsNotExist(err) { + if err := i.installFromGit(dependency, tempDestPath); err != nil { + return nil, err + } + } + + // now load the installed mod and return it + modDef, err := i.loadModfile(tempDestPath, false) + if err != nil { + return nil, err + } + if modDef == nil { + return nil, fmt.Errorf("'%s' has no mod definition file", dependency.FullName()) + } + // hack set mod dependency path + if err := i.setModDependencyPath(modDef, i.tmpPath); err != nil { + return nil, err + } + + // so we have successfully installed this dependency to the temp location, now copy to the mod location + if !i.dryRun { + destPath := i.getDependencyDestPath(fullName) + if err := i.copyModFromTempToModsFolder(tempDestPath, destPath); err != nil { + return nil, err + } + } + + return modDef, nil +} + +func (i *ModInstaller) copyModFromTempToModsFolder(tmpPath string, destPath string) error { + if err := os.RemoveAll(destPath); err != nil { + return err + } + + if err := copy.Copy(tmpPath, destPath); err != nil { + return err + } + return nil +} + +func (i *ModInstaller) installFromGit(dependency *ResolvedModRef, installPath string) error { // get the mod from git - - gitUrl := fmt.Sprintf("https://%s", dependency.Name) + gitUrl := getGitUrl(dependency.Name) _, err := git.PlainClone(installPath, false, &git.CloneOptions{ - URL: gitUrl, - //Progress: os.Stdout, + URL: gitUrl, ReferenceName: dependency.GitReference, Depth: 1, SingleBranch: true, @@ -195,14 +411,54 @@ func (i *ModInstaller) installDependencyFromGit(dependency *ResolvedModRef, inst return err } -func (i *ModInstaller) InstallReport() string { - if len(i.InstalledDependencies) == 0 { - return "No dependencies installed" +// build the path of the temp location to copy this depednency to +func (i *ModInstaller) getDependencyTmpPath(dependencyFullName string) string { + return filepath.Join(i.tmpPath, dependencyFullName) +} + +// build the path of the temp location to copy this depednency to +func (i *ModInstaller) getDependencyDestPath(dependencyFullName string) string { + return filepath.Join(i.modsPath, dependencyFullName) +} + +func (i *ModInstaller) loadDependencyMod(modVersion *version_map.ResolvedVersionConstraint) (*modconfig.Mod, error) { + modPath := i.getDependencyDestPath(modconfig.ModVersionFullName(modVersion.Name, modVersion.Version)) + modDef, err := i.loadModfile(modPath, false) + if err != nil { + return nil, err } - strs := make([]string, len(i.InstalledDependencies)) - for idx, dep := range i.InstalledDependencies { - strs[idx] = dep.FullName() + if err := i.setModDependencyPath(modDef, modPath); err != nil { + return nil, err } - return fmt.Sprintf("\nInstalled %d dependencies:\n - %s\n", len(i.InstalledDependencies), strings.Join(strs, "\n - ")) + return modDef, nil } + +// HACK set the mod depdnency path +func (i *ModInstaller) setModDependencyPath(mod *modconfig.Mod, modPath string) (err error) { + mod.ModDependencyPath, err = filepath.Rel(i.modsPath, modPath) + return +} + +func (i *ModInstaller) loadModfile(modPath string, createDefault bool) (*modconfig.Mod, error) { + if !parse.ModfileExists(modPath) { + if createDefault { + return modconfig.CreateDefaultMod(i.workspacePath), nil + } + return nil, nil + } + mod, err := parse.ParseModDefinition(modPath) + if err != nil { + return nil, err + } + + return mod, nil +} + +func (i *ModInstaller) updating() bool { + return i.command == "update" +} + +func (i *ModInstaller) uninstalling() bool { + return i.command == "uninstall" +} diff --git a/mod_installer/mod_installer_args.go b/mod_installer/mod_installer_args.go new file mode 100644 index 000000000..9d1a264e8 --- /dev/null +++ b/mod_installer/mod_installer_args.go @@ -0,0 +1,63 @@ +package mod_installer + +import ( + "fmt" + + "github.com/turbot/steampipe/steampipeconfig/modconfig" + "github.com/turbot/steampipe/steampipeconfig/version_map" + "github.com/turbot/steampipe/utils" +) + +func (i *ModInstaller) GetRequiredModVersionsFromArgs(modsArgs []string) (version_map.VersionConstraintMap, error) { + var errors []error + mods := make(version_map.VersionConstraintMap, len(modsArgs)) + for _, modArg := range modsArgs { + // create mod version from arg + modVersion, err := modconfig.NewModVersionConstraint(modArg) + if err != nil { + errors = append(errors, err) + continue + } + // if we are updating there are a few checks we need to make + if i.updating() { + modVersion, err = i.getUpdateVersion(modArg, modVersion) + if err != nil { + errors = append(errors, err) + continue + } + } + if i.uninstalling() { + // it is not valid to specify a mod version for uninstall + if modVersion.HasVersion() { + errors = append(errors, fmt.Errorf("invalid arg '%s' - cannot specify a version when uninstalling", modArg)) + continue + } + } + + mods[modVersion.Name] = modVersion + } + if len(errors) > 0 { + return nil, utils.CombineErrors(errors...) + } + return mods, nil +} + +func (i *ModInstaller) getUpdateVersion(modArg string, modVersion *modconfig.ModVersionConstraint) (*modconfig.ModVersionConstraint, error) { + // verify the mod is already installed + if i.installData.Lock.GetMod(modVersion.Name, i.workspaceMod) == nil { + return nil, fmt.Errorf("cannot update '%s' as it is not installed", modArg) + } + + // find the current dependency with this mod name + // - this is what we will be using, to ensure we keep the same version constraint + currentDependency := i.workspaceMod.GetModDependency(modVersion.Name) + if currentDependency == nil { + return nil, fmt.Errorf("cannot update '%s' as it is not a dependency of this workspace", modArg) + } + + // it is not valid to specify a mod version - we will set the constraint from the modfile + if modVersion.HasVersion() { + return nil, fmt.Errorf("invalid arg '%s' - cannot specify a version when updating", modArg) + } + return currentDependency, nil +} diff --git a/mod_installer/mod_installer_prune.go b/mod_installer/mod_installer_prune.go new file mode 100644 index 000000000..c752e896e --- /dev/null +++ b/mod_installer/mod_installer_prune.go @@ -0,0 +1,42 @@ +package mod_installer + +import ( + "os" + "path/filepath" + + "github.com/turbot/steampipe/steampipeconfig/modconfig" + "github.com/turbot/steampipe/steampipeconfig/version_map" +) + +func (i *ModInstaller) Prune() (version_map.VersionListMap, error) { + unusedMods := i.installData.Lock.GetUnreferencedMods() + // now delete any mod folders which are not in the lock file + for name, versions := range unusedMods { + for _, version := range versions { + depPath := i.getDependencyDestPath(modconfig.ModVersionFullName(name, version)) + if err := i.deleteDependencyItem(depPath); err != nil { + return nil, err + } + } + } + + return unusedMods, nil +} + +func (i *ModInstaller) deleteDependencyItem(depPath string) error { + if err := os.RemoveAll(depPath); err != nil { + return err + } + return i.deleteEmptyFolderTree(filepath.Dir(depPath)) + +} + +func (i *ModInstaller) deleteEmptyFolderTree(folderPath string) error { + // if the parent folder is empty, delete it + err := os.Remove(folderPath) + if err == nil { + parent := filepath.Dir(folderPath) + return i.deleteEmptyFolderTree(parent) + } + return nil +} diff --git a/mod_installer/mod_installer_test.go b/mod_installer/mod_installer_test.go new file mode 100644 index 000000000..8fb700dde --- /dev/null +++ b/mod_installer/mod_installer_test.go @@ -0,0 +1,18 @@ +package mod_installer + +import ( + "fmt" + "testing" + + "github.com/Masterminds/semver" +) + +func TestModInstaller(t *testing.T) { + cs, err := semver.NewConstraint("^3") + v, _ := semver.NewVersion("3.1") + res := cs.Check(v) + fmt.Println(res) + + fmt.Println(cs) + fmt.Println(err) +} diff --git a/mod_installer/mod_ref.go b/mod_installer/mod_ref.go deleted file mode 100644 index ebc5a674b..000000000 --- a/mod_installer/mod_ref.go +++ /dev/null @@ -1,53 +0,0 @@ -package mod_installer - -import ( - "fmt" - "strings" - - goVersion "github.com/hashicorp/go-version" -) - -// ModRef is a struct to represent an unresolved mod reference -type ModRef struct { - // the Git URL of the mod repo - Name string - // the version constraint of the mod - versionConstraint *goVersion.Version - // the branch to use - branch string - // the local file location to use - filePath string - // raw reference - raw string -} - -func NewModRef(modRef string) (*ModRef, error) { - split := strings.Split(modRef, "@") - if len(split) > 2 { - return nil, fmt.Errorf("invalid mod ref %s", modRef) - } - res := &ModRef{ - raw: modRef, - Name: split[0], - } - if len(split) == 2 { - res.setVersion(split[1]) - } - - return res, nil -} - -func (r *ModRef) setVersion(versionString string) { - if strings.HasPrefix(versionString, "file:") { - r.filePath = versionString - return - } - // does the verison parse as a semver version - if v, err := goVersion.NewVersion(versionString); err == nil { - r.versionConstraint = v - return - } - - // otherwise assume it is a branch - r.branch = versionString -} diff --git a/mod_installer/resolved_mod_ref.go b/mod_installer/resolved_mod_ref.go index 781686936..d18d48042 100644 --- a/mod_installer/resolved_mod_ref.go +++ b/mod_installer/resolved_mod_ref.go @@ -1,73 +1,49 @@ package mod_installer import ( - "fmt" - + "github.com/Masterminds/semver" "github.com/go-git/go-git/v5/plumbing" - goVersion "github.com/hashicorp/go-version" "github.com/turbot/steampipe/steampipeconfig/modconfig" + "github.com/turbot/steampipe/version_helpers" ) -// ResolvedModRef is a struct to represent a resolved mod reference +// ResolvedModRef is a struct to represent a resolved mod git reference type ResolvedModRef struct { // the FQN of the mod - also the Git URL of the mod repo Name string + // the mod version + Version *semver.Version + // the vestion constraint + Constraint *version_helpers.Constraints // the Git branch/tag GitReference plumbing.ReferenceName - // the monotonic version - may be unknown for local or branch - // although version will be monotonic, we can still use semver - Version *goVersion.Version // the file path for local mods FilePath string } -func NewResolvedModRef(modVersion *modconfig.ModVersion) (*ResolvedModRef, error) { +func NewResolvedModRef(requiredModVersion *modconfig.ModVersionConstraint, version *semver.Version) (*ResolvedModRef, error) { res := &ResolvedModRef{ - Name: modVersion.Name, - + Name: requiredModVersion.Name, + Version: version, + Constraint: requiredModVersion.Constraint, // this may be empty strings - FilePath: modVersion.FilePath, + FilePath: requiredModVersion.FilePath, } if res.FilePath == "" { - // NOTE we currently only support explicit (i.e. minor) versions - // if the mod version has either a version constraint or branch, set the git ref - res.SetGitReference(modVersion) + res.setGitReference() } return res, nil } -func (r *ResolvedModRef) SetGitReference(modVersion *modconfig.ModVersion) { +func (r *ResolvedModRef) setGitReference() { + // TODO handle branches - if modVersion.Branch != "" { - r.GitReference = plumbing.NewBranchReferenceName(modVersion.Branch) - // NOTE: we need to set version from branch - return - } - - // so there is a version constraint - - // NOTE: if it is just a major constraint, we need to find the latest version in the major - // for now assume it is a full version - - // NOTE: we cannot just ToString the version as we need the 'v' at the beginning - r.GitReference = plumbing.NewTagReferenceName(modVersion.VersionString) - r.Version = modVersion.VersionConstraint + // NOTE: use the original version string - this will be the tag name + r.GitReference = plumbing.NewTagReferenceName(r.Version.Original()) } // FullName returns name in the format @v func (r *ResolvedModRef) FullName() string { - segments := r.Version.Segments() - return fmt.Sprintf("%s@v%d.%d", r.Name, segments[0], segments[1]) -} - -// SatisfiesVersionConstraint return whether this resolved ref satisfies a version constraint -func (r *ResolvedModRef) SatisfiesVersionConstraint(versionConstraint *goVersion.Version) bool { - // if we do not have a version set, then we cannot satisfy a version constraint - // this may happen if we are a local file or unversioned branch - if r.Version == nil { - return false - } - - return r.Version.GreaterThanOrEqual(versionConstraint) + return modconfig.ModVersionFullName(r.Name, r.Version) } diff --git a/mod_installer/summary_builder.go b/mod_installer/summary_builder.go new file mode 100644 index 000000000..7f31764a6 --- /dev/null +++ b/mod_installer/summary_builder.go @@ -0,0 +1,97 @@ +package mod_installer + +import ( + "fmt" + + "github.com/spf13/viper" + "github.com/turbot/steampipe/constants" + "github.com/turbot/steampipe/steampipeconfig/version_map" + "github.com/turbot/steampipe/utils" +) + +const ( + VerbInstalled = "Installed" + VerbUninstalled = "Uninstalled" + VerbUpgraded = "Upgraded" + VerbDowngraded = "Downgraded" + VerbPruned = "Pruned" +) + +var dryRunVerbs = map[string]string{ + VerbInstalled: "Would install", + VerbUninstalled: "Would uninstall", + VerbUpgraded: "Would upgrade", + VerbDowngraded: "Would downgrade", + VerbPruned: "Would prune", +} + +func getVerb(verb string) string { + if viper.GetBool(constants.ArgDryRun) { + verb = dryRunVerbs[verb] + } + return verb +} + +func BuildInstallSummary(installData *InstallData) string { + // for now treat an install as update - we only install deps which are in the mod.sp but missing in the mod folder + modDependencyPath := installData.WorkspaceMod.GetModDependencyPath() + installCount, installedTreeString := getInstallationResultString(installData.Installed, modDependencyPath) + uninstallCount, uninstalledTreeString := getInstallationResultString(installData.Uninstalled, modDependencyPath) + upgradeCount, upgradeTreeString := getInstallationResultString(installData.Upgraded, modDependencyPath) + downgradeCount, downgradeTreeString := getInstallationResultString(installData.Downgraded, modDependencyPath) + + var installString, upgradeString, downgradeString, uninstallString string + if installCount > 0 { + verb := getVerb(VerbInstalled) + installString = fmt.Sprintf("\n%s %d %s:\n\n%s\n", verb, installCount, utils.Pluralize("mod", installCount), installedTreeString) + } + if uninstallCount > 0 { + verb := getVerb(VerbUninstalled) + uninstallString = fmt.Sprintf("\n%s %d %s:\n\n%s\n", verb, uninstallCount, utils.Pluralize("mod", uninstallCount), uninstalledTreeString) + } + if upgradeCount > 0 { + verb := getVerb(VerbUpgraded) + upgradeString = fmt.Sprintf("\n%s %d %s:\n\n%s\n", verb, upgradeCount, utils.Pluralize("mod", upgradeCount), upgradeTreeString) + } + if downgradeCount > 0 { + verb := getVerb(VerbDowngraded) + downgradeString = fmt.Sprintf("\n%s %d %s:\n\n%s\n", verb, downgradeCount, utils.Pluralize("mod", downgradeCount), downgradeTreeString) + } + + if installCount+uninstallCount+upgradeCount+downgradeCount == 0 { + if len(installData.Lock.InstallCache) == 0 { + return "No mods are installed" + } + return "All mods are up to date" + } + return fmt.Sprintf("%s%s%s%s", installString, upgradeString, downgradeString, uninstallString) +} + +func getInstallationResultString(items version_map.DependencyVersionMap, modDependencyPath string) (int, string) { + var res string + count := len(items.FlatMap()) + if count > 0 { + tree := items.GetDependencyTree(modDependencyPath) + res = tree.String() + } + return count, res +} + +func BuildUninstallSummary(installData *InstallData) string { + // for now treat an install as update - we only install deps which are in the mod.sp but missing in the mod folder + uninstallCount := len(installData.Uninstalled.FlatMap()) + if uninstallCount == 0 { + return "Nothing uninstalled" + } + uninstalledTree := installData.GetUninstalledTree() + + verb := getVerb(VerbUninstalled) + return fmt.Sprintf("\n%s %d %s:\n\n%s", verb, uninstallCount, utils.Pluralize("mod", uninstallCount), uninstalledTree.String()) +} + +func BuildPruneSummary(pruned version_map.VersionListMap) string { + pruneCount := len(pruned.FlatMap()) + + verb := getVerb(VerbPruned) + return fmt.Sprintf("\n%s %d %s:\n", verb, pruneCount, utils.Pluralize("mod", pruneCount)) +} diff --git a/mod_installer/test_data/mods/dep1/mod.sp b/mod_installer/test_data/mods/dep1/mod.sp index ce9ce646b..0b7f0985a 100644 --- a/mod_installer/test_data/mods/dep1/mod.sp +++ b/mod_installer/test_data/mods/dep1/mod.sp @@ -1,7 +1,7 @@ -mod "dep1"{ - requires { - mod "github.com/kaidaguerre/steampipe-mod-m2" { - version = "v1.0" +mod "dep1" { + require { + mod "github.com/turbot/steampipe-mod-aws-compliance" { + version = "0" } } } diff --git a/mod_installer/test_data/mods/dep2/control.sp b/mod_installer/test_data/mods/dep2/control.sp new file mode 100644 index 000000000..f15fe7d09 --- /dev/null +++ b/mod_installer/test_data/mods/dep2/control.sp @@ -0,0 +1,4 @@ +control "c1"{ + description = "control 1" + query = m2.query.m2_q1 +} diff --git a/mod_installer/test_data/mods/dep2/mod.sp b/mod_installer/test_data/mods/dep2/mod.sp new file mode 100644 index 000000000..dd3b821ba --- /dev/null +++ b/mod_installer/test_data/mods/dep2/mod.sp @@ -0,0 +1,7 @@ +mod "dep2" { + require { + mod "github.com/kaidaguerre/steampipe-mod-m2" { + version = "latest" + } + } +} diff --git a/mod_installer/test_data/mods/dep3/control.sp b/mod_installer/test_data/mods/dep3/control.sp new file mode 100644 index 000000000..f15fe7d09 --- /dev/null +++ b/mod_installer/test_data/mods/dep3/control.sp @@ -0,0 +1,4 @@ +control "c1"{ + description = "control 1" + query = m2.query.m2_q1 +} diff --git a/mod_installer/test_data/mods/dep3/mod.sp b/mod_installer/test_data/mods/dep3/mod.sp new file mode 100644 index 000000000..6ad06b8c2 --- /dev/null +++ b/mod_installer/test_data/mods/dep3/mod.sp @@ -0,0 +1,10 @@ +mod "dep3"{ + require { + mod "github.com/kaidaguerre/steampipe-mod-m1" { + version = "v1.*" + } + mod "github.com/kaidaguerre/steampipe-mod-m2" { + version = "v3.1" + } + } +} diff --git a/mod_installer/test_data/mods/dep4/control.sp b/mod_installer/test_data/mods/dep4/control.sp new file mode 100644 index 000000000..f15fe7d09 --- /dev/null +++ b/mod_installer/test_data/mods/dep4/control.sp @@ -0,0 +1,4 @@ +control "c1"{ + description = "control 1" + query = m2.query.m2_q1 +} diff --git a/mod_installer/test_data/mods/dep4/mod.sp b/mod_installer/test_data/mods/dep4/mod.sp new file mode 100644 index 000000000..3ff0c8e12 --- /dev/null +++ b/mod_installer/test_data/mods/dep4/mod.sp @@ -0,0 +1,10 @@ +mod "dep4"{ + require { + mod "github.com/kaidaguerre/steampipe-mod-m1" { + version = "v1.1" + } + mod "github.com/kaidaguerre/steampipe-mod-m2" { + version = "v3.0" + } + } +} diff --git a/mod_installer/test_data/mods/dep5/control.sp b/mod_installer/test_data/mods/dep5/control.sp new file mode 100644 index 000000000..f15fe7d09 --- /dev/null +++ b/mod_installer/test_data/mods/dep5/control.sp @@ -0,0 +1,4 @@ +control "c1"{ + description = "control 1" + query = m2.query.m2_q1 +} diff --git a/mod_installer/test_data/mods/dep5/mod.sp b/mod_installer/test_data/mods/dep5/mod.sp new file mode 100644 index 000000000..82c6a119b --- /dev/null +++ b/mod_installer/test_data/mods/dep5/mod.sp @@ -0,0 +1,10 @@ +mod "dep5"{ + require { + mod "github.com/kaidaguerre/steampipe-mod-m1" { + version = "v1.*" + } + mod "github.com/kaidaguerre/steampipe-mod-m2" { + version = "v3.2" + } + } +} diff --git a/mod_installer/test_data/mods/dep6_x/control.sp b/mod_installer/test_data/mods/dep6_x/control.sp new file mode 100644 index 000000000..f15fe7d09 --- /dev/null +++ b/mod_installer/test_data/mods/dep6_x/control.sp @@ -0,0 +1,4 @@ +control "c1"{ + description = "control 1" + query = m2.query.m2_q1 +} diff --git a/mod_installer/test_data/mods/dep6_x/mod.sp b/mod_installer/test_data/mods/dep6_x/mod.sp new file mode 100644 index 000000000..c51b0003e --- /dev/null +++ b/mod_installer/test_data/mods/dep6_x/mod.sp @@ -0,0 +1,10 @@ +mod "dep6"{ + require { + mod "github.com/kaidaguerre/steampipe-mod-m1" { + version = "v1.*" + } + mod "github.com/kaidaguerre/steampipe-mod-m2" { + version = "v3.3" + } + } +} diff --git a/mod_installer/test_data/mods/dep7_x/control.sp b/mod_installer/test_data/mods/dep7_x/control.sp new file mode 100644 index 000000000..f15fe7d09 --- /dev/null +++ b/mod_installer/test_data/mods/dep7_x/control.sp @@ -0,0 +1,4 @@ +control "c1"{ + description = "control 1" + query = m2.query.m2_q1 +} diff --git a/mod_installer/test_data/mods/dep7_x/mod.sp b/mod_installer/test_data/mods/dep7_x/mod.sp new file mode 100644 index 000000000..1240cbdd3 --- /dev/null +++ b/mod_installer/test_data/mods/dep7_x/mod.sp @@ -0,0 +1,10 @@ +mod "dep7"{ + require { + mod "github.com/kaidaguerre/steampipe-mod-m1" { + version = "v2.*" + } + mod "github.com/kaidaguerre/steampipe-mod-m2" { + version = "v3.0" + } + } +} diff --git a/mod_installer/test_data/mods/dep8_x/control.sp b/mod_installer/test_data/mods/dep8_x/control.sp new file mode 100644 index 000000000..f15fe7d09 --- /dev/null +++ b/mod_installer/test_data/mods/dep8_x/control.sp @@ -0,0 +1,4 @@ +control "c1"{ + description = "control 1" + query = m2.query.m2_q1 +} diff --git a/mod_installer/test_data/mods/dep8_x/mod.sp b/mod_installer/test_data/mods/dep8_x/mod.sp new file mode 100644 index 000000000..16a557170 --- /dev/null +++ b/mod_installer/test_data/mods/dep8_x/mod.sp @@ -0,0 +1,10 @@ +mod "dep8"{ + require { + mod "github.com/kaidaguerre/steampipe-mod-m1" { + version = "v1.0" + } + mod "github.com/kaidaguerre/steampipe-mod-m2" { + version = "v3.2" + } + } +} diff --git a/mod_installer/test_data/mods/dep_empty/c.sp b/mod_installer/test_data/mods/dep_empty/c.sp deleted file mode 100644 index df4e12d8f..000000000 --- a/mod_installer/test_data/mods/dep_empty/c.sp +++ /dev/null @@ -1,8 +0,0 @@ - -benchmark "my_mod_public_resources" { - title = "Public Resources" - description = "Resources that are public." - children = [ - aws_compliance.benchmark.cis_v140_1, - ] -} \ No newline at end of file diff --git a/mod_installer/uninstall.go b/mod_installer/uninstall.go new file mode 100644 index 000000000..c5053f767 --- /dev/null +++ b/mod_installer/uninstall.go @@ -0,0 +1,29 @@ +package mod_installer + +import ( + "github.com/turbot/go-kit/helpers" + "github.com/turbot/steampipe/utils" +) + +func UninstallWorkspaceDependencies(opts *InstallOpts) (*InstallData, error) { + utils.LogTime("cmd.UninstallWorkspaceDependencies") + defer func() { + utils.LogTime("cmd.UninstallWorkspaceDependencies end") + if r := recover(); r != nil { + utils.ShowError(helpers.ToError(r)) + } + }() + + // uninstall workspace dependencies + installer, err := NewModInstaller(opts) + if err != nil { + return nil, err + } + + if err := installer.UninstallWorkspaceDependencies(); err != nil { + return nil, err + } + + return installer.installData, nil + +} diff --git a/steampipeconfig/connection_plugin.go b/steampipeconfig/connection_plugin.go index 9eb4e9ad7..f77725433 100644 --- a/steampipeconfig/connection_plugin.go +++ b/steampipeconfig/connection_plugin.go @@ -34,7 +34,7 @@ type ConnectionPlugin struct { } // CreateConnectionPlugins instantiates plugins for specified connections, fetches schemas and sends connection config -func CreateConnectionPlugins(connections []*modconfig.Connection) (connectionPluginMap map[string]*ConnectionPlugin, err error) { +func CreateConnectionPlugins(connections ...*modconfig.Connection) (connectionPluginMap map[string]*ConnectionPlugin, err error) { log.Printf("[TRACE] CreateConnectionPlugin creating %d connections", len(connections)) // build result map diff --git a/steampipeconfig/connection_updates.go b/steampipeconfig/connection_updates.go index 32529da1a..57f34e852 100644 --- a/steampipeconfig/connection_updates.go +++ b/steampipeconfig/connection_updates.go @@ -140,7 +140,7 @@ func (u *ConnectionUpdates) populateConnectionPlugins(alreadyCreatedConnectionPl // - remove these from list of plugins to create connectionsToCreate := removeConnectionsFromList(updateConnections, alreadyCreatedConnectionPlugins) // now crerate them - connectionPlugins, err := CreateConnectionPlugins(connectionsToCreate) + connectionPlugins, err := CreateConnectionPlugins(connectionsToCreate...) if err != nil { return err } @@ -186,7 +186,7 @@ func getSchemaHashesForDynamicSchemas(requiredConnectionData ConnectionDataMap, } } - connectionsPluginsWithDynamicSchema, err := CreateConnectionPlugins(connectionsWithDynamicSchema.Connections()) + connectionsPluginsWithDynamicSchema, err := CreateConnectionPlugins(connectionsWithDynamicSchema.Connections()...) if err != nil { return nil, nil, err } diff --git a/steampipeconfig/load_mod.go b/steampipeconfig/load_mod.go index 224bc0787..b73e28874 100644 --- a/steampipeconfig/load_mod.go +++ b/steampipeconfig/load_mod.go @@ -9,7 +9,7 @@ import ( "github.com/turbot/steampipe/utils" - goVersion "github.com/hashicorp/go-version" + "github.com/Masterminds/semver" filehelpers "github.com/turbot/go-kit/files" "github.com/turbot/go-kit/helpers" @@ -23,13 +23,29 @@ import ( // if CreatePseudoResources flag is set, construct hcl resources for files with specific extensions // NOTE: it is an error if there is more than 1 mod defined, however zero mods is acceptable // - a default mod will be created assuming there are any resource files -func LoadMod(modPath string, runCtx *parse.RunContext) (mod *modconfig.Mod, err error) { +func LoadMod(modPath string, parentRunCtx *parse.RunContext) (mod *modconfig.Mod, err error) { defer func() { if r := recover(); r != nil { err = helpers.ToError(r) } }() + runCtx := parentRunCtx + // if this it not the root mod, create child a run context for its evaluation + if modPath != parentRunCtx.RootEvalPath { + runCtx = parse.NewRunContext( + parentRunCtx.WorkspaceLock, + modPath, + parse.CreatePseudoResources, + &filehelpers.ListOptions{ + // listFlag specifies whether to load files recursively + Flags: filehelpers.FilesRecursive, + // only load .sp files + Include: filehelpers.InclusionsFromExtensions([]string{constants.ModDataExtension}), + }) + runCtx.BlockTypes = parentRunCtx.BlockTypes + } + // verify the mod folder exists if _, err := os.Stat(modPath); os.IsNotExist(err) { return nil, fmt.Errorf("mod folder %s does not exist", modPath) @@ -87,21 +103,43 @@ func LoadMod(modPath string, runCtx *parse.RunContext) (mod *modconfig.Mod, err return nil, err } + // now add fully populated mod to the parent run context + if modPath != parentRunCtx.RootEvalPath { + parentRunCtx.CurrentMod = mod + parentRunCtx.AddMod(mod) + } + return mod, err } func loadModDependencies(mod *modconfig.Mod, runCtx *parse.RunContext) error { var errors []error - if mod.Requires != nil { - for _, dependencyMod := range mod.Requires.Mods { + if mod.Require != nil { + // now ensure there is a lock file - if we have any mod dependnecies there MUST be a lock file - + // otherwise 'steampipe install' must be run + if err := runCtx.EnsureWorkspaceLock(mod); err != nil { + return err + } + + for _, requiredModVersion := range mod.Require.Mods { + // if we have a locked version, update the required version to reflect this + lockedVersion, err := runCtx.WorkspaceLock.GetLockedModVersionConstraint(requiredModVersion, mod) + if err != nil { + errors = append(errors, err) + continue + } + if lockedVersion != nil { + requiredModVersion = lockedVersion + } + // have we already loaded a mod which satisfied this - if loadedMod, ok := runCtx.LoadedDependencyMods[dependencyMod.Name]; ok { - if loadedMod.Version.GreaterThanOrEqual(dependencyMod.VersionConstraint) { + if loadedMod, ok := runCtx.LoadedDependencyMods[requiredModVersion.Name]; ok { + if requiredModVersion.Constraint.Check(loadedMod.Version) { continue } } - if err := loadModDependency(dependencyMod, runCtx); err != nil { + if err := loadModDependency(requiredModVersion, runCtx); err != nil { errors = append(errors, err) } } @@ -109,17 +147,16 @@ func loadModDependencies(mod *modconfig.Mod, runCtx *parse.RunContext) error { return utils.CombineErrors(errors...) } -func loadModDependency(modDependency *modconfig.ModVersion, runCtx *parse.RunContext) error { +func loadModDependency(modDependency *modconfig.ModVersionConstraint, runCtx *parse.RunContext) error { // dependency mods are installed to /@version // for example workspace_folder/.steampipe/mods/github.com/turbot/steampipe-mod-aws-compliance@v1.0 // we need to list all mod folder in the parent folder: workspace_folder/.steampipe/mods/github.com/turbot/ // for each folder we parse the mod name and version and determine whether it meets the version constraint - // we need to iterate through all mods in the parent folder and find one that sarifies requirements - parentFolder := filepath.Dir(filepath.Join(runCtx.ModInstallationPath, modDependency.Name)) - // get th elast segment of mod name - + // we need to iterate through all mods in the parent folder and find one that satisfies requirements + parentFolder := filepath.Dir(filepath.Join(runCtx.WorkspaceLock.ModInstallationPath, modDependency.Name)) + // get the last segment of mod name dependencyPath, version, err := findInstalledDependency(modDependency, parentFolder) if err != nil { return err @@ -146,13 +183,17 @@ func loadModDependency(modDependency *modconfig.ModVersion, runCtx *parse.RunCon } -func findInstalledDependency(modDependency *modconfig.ModVersion, parentFolder string) (string, *goVersion.Version, error) { +func findInstalledDependency(modDependency *modconfig.ModVersionConstraint, parentFolder string) (string, *semver.Version, error) { shortDepName := filepath.Base(modDependency.Name) entries, err := os.ReadDir(parentFolder) if err != nil { - return "", nil, fmt.Errorf("mod dependency %s is not installed", modDependency.Name) + return "", nil, fmt.Errorf("mod satisfying '%s' is not installed", modDependency) } + // results vars + var dependencyPath string + var dependencyVersion *semver.Version + for _, entry := range entries { split := strings.Split(entry.Name(), "@") if len(split) != 2 { @@ -162,19 +203,28 @@ func findInstalledDependency(modDependency *modconfig.ModVersion, parentFolder s modName := split[0] versionString := strings.TrimPrefix(split[1], "v") if modName == shortDepName { - v, err := goVersion.NewVersion(versionString) + v, err := semver.NewVersion(versionString) if err != nil { // invalid format - ignore continue } - if v.GreaterThanOrEqual(modDependency.VersionConstraint) { - return filepath.Join(parentFolder, entry.Name()), v, nil + if modDependency.Constraint.Check(v) { + // if there is more than 1 mod which satisfied the dependency, fail (for now) + if dependencyVersion != nil { + return "", nil, fmt.Errorf("more than one mod found which satisfies dependency %s@%s", modDependency.Name, modDependency.VersionString) + } + dependencyPath = filepath.Join(parentFolder, entry.Name()) + dependencyVersion = v } } } - return "", nil, fmt.Errorf("mod dependency %s is not installed", modDependency.Name) + // did we find a result? + if dependencyVersion != nil { + return dependencyPath, dependencyVersion, nil + } + return "", nil, fmt.Errorf("mod satisfying '%s' is not installed", modDependency) } // LoadModResourceNames parses all hcl files in modPath and returns the names of all resources diff --git a/steampipeconfig/load_mod_test.go b/steampipeconfig/load_mod_test.go index b2e9884e1..85432528c 100644 --- a/steampipeconfig/load_mod_test.go +++ b/steampipeconfig/load_mod_test.go @@ -428,9 +428,9 @@ Benchmarks: expected: &modconfig.Mod{ ShortName: "m1", FullName: "mod.m1", - Requires: &modconfig.Requires{ + Require: &modconfig.Require{ SteampipeVersionString: "v0.8.0", - Mods: []*modconfig.ModVersion{ + Mods: []*modconfig.ModVersionConstraint{ { Name: "github.com/turbot/aws-core", VersionString: "v1.0", @@ -444,13 +444,13 @@ Benchmarks: expected: &modconfig.Mod{ ShortName: "m1", FullName: "mod.m1", - Requires: &modconfig.Requires{ + Require: &modconfig.Require{ SteampipeVersionString: "v0.8.0", - Mods: []*modconfig.ModVersion{ + Mods: []*modconfig.ModVersionConstraint{ { Name: "github.com/turbot/aws-core", VersionString: "v1.0", - Alias: utils.ToStringPointer("core"), + //Alias: utils.ToStringPointer("core"), }, }, }, diff --git a/steampipeconfig/modconfig/benchmark.go b/steampipeconfig/modconfig/benchmark.go index 9a601c577..ec1832719 100644 --- a/steampipeconfig/modconfig/benchmark.go +++ b/steampipeconfig/modconfig/benchmark.go @@ -24,11 +24,11 @@ type Benchmark struct { ShortName string FullName string `cty:"name"` - ChildNames *[]NamedItem `cty:"children" hcl:"children"` - Description *string `cty:"description" hcl:"description" column:"description,text"` - Documentation *string `cty:"documentation" hcl:"documentation" column:"documentation,text"` - Tags *map[string]string `cty:"tags" hcl:"tags" column:"tags,jsonb"` - Title *string `cty:"title" hcl:"title" column:"title,text"` + ChildNames []NamedItem `cty:"children" hcl:"children,optional"` + Description *string `cty:"description" hcl:"description" column:"description,text"` + Documentation *string `cty:"documentation" hcl:"documentation" column:"documentation,text"` + Tags map[string]string `cty:"tags" hcl:"tags,optional" column:"tags,jsonb"` + Title *string `cty:"title" hcl:"title" column:"title,text"` // list of all block referenced by the resource References []*ResourceReference @@ -37,16 +37,18 @@ type Benchmark struct { ChildNameStrings []string `column:"children,jsonb"` DeclRange hcl.Range - parents []ModTreeItem - children []ModTreeItem - metadata *ResourceMetadata + parents []ModTreeItem + children []ModTreeItem + metadata *ResourceMetadata + UnqualifiedName string } func NewBenchmark(block *hcl.Block) *Benchmark { return &Benchmark{ - ShortName: block.Labels[0], - FullName: fmt.Sprintf("benchmark.%s", block.Labels[0]), - DeclRange: block.DefRange, + ShortName: block.Labels[0], + FullName: fmt.Sprintf("benchmark.%s", block.Labels[0]), + UnqualifiedName: fmt.Sprintf("benchmark.%s", block.Labels[0]), + DeclRange: block.DefRange, } } @@ -60,20 +62,13 @@ func (b *Benchmark) Equals(other *Benchmark) bool { return res } // tags - if b.Tags == nil { - if other.Tags != nil { + if len(b.Tags) != len(other.Tags) { + return false + } + for k, v := range b.Tags { + if otherVal := other.Tags[k]; v != otherVal { return false } - } else { - // we have tags - if other.Tags == nil { - return false - } - for k, v := range *b.Tags { - if otherVal, ok := (*other.Tags)[k]; !ok && v != otherVal { - return false - } - } } if len(b.ChildNameStrings) != len(other.ChildNameStrings) { @@ -100,14 +95,14 @@ func (b *Benchmark) GetDeclRange() *hcl.Range { // OnDecoded implements HclResource func (b *Benchmark) OnDecoded(block *hcl.Block) hcl.Diagnostics { var res hcl.Diagnostics - if b.ChildNames == nil || len(*b.ChildNames) == 0 { + if len(b.ChildNames) == 0 { return nil } // validate each child name appears only once nameMap := make(map[string]bool) - b.ChildNameStrings = make([]string, len(*b.ChildNames)) - for i, n := range *b.ChildNames { + b.ChildNameStrings = make([]string, len(b.ChildNames)) + for i, n := range b.ChildNames { if nameMap[n.Name] { res = append(res, &hcl.Diagnostic{ Severity: hcl.DiagError, @@ -120,6 +115,7 @@ func (b *Benchmark) OnDecoded(block *hcl.Block) hcl.Diagnostics { nameMap[n.Name] = true } + // in order to populate th echildren in the order specified, we create an empty array and populate by index in AddChild b.children = make([]ModTreeItem, len(b.ChildNameStrings)) return res } @@ -132,6 +128,8 @@ func (b *Benchmark) AddReference(ref *ResourceReference) { // SetMod implements HclResource func (b *Benchmark) SetMod(mod *Mod) { b.Mod = mod + b.UnqualifiedName = b.FullName + b.FullName = fmt.Sprintf("%s.%s", mod.ShortName, b.FullName) } // GetMod implements HclResource @@ -222,7 +220,7 @@ func (b *Benchmark) GetDescription() string { // GetTags implements ModTreeItem func (b *Benchmark) GetTags() map[string]string { if b.Tags != nil { - return *b.Tags + return b.Tags } return map[string]string{} } @@ -244,7 +242,7 @@ func (b *Benchmark) GetPaths() []NodePath { } // Name implements ModTreeItem, HclResource, ResourceWithMetadata -// return name in format: 'control.' +// return name in format: '.control.' func (b *Benchmark) Name() string { return b.FullName } @@ -258,8 +256,3 @@ func (b *Benchmark) GetMetadata() *ResourceMetadata { func (b *Benchmark) SetMetadata(metadata *ResourceMetadata) { b.metadata = metadata } - -// QualifiedName returns the name in format: '.control.' -func (b *Benchmark) QualifiedName() string { - return fmt.Sprintf("%s.%s", b.metadata.ModName, b.FullName) -} diff --git a/steampipeconfig/modconfig/control.go b/steampipeconfig/modconfig/control.go index c9236f99f..4ba05e5e5 100644 --- a/steampipeconfig/modconfig/control.go +++ b/steampipeconfig/modconfig/control.go @@ -14,15 +14,15 @@ import ( // Control is a struct representing the Control resource type Control struct { ShortName string - FullName string `cty:"name"` - Description *string `cty:"description" column:"description,text"` - Documentation *string `cty:"documentation" column:"documentation,text"` - SearchPath *string `cty:"search_path" column:"search_path,text"` - SearchPathPrefix *string `cty:"search_path_prefix" column:"search_path_prefix,text"` - Severity *string `cty:"severity" column:"severity,text"` - SQL *string `cty:"sql" column:"sql,text"` - Tags *map[string]string `cty:"tags" column:"tags,jsonb"` - Title *string `cty:"title" column:"title,text"` + FullName string `cty:"name"` + Description *string `cty:"description" column:"description,text"` + Documentation *string `cty:"documentation" column:"documentation,text"` + SearchPath *string `cty:"search_path" column:"search_path,text"` + SearchPathPrefix *string `cty:"search_path_prefix" column:"search_path_prefix,text"` + Severity *string `cty:"severity" column:"severity,text"` + SQL *string `cty:"sql" column:"sql,text"` + Tags map[string]string `cty:"tags" column:"tags,jsonb"` + Title *string `cty:"title" column:"title,text"` Query *Query // args // arguments may be specified by either a map of named args or as a list of positional args @@ -39,14 +39,16 @@ type Control struct { parents []ModTreeItem metadata *ResourceMetadata PreparedStatementName string `column:"prepared_statement_name,text"` + UnqualifiedName string } func NewControl(block *hcl.Block) *Control { control := &Control{ - ShortName: block.Labels[0], - FullName: fmt.Sprintf("control.%s", block.Labels[0]), - DeclRange: block.DefRange, - Args: NewQueryArgs(), + ShortName: block.Labels[0], + FullName: fmt.Sprintf("control.%s", block.Labels[0]), + UnqualifiedName: fmt.Sprintf("control.%s", block.Labels[0]), + DeclRange: block.DefRange, + Args: NewQueryArgs(), } return control } @@ -64,21 +66,13 @@ func (c *Control) Equals(other *Control) bool { if !res { return res } - // tags - if c.Tags == nil { - if other.Tags != nil { + if len(c.Tags) != len(other.Tags) { + return false + } + for k, v := range c.Tags { + if otherVal := other.Tags[k]; v != otherVal { return false } - } else { - // we have tags - if other.Tags == nil { - return false - } - for k, v := range *c.Tags { - if otherVal, ok := (*other.Tags)[k]; !ok && v != otherVal { - return false - } - } } // args @@ -194,7 +188,7 @@ func (c *Control) GetDescription() string { // GetTags implements ModTreeItem func (c *Control) GetTags() map[string]string { if c.Tags != nil { - return *c.Tags + return c.Tags } return map[string]string{} } @@ -210,9 +204,9 @@ func (c *Control) Name() string { return c.FullName } -// QualifiedName returns the name in format: '.control.' -func (c *Control) QualifiedName() string { - return fmt.Sprintf("%s.%s", c.metadata.ModName, c.FullName) +// QualifiedNameWithVersion returns the name in format: '@version.control.' +func (q *Control) QualifiedNameWithVersion() string { + return fmt.Sprintf("%s.%s", q.Mod.NameWithVersion(), q.FullName) } // GetPaths implements ModTreeItem @@ -242,6 +236,8 @@ func (c *Control) AddReference(ref *ResourceReference) { // SetMod implements HclResource func (c *Control) SetMod(mod *Mod) { c.Mod = mod + c.UnqualifiedName = c.FullName + c.FullName = fmt.Sprintf("%s.%s", mod.ShortName, c.FullName) } // GetMod implements HclResource @@ -280,5 +276,5 @@ func (c *Control) GetPreparedStatementName() string { // ModName implements QueryProvider func (c *Control) ModName() string { - return c.Mod.ShortName + return c.Mod.NameWithVersion() } diff --git a/steampipeconfig/modconfig/interfaces.go b/steampipeconfig/modconfig/interfaces.go index 3e5fab0aa..746ef6e28 100644 --- a/steampipeconfig/modconfig/interfaces.go +++ b/steampipeconfig/modconfig/interfaces.go @@ -29,6 +29,7 @@ type ModTreeItem interface { GetTags() map[string]string // GetPaths returns an array resource paths GetPaths() []NodePath + GetMod() *Mod } // HclResource must be implemented by resources defined in HCL diff --git a/steampipeconfig/modconfig/local.go b/steampipeconfig/modconfig/local.go index a5249c89a..e358f1361 100644 --- a/steampipeconfig/modconfig/local.go +++ b/steampipeconfig/modconfig/local.go @@ -51,6 +51,7 @@ func (l *Local) AddReference(*ResourceReference) {} // SetMod implements HclResource func (l *Local) SetMod(mod *Mod) { l.Mod = mod + l.FullName = fmt.Sprintf("%s.%s", mod.ShortName, l.FullName) } // GetMod implements HclResource diff --git a/steampipeconfig/modconfig/mod.go b/steampipeconfig/modconfig/mod.go index 513e048b6..b1e97bdae 100644 --- a/steampipeconfig/modconfig/mod.go +++ b/steampipeconfig/modconfig/mod.go @@ -3,15 +3,18 @@ package modconfig import ( "errors" "fmt" + "os" "path/filepath" "sort" "strings" - goVersion "github.com/hashicorp/go-version" - + "github.com/Masterminds/semver" "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/turbot/go-kit/helpers" "github.com/turbot/go-kit/types" typehelpers "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe/constants" "github.com/zclconf/go-cty/cty" ) @@ -30,22 +33,24 @@ type Mod struct { ModDependencyPath string `cty:"mod_dependency_path"` // attributes - Categories *[]string `cty:"categories" hcl:"categories" column:"categories,jsonb"` - Color *string `cty:"color" hcl:"color" column:"color,text"` - Description *string `cty:"description" hcl:"description" column:"description,text"` - Documentation *string `cty:"documentation" hcl:"documentation" column:"documentation,text"` - Icon *string `cty:"icon" hcl:"icon" column:"icon,text"` - Tags *map[string]string `cty:"tags" hcl:"tags" column:"tags,jsonb"` - Title *string `cty:"title" hcl:"title" column:"title,text"` + Categories []string `cty:"categories" hcl:"categories,optional" column:"categories,jsonb"` + Color *string `cty:"color" hcl:"color" column:"color,text"` + Description *string `cty:"description" hcl:"description" column:"description,text"` + Documentation *string `cty:"documentation" hcl:"documentation" column:"documentation,text"` + Icon *string `cty:"icon" hcl:"icon" column:"icon,text"` + Tags map[string]string `cty:"tags" hcl:"tags,optional" column:"tags,jsonb"` + Title *string `cty:"title" hcl:"title" column:"title,text"` // list of all blocks referenced by the resource References []*ResourceReference // blocks - Requires *Requires `hcl:"requires,block"` - OpenGraph *OpenGraph `hcl:"opengraph,block" column:"open_graph,jsonb"` + Require *Require `hcl:"require,block"` + LegacyRequire *Require `hcl:"requires,block"` + OpenGraph *OpenGraph `hcl:"opengraph,block" column:"open_graph,jsonb"` - Version *goVersion.Version + VersionString string `cty:"version"` + Version *semver.Version Queries map[string]*Query Controls map[string]*Control @@ -55,34 +60,49 @@ type Mod struct { Variables map[string]*Variable Locals map[string]*Local - // flat list of all resources - AllResources map[string]HclResource - - // list of benchmark names, sorted alphabetically - benchmarksOrdered []string - // ModPath is the installation location of the mod ModPath string DeclRange hcl.Range + // all children as an array of hcl resources - built before the 'children' array + flatChildren []HclResource + // array of direct mod children - excluds resources which are children of othe rresources children []ModTreeItem metadata *ResourceMetadata } func NewMod(shortName, modPath string, defRange hcl.Range) *Mod { - return &Mod{ - ShortName: shortName, - FullName: fmt.Sprintf("mod.%s", shortName), - Queries: make(map[string]*Query), - Controls: make(map[string]*Control), - Benchmarks: make(map[string]*Benchmark), - Reports: make(map[string]*Report), - Panels: make(map[string]*Panel), - Variables: make(map[string]*Variable), - Locals: make(map[string]*Local), - ModPath: modPath, - DeclRange: defRange, - AllResources: make(map[string]HclResource), + mod := &Mod{ + ShortName: shortName, + FullName: fmt.Sprintf("mod.%s", shortName), + Queries: make(map[string]*Query), + Controls: make(map[string]*Control), + Benchmarks: make(map[string]*Benchmark), + Reports: make(map[string]*Report), + Panels: make(map[string]*Panel), + Variables: make(map[string]*Variable), + Locals: make(map[string]*Local), + ModPath: modPath, + DeclRange: defRange, + Require: newRequire(), + } + + // try to derive mod version from the path + mod.setVersion() + return mod +} + +func (m *Mod) setVersion() { + segments := strings.Split(m.ModPath, "@") + if len(segments) == 1 { + return + } + versionString := segments[len(segments)-1] + // try to set version, ignoring error + version, err := semver.NewVersion(versionString) + if err == nil { + m.Version = version + m.VersionString = fmt.Sprintf("%d.%d", version.Major(), version.Minor()) } } @@ -108,30 +128,23 @@ func (m *Mod) Equals(other *Mod) bool { return false } - if len(*m.Categories) != len(*other.Categories) { + if len(m.Categories) != len(other.Categories) { return false } - for i, c := range *m.Categories { - if (*other.Categories)[i] != c { + for i, c := range m.Categories { + if (other.Categories)[i] != c { return false } } } // tags - if m.Tags == nil { - if other.Tags != nil { + if len(m.Tags) != len(other.Tags) { + return false + } + for k, v := range m.Tags { + if otherVal := other.Tags[k]; v != otherVal { return false } - } else { - // we have tags - if other.Tags == nil { - return false - } - for k, v := range *m.Tags { - if otherVal, ok := (*other.Tags)[k]; !ok && v != otherVal { - return false - } - } } // controls @@ -266,22 +279,22 @@ func (m *Mod) String() string { } versionString := "" - if m.Version != nil { - versionString = fmt.Sprintf("\nVersion: %s", types.SafeString(m.Version)) + if m.VersionString == "" { + versionString = fmt.Sprintf("\nVersion: v%s", m.VersionString) } var requiresStrings []string var requiresString string - if m.Requires != nil { - if m.Requires.SteampipeVersionString != "" { - requiresStrings = append(requiresStrings, fmt.Sprintf("Steampipe %s", m.Requires.SteampipeVersionString)) + if m.Require != nil { + if m.Require.SteampipeVersionString != "" { + requiresStrings = append(requiresStrings, fmt.Sprintf("Steampipe %s", m.Require.SteampipeVersionString)) } - for _, m := range m.Requires.Mods { + for _, m := range m.Require.Mods { requiresStrings = append(requiresStrings, m.String()) } - for _, p := range m.Requires.Plugins { + for _, p := range m.Require.Plugins { requiresStrings = append(requiresStrings, p.String()) } - requiresString = fmt.Sprintf("Requires: \n%s", strings.Join(requiresStrings, "\n")) + requiresString = fmt.Sprintf("Require: \n%s", strings.Join(requiresStrings, "\n")) } return fmt.Sprintf(`Name: %s @@ -306,146 +319,14 @@ Benchmarks: ) } -// BuildResourceTree builds the control tree structure by setting the parent property for each control and benchmar -// NOTE: this also builds the sorted benchmark list -func (m *Mod) BuildResourceTree() error { - // build sorted list of benchmarks - m.benchmarksOrdered = make([]string, len(m.Benchmarks)) - idx := 0 - for name, benchmark := range m.Benchmarks { - // save this benchmark name - m.benchmarksOrdered[idx] = name - idx++ - - // add benchmark into control tree - if err := m.addItemIntoResourceTree(benchmark); err != nil { - return err - } +func (m *Mod) NameWithVersion() string { + if m.VersionString == "" { + return m.ShortName } - // now sort the benchmark names - sort.Strings(m.benchmarksOrdered) - - for _, control := range m.Controls { - if err := m.addItemIntoResourceTree(control); err != nil { - return err - } - } - for _, panel := range m.Panels { - if err := m.addItemIntoResourceTree(panel); err != nil { - return err - } - } - for _, report := range m.Reports { - if err := m.addItemIntoResourceTree(report); err != nil { - return err - } - } - return nil + return fmt.Sprintf("%s@%s", m.ShortName, m.VersionString) } -func (m *Mod) addItemIntoResourceTree(item ModTreeItem) error { - parents := m.getParents(item) - - // so we have a result - add into tree - for _, p := range parents { - // TODO validity checking - //for _, parentPath := range p.GetPaths() { - // // check this item does not exist in the parent path - // if helpers.StringSliceContains(parentPath, item.Name()) { - // return fmt.Errorf("cyclical dependency adding '%s' into control tree - parent '%s'", item.Name(), p.Name()) - // } - item.AddParent(p) - p.AddChild(item) - //} - } - - return nil -} - -func (m *Mod) AddResource(item HclResource) hcl.Diagnostics { - var diags hcl.Diagnostics - switch r := item.(type) { - case *Query: - name := r.Name() - // check for dupes - if _, ok := m.Queries[name]; ok { - diags = append(diags, duplicateResourceDiagnostics(item)) - break - } - m.Queries[name] = r - - case *Control: - name := r.Name() - // check for dupes - if _, ok := m.Controls[name]; ok { - diags = append(diags, duplicateResourceDiagnostics(item)) - break - } - m.Controls[name] = r - - case *Benchmark: - name := r.Name() - // check for dupes - if _, ok := m.Benchmarks[name]; ok { - diags = append(diags, duplicateResourceDiagnostics(item)) - break - } else { - m.Benchmarks[name] = r - } - - case *Panel: - name := r.Name() - // check for dupes - if _, ok := m.Panels[name]; ok { - diags = append(diags, duplicateResourceDiagnostics(item)) - break - } else { - m.Panels[name] = r - } - - case *Report: - name := r.Name() - // check for dupes - if _, ok := m.Reports[name]; ok { - diags = append(diags, duplicateResourceDiagnostics(item)) - break - } else { - m.Reports[name] = r - } - - case *Variable: - name := r.Name() - // check for dupes - if _, ok := m.Variables[name]; ok { - diags = append(diags, duplicateResourceDiagnostics(item)) - break - } else { - m.Variables[name] = r - } - - case *Local: - name := r.Name() - // check for dupes - if _, ok := m.Locals[name]; ok { - diags = append(diags, duplicateResourceDiagnostics(item)) - break - } else { - m.Locals[name] = r - } - } - m.AllResources[item.Name()] = item - return diags -} - -func duplicateResourceDiagnostics(item HclResource) *hcl.Diagnostic { - return &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: fmt.Sprintf("mod defines more than one resource named %s", item.Name()), - Subject: item.GetDeclRange(), - } -} - -// AddChild implements ModTreeItem +// AddChild implements ModTreeItem func (m *Mod) AddChild(child ModTreeItem) error { m.children = append(m.children, child) return nil @@ -463,10 +344,15 @@ func (m *Mod) GetParents() []ModTreeItem { // Name implements ModTreeItem, HclResource func (m *Mod) Name() string { - if m.Version == nil { - return m.FullName + return m.FullName +} + +// GetModDependencyPath ModDependencyPath if it is set. If not it returns NameWithVersion() +func (m *Mod) GetModDependencyPath() string { + if m.ModDependencyPath != "" { + return m.ModDependencyPath } - return fmt.Sprintf("%s@%s", m.FullName, types.SafeString(m.Version)) + return m.NameWithVersion() } // GetTitle implements ModTreeItem @@ -482,7 +368,7 @@ func (m *Mod) GetDescription() string { // GetTags implements ModTreeItem func (m *Mod) GetTags() map[string]string { if m.Tags != nil { - return *m.Tags + return m.Tags } return map[string]string{} } @@ -520,13 +406,31 @@ func (m *Mod) CtyValue() (cty.Value, error) { } // OnDecoded implements HclResource -func (m *Mod) OnDecoded(*hcl.Block) hcl.Diagnostics { +func (m *Mod) OnDecoded(block *hcl.Block) hcl.Diagnostics { + // if VersionString is set, set Version + if m.VersionString != "" && m.Version == nil { + m.Version, _ = semver.NewVersion(m.VersionString) + } + // build flat children + m.buildFlatChilden() - // initialise our Requires - if m.Requires == nil { + // handle legacy requires block + if m.LegacyRequire != nil && !m.Require.Empty() { + if m.Require != nil && !m.Require.Empty() { + return hcl.Diagnostics{&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Both 'require' and legacy 'requires' blocks are defined", + Subject: &block.DefRange, + }} + } + m.Require = m.LegacyRequire + } + + // initialise our Require + if m.Require == nil { return nil } - return m.Requires.Initialise() + return m.Require.initialise() } // AddReference implements HclResource @@ -558,15 +462,15 @@ func (m *Mod) SetMetadata(metadata *ResourceMetadata) { } // get the parent item for this ModTreeItem -// first check all benchmarks - if they do not have this as child, default to the mod func (m *Mod) getParents(item ModTreeItem) []ModTreeItem { var parents []ModTreeItem + for _, benchmark := range m.Benchmarks { if benchmark.ChildNames == nil { continue } // check all child names of this benchmark for a matching name - for _, childName := range *benchmark.ChildNames { + for _, childName := range benchmark.ChildNames { if childName.Name == item.Name() { parents = append(parents, benchmark) } @@ -581,6 +485,9 @@ func (m *Mod) getParents(item ModTreeItem) []ModTreeItem { } } for _, panel := range m.Panels { + if panel.Name() == item.Name() { + parents = append(parents, m) + } // check all child names of this benchmark for a matching name for _, child := range panel.GetChildren() { if child.Name() == item.Name() { @@ -588,13 +495,24 @@ func (m *Mod) getParents(item ModTreeItem) []ModTreeItem { } } } - if len(parents) == 0 { - // fall back on mod + // if this item has no parents and is a child of the mod, set the mod as parent + if len(parents) == 0 && m.hasChild(item) { parents = []ModTreeItem{m} + } return parents } +// is the given item a child of the mod +func (m *Mod) hasChild(item ModTreeItem) bool { + for _, c := range m.flatChildren { + if c.Name() == item.Name() { + return true + } + } + return false +} + // GetChildControls return a flat list of controls underneath the mod func (m *Mod) GetChildControls() []*Control { var res []*Control @@ -603,3 +521,168 @@ func (m *Mod) GetChildControls() []*Control { } return res } + +func (m *Mod) AddModDependencies(modVersions map[string]*ModVersionConstraint) { + m.Require.AddModDependencies(modVersions) +} + +func (m *Mod) RemoveModDependencies(modVersions map[string]*ModVersionConstraint) { + m.Require.RemoveModDependencies(modVersions) +} + +func (m *Mod) RemoveAllModDependencies() { + m.Require.RemoveAllModDependencies() +} + +func (m *Mod) Save() error { + f := hclwrite.NewEmptyFile() + rootBody := f.Body() + + modBody := rootBody.AppendNewBlock("mod", []string{m.ShortName}).Body() + if m.Title != nil { + modBody.SetAttributeValue("title", cty.StringVal(*m.Title)) + } + if m.Description != nil { + modBody.SetAttributeValue("description", cty.StringVal(*m.Description)) + } + if m.Color != nil { + modBody.SetAttributeValue("color", cty.StringVal(*m.Color)) + } + if m.Documentation != nil { + modBody.SetAttributeValue("documentation", cty.StringVal(*m.Documentation)) + } + if m.Icon != nil { + modBody.SetAttributeValue("icon", cty.StringVal(*m.Icon)) + } + if len(m.Categories) > 0 { + categoryValues := make([]cty.Value, len(m.Categories)) + for i, c := range m.Categories { + categoryValues[i] = cty.StringVal(typehelpers.SafeString(c)) + } + modBody.SetAttributeValue("categories", cty.ListVal(categoryValues)) + } + + if len(m.Tags) > 0 { + tagMap := make(map[string]cty.Value, len(m.Tags)) + for k, v := range m.Tags { + tagMap[k] = cty.StringVal(v) + } + modBody.SetAttributeValue("tags", cty.MapVal(tagMap)) + } + + // opengraph + if opengraph := m.OpenGraph; opengraph != nil { + opengraphBody := modBody.AppendNewBlock("opengraph", nil).Body() + if opengraph.Title != nil { + opengraphBody.SetAttributeValue("title", cty.StringVal(*opengraph.Title)) + } + if opengraph.Description != nil { + opengraphBody.SetAttributeValue("description", cty.StringVal(*opengraph.Description)) + } + if opengraph.Image != nil { + opengraphBody.SetAttributeValue("image", cty.StringVal(*opengraph.Image)) + } + + } + + // require + if require := m.Require; require != nil && !m.Require.Empty() { + requiresBody := modBody.AppendNewBlock("require", nil).Body() + if require.SteampipeVersionString != "" { + requiresBody.SetAttributeValue("steampipe", cty.StringVal(require.SteampipeVersionString)) + } + if len(require.Plugins) > 0 { + pluginValues := make([]cty.Value, len(require.Plugins)) + for i, p := range require.Plugins { + pluginValues[i] = cty.StringVal(typehelpers.SafeString(p)) + } + requiresBody.SetAttributeValue("plugins", cty.ListVal(pluginValues)) + } + if len(require.Mods) > 0 { + for _, m := range require.Mods { + modBody := requiresBody.AppendNewBlock("mod", []string{m.Name}).Body() + modBody.SetAttributeValue("version", cty.StringVal(m.VersionString)) + } + } + } + + // load existing mod data and remove the mod definitions from it + nonModData, err := m.loadNonModDataInModFile() + if err != nil { + return err + } + modData := append(f.Bytes(), nonModData...) + return os.WriteFile(constants.ModFilePath(m.ModPath), modData, 0644) +} + +func (m *Mod) HasDependentMods() bool { + return m.Require != nil && len(m.Require.Mods) > 0 +} + +func (m *Mod) GetModDependency(modName string) *ModVersionConstraint { + if m.Require == nil { + return nil + } + return m.Require.GetModDependency(modName) +} + +func (m *Mod) buildFlatChilden() { + res := make([]HclResource, len(m.Queries)+len(m.Controls)+len(m.Benchmarks)+len(m.Reports)+len(m.Panels)+len(m.Variables)+len(m.Locals)) + + idx := 0 + for _, r := range m.Queries { + res[idx] = r + idx++ + } + for _, r := range m.Controls { + res[idx] = r + idx++ + } + for _, r := range m.Benchmarks { + res[idx] = r + idx++ + } + for _, r := range m.Reports { + res[idx] = r + idx++ + } + for _, r := range m.Panels { + res[idx] = r + idx++ + } + for _, r := range m.Variables { + res[idx] = r + idx++ + } + for _, r := range m.Locals { + res[idx] = r + idx++ + } + m.flatChildren = res +} + +func (m *Mod) loadNonModDataInModFile() ([]byte, error) { + modFilePath := constants.ModFilePath(m.ModPath) + if !helpers.FileExists(modFilePath) { + return nil, nil + } + + fileData, err := os.ReadFile(modFilePath) + if err != nil { + return nil, err + } + + fileLines := strings.Split(string(fileData), "\n") + decl := m.DeclRange + // just use line positions + start := decl.Start.Line - 1 + end := decl.End.Line - 1 + + var resLines []string + for i, line := range fileLines { + if (i < start || i > end) && line != "" { + resLines = append(resLines, line) + } + } + return []byte(strings.Join(resLines, "\n")), nil +} diff --git a/steampipeconfig/modconfig/mod_map.go b/steampipeconfig/modconfig/mod_map.go index 1c0f29222..f6c05e871 100644 --- a/steampipeconfig/modconfig/mod_map.go +++ b/steampipeconfig/modconfig/mod_map.go @@ -4,7 +4,7 @@ import ( "strings" ) -// ModMap is a map of mod name to mod-version +// ModMap is a map of mod name to mod type ModMap map[string]*Mod func (m ModMap) String() string { diff --git a/steampipeconfig/modconfig/mod_name.go b/steampipeconfig/modconfig/mod_name.go new file mode 100644 index 000000000..7deadf943 --- /dev/null +++ b/steampipeconfig/modconfig/mod_name.go @@ -0,0 +1,42 @@ +package modconfig + +import ( + "bytes" + "fmt" + "strings" + + "github.com/Masterminds/semver" +) + +func ModVersionFullName(name string, version *semver.Version) string { + if version == nil { + return name + } + versionString := GetMonotonicVersionString(version) + return fmt.Sprintf("%s@v%s", name, versionString) +} + +func GetMonotonicVersionString(v *semver.Version) string { + var buf bytes.Buffer + fmt.Fprintf(&buf, "%d.%d", v.Major(), v.Minor()) + if v.Metadata() != "" { + fmt.Fprintf(&buf, "+%s", v.Metadata()) + } + return buf.String() +} + +func ParseModFullName(fullName string) (modName string, modVersion *semver.Version, err error) { + // we expect modLongName to be of form github.com/turbot/steampipe-mod-m2@v1.0 + // split to get the name and version + parts := strings.Split(fullName, "@") + if len(parts) != 2 { + err = fmt.Errorf("invalid mod full name %s", fullName) + return + } + modName = parts[0] + modVersion, err = semver.NewVersion(parts[1]) + if err != nil { + err = fmt.Errorf("mod file %s has invalid version", fullName) + } + return +} diff --git a/steampipeconfig/modconfig/mod_resource_tree.go b/steampipeconfig/modconfig/mod_resource_tree.go new file mode 100644 index 000000000..4a19f86cb --- /dev/null +++ b/steampipeconfig/modconfig/mod_resource_tree.go @@ -0,0 +1,154 @@ +package modconfig + +import ( + "fmt" + + "github.com/hashicorp/hcl/v2" +) + +// BuildResourceTree builds the control tree structure by setting the parent property for each control and benchmar +// NOTE: this also builds the sorted benchmark list +func (m *Mod) BuildResourceTree(loadedDependencyMods ModMap) error { + if err := m.addResourcesIntoTree(m); err != nil { + return err + } + defer m.validateResourceTree() + + if !m.HasDependentMods() { + return nil + } + // add dependent mods into tree + for _, requiredMod := range m.Require.Mods { + // find this mod in installed dependency mods + depMod, ok := loadedDependencyMods[requiredMod.Name] + if !ok { + return fmt.Errorf("dependency mod %s is not loaded", requiredMod.Name) + } + if err := m.addResourcesIntoTree(depMod); err != nil { + return err + } + } + + return nil +} + +func (m *Mod) addResourcesIntoTree(sourceMod *Mod) error { + for _, benchmark := range sourceMod.Benchmarks { + // add benchmark into control tree + if err := m.addItemIntoResourceTree(benchmark); err != nil { + return err + } + } + for _, control := range sourceMod.Controls { + if err := m.addItemIntoResourceTree(control); err != nil { + return err + } + } + for _, panel := range sourceMod.Panels { + if err := m.addItemIntoResourceTree(panel); err != nil { + return err + } + } + for _, report := range sourceMod.Reports { + if err := m.addItemIntoResourceTree(report); err != nil { + return err + } + } + return nil +} + +func (m *Mod) addItemIntoResourceTree(item ModTreeItem) error { + parents := append(m.getParents(item)) + + // so we have a result - add into tree + for _, p := range parents { + item.AddParent(p) + p.AddChild(item) + + } + + return nil +} + +func (m *Mod) AddResource(item HclResource) hcl.Diagnostics { + var diags hcl.Diagnostics + switch r := item.(type) { + case *Query: + name := r.Name() + // check for dupes + if _, ok := m.Queries[name]; ok { + diags = append(diags, duplicateResourceDiagnostics(item)) + break + } + m.Queries[name] = r + + case *Control: + name := r.Name() + // check for dupes + if _, ok := m.Controls[name]; ok { + diags = append(diags, duplicateResourceDiagnostics(item)) + break + } + m.Controls[name] = r + + case *Benchmark: + name := r.Name() + // check for dupes + if _, ok := m.Benchmarks[name]; ok { + diags = append(diags, duplicateResourceDiagnostics(item)) + break + } else { + m.Benchmarks[name] = r + } + + case *Panel: + name := r.Name() + // check for dupes + if _, ok := m.Panels[name]; ok { + diags = append(diags, duplicateResourceDiagnostics(item)) + break + } else { + m.Panels[name] = r + } + + case *Report: + name := r.Name() + // check for dupes + if _, ok := m.Reports[name]; ok { + diags = append(diags, duplicateResourceDiagnostics(item)) + break + } else { + m.Reports[name] = r + } + + case *Variable: + // NOTE: add variable by unqualified name + name := r.UnqualifiedName + // check for dupes + if _, ok := m.Variables[name]; ok { + diags = append(diags, duplicateResourceDiagnostics(item)) + break + } else { + m.Variables[name] = r + } + + case *Local: + name := r.Name() + // check for dupes + if _, ok := m.Locals[name]; ok { + diags = append(diags, duplicateResourceDiagnostics(item)) + break + } else { + m.Locals[name] = r + } + } + return diags +} + +func duplicateResourceDiagnostics(item HclResource) *hcl.Diagnostic { + return &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("mod defines more than one resource named %s", item.Name()), + Subject: item.GetDeclRange(), + } +} diff --git a/steampipeconfig/modconfig/mod_validate.go b/steampipeconfig/modconfig/mod_validate.go new file mode 100644 index 000000000..ac3a09ace --- /dev/null +++ b/steampipeconfig/modconfig/mod_validate.go @@ -0,0 +1,32 @@ +package modconfig + +import ( + "fmt" + + "github.com/turbot/steampipe/utils" +) + +// ensure we have resolved all children in the resource tree +func (m *Mod) validateResourceTree() error { + var errors []error + for _, child := range m.GetChildren() { + if err := m.validateChildren(child); err != nil { + errors = append(errors, err) + } + } + return utils.CombineErrorsWithPrefix(fmt.Sprintf("failed to resolve children for %d resources", len(errors)), errors...) +} + +func (m *Mod) validateChildren(item ModTreeItem) error { + missing := 0 + for _, child := range item.GetChildren() { + if child == nil { + missing++ + + } + } + if missing > 0 { + return fmt.Errorf("%s has %d unresolved children", item.Name(), missing) + } + return nil +} diff --git a/steampipeconfig/modconfig/mod_version.go b/steampipeconfig/modconfig/mod_version.go deleted file mode 100644 index e5d29d7b4..000000000 --- a/steampipeconfig/modconfig/mod_version.go +++ /dev/null @@ -1,69 +0,0 @@ -package modconfig - -import ( - "fmt" - "strings" - - goVersion "github.com/hashicorp/go-version" - typehelpers "github.com/turbot/go-kit/types" - - "github.com/hashicorp/hcl/v2" - - "github.com/turbot/go-kit/helpers" -) - -type ModVersion struct { - // the fully qualified mod name, e.g. github.com/turbot/mod1 - Name string `cty:"name" hcl:"name,label"` - VersionString string `cty:"version" hcl:"version"` - Alias *string `cty:"alias" hcl:"alias,optional"` - - // only one of VersionConstraint, Branch and FilePath will be set - VersionConstraint *goVersion.Version - // the branch to use - Branch string - // the local file location to use - FilePath string - - DeclRange hcl.Range -} - -func (m *ModVersion) FullName() string { - if m.HasVersion() { - return fmt.Sprintf("%s@%s", m.Name, m.VersionString) - } - return m.Name -} - -// HasVersion returns whether the mod has a version specified, or is the latest -// if no version is specified, or the version is "latest", this is the latest version -func (m *ModVersion) HasVersion() bool { - return !helpers.StringSliceContains([]string{"", "latest"}, m.VersionString) -} - -func (m *ModVersion) String() string { - if alias := typehelpers.SafeString(m.Alias); alias != "" { - return fmt.Sprintf("mod %s (%s)", m.FullName(), alias) - } - return fmt.Sprintf("mod %s", m.FullName()) -} - -// Initialise parses the version and name properties -func (m *ModVersion) Initialise() hcl.Diagnostics { - var diags hcl.Diagnostics - - if strings.HasPrefix(m.VersionString, "file:") { - m.FilePath = m.VersionString - return diags - } - // does the version parse as a semver version - if v, err := goVersion.NewVersion(m.VersionString); err == nil { - m.VersionConstraint = v - return diags - } - - // otherwise assume it is a branch - m.Branch = m.VersionString - - return diags -} diff --git a/steampipeconfig/modconfig/mod_version_constrain_collection.go b/steampipeconfig/modconfig/mod_version_constrain_collection.go new file mode 100644 index 000000000..3f96f7aeb --- /dev/null +++ b/steampipeconfig/modconfig/mod_version_constrain_collection.go @@ -0,0 +1,25 @@ +package modconfig + +// ModVersionConstraintCollection is a collection of ModVersionConstraint instances and implements the sort +// interface. See the sort package for more details. +// https://golang.org/pkg/sort/ +type ModVersionConstraintCollection []*ModVersionConstraint + +// Len returns the length of a collection. The number of Version instances +// on the slice. +func (c ModVersionConstraintCollection) Len() int { + return len(c) +} + +// Less is needed for the sort interface to compare two Version objects on the +// slice. If checks if one is less than the other. +func (c ModVersionConstraintCollection) Less(i, j int) bool { + // sort by name + return c[i].Name < (c[j].Name) +} + +// Swap is needed for the sort interface to replace the Version objects +// at two different positions in the slice. +func (c ModVersionConstraintCollection) Swap(i, j int) { + c[i], c[j] = c[j], c[i] +} diff --git a/steampipeconfig/modconfig/mod_version_constraint.go b/steampipeconfig/modconfig/mod_version_constraint.go new file mode 100644 index 000000000..6545dc89e --- /dev/null +++ b/steampipeconfig/modconfig/mod_version_constraint.go @@ -0,0 +1,114 @@ +package modconfig + +import ( + "fmt" + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/turbot/go-kit/helpers" + "github.com/turbot/steampipe/version_helpers" +) + +const filePrefix = "file:" + +type VersionConstrainCollection []*ModVersionConstraint + +type ModVersionConstraint struct { + // the fully qualified mod name, e.g. github.com/turbot/mod1 + Name string `cty:"name" hcl:"name,label"` + VersionString string `cty:"version" hcl:"version"` + // only one of Constraint, Branch and FilePath will be set + Constraint *version_helpers.Constraints + // // NOTE: aliases will be supported in the future + //Alias string `cty:"alias" hcl:"alias"` + // the branch to use + Branch string + // the local file location to use + FilePath string + DeclRange hcl.Range +} + +func NewModVersionConstraint(modFullName string) (*ModVersionConstraint, error) { + var m *ModVersionConstraint + // if name has `file:` prefix, just set the name and ignore version + if strings.HasPrefix(modFullName, filePrefix) { + m = &ModVersionConstraint{Name: modFullName} + } else { + // otherwise try to extract version from name + segments := strings.Split(modFullName, "@") + if len(segments) > 2 { + return nil, fmt.Errorf("invalid mod name %s", modFullName) + } + m = &ModVersionConstraint{Name: segments[0]} + if len(segments) == 2 { + m.VersionString = segments[1] + } + } + + // try to convert version into a semver constraint + if err := m.Initialise(); err != nil { + return nil, err + } + return m, nil +} + +func (m *ModVersionConstraint) FullName() string { + if m.HasVersion() { + return fmt.Sprintf("%s@%s", m.Name, m.VersionString) + } + return m.Name +} + +// HasVersion returns whether the mod has a version specified, or is the latest +// if no version is specified, or the version is "latest", this is the latest version +func (m *ModVersionConstraint) HasVersion() bool { + return !helpers.StringSliceContains([]string{"", "latest", "*"}, m.VersionString) +} + +func (m *ModVersionConstraint) String() string { + return fmt.Sprintf("%s", m.FullName()) +} + +// Initialise parses the version and name properties +func (m *ModVersionConstraint) Initialise() hcl.Diagnostics { + if strings.HasPrefix(m.Name, filePrefix) { + m.setFilePath() + return nil + } + var diags hcl.Diagnostics + + if m.VersionString == "" { + m.Constraint, _ = version_helpers.NewConstraint("*") + m.VersionString = "latest" + return diags + } + if m.VersionString == "latest" { + m.Constraint, _ = version_helpers.NewConstraint("*") + return diags + } + // does the version parse as a semver version + if c, err := version_helpers.NewConstraint(m.VersionString); err == nil { + // no error + m.Constraint = c + return diags + } + + // todo handle branch and commit hash + + // so there was an error + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("invalid mod version %s", m.VersionString), + Subject: &m.DeclRange, + }) + return diags +} + +func (m *ModVersionConstraint) setFilePath() { + m.FilePath = strings.TrimPrefix(m.FilePath, filePrefix) +} + +func (m *ModVersionConstraint) Equals(other *ModVersionConstraint) bool { + // just check the hcl properties + return m.Name == other.Name && m.VersionString == other.VersionString +} diff --git a/steampipeconfig/modconfig/open_graph.go b/steampipeconfig/modconfig/open_graph.go index 241f0a76e..047a12eb4 100644 --- a/steampipeconfig/modconfig/open_graph.go +++ b/steampipeconfig/modconfig/open_graph.go @@ -7,9 +7,9 @@ import ( // OpenGraph is a struct representing the OpenGraph group mod resource type OpenGraph struct { // The opengraph description (og:description) of the mod, for use in social media applications - Description string `cty:"description" hcl:"description" json:"description"` + Description *string `cty:"description" hcl:"description" json:"description"` // The opengraph display title (og:title) of the mod, for use in social media applications. - Title string `cty:"title" hcl:"title" json:"title"` + Title *string `cty:"title" hcl:"title" json:"title"` Image *string `cty:"image" hcl:"image" json:"image"` DeclRange hcl.Range `json:"-"` } diff --git a/steampipeconfig/modconfig/panel.go b/steampipeconfig/modconfig/panel.go index 6db025b80..b5afb82e9 100644 --- a/steampipeconfig/modconfig/panel.go +++ b/steampipeconfig/modconfig/panel.go @@ -30,15 +30,17 @@ type Panel struct { DeclRange hcl.Range Mod *Mod `cty:"mod"` - parents []ModTreeItem - metadata *ResourceMetadata + parents []ModTreeItem + metadata *ResourceMetadata + UnqualifiedName string } func NewPanel(block *hcl.Block) *Panel { panel := &Panel{ - ShortName: block.Labels[0], - FullName: fmt.Sprintf("panel.%s", block.Labels[0]), - DeclRange: block.DefRange, + ShortName: block.Labels[0], + FullName: fmt.Sprintf("panel.%s", block.Labels[0]), + UnqualifiedName: fmt.Sprintf("panel.%s", block.Labels[0]), + DeclRange: block.DefRange, } return panel } @@ -85,11 +87,6 @@ func (p *Panel) Name() string { return p.FullName } -// QualifiedName returns the name in format: '.panel.' -func (p *Panel) QualifiedName() string { - return fmt.Sprintf("%s.%s", p.metadata.ModName, p.FullName) -} - // OnDecoded implements HclResource func (p *Panel) OnDecoded(*hcl.Block) hcl.Diagnostics { return nil } @@ -99,6 +96,8 @@ func (p *Panel) AddReference(*ResourceReference) {} // SetMod implements HclResource func (p *Panel) SetMod(mod *Mod) { p.Mod = mod + p.UnqualifiedName = p.FullName + p.FullName = fmt.Sprintf("%s.%s", mod.ShortName, p.FullName) } // GetMod implements HclResource diff --git a/steampipeconfig/modconfig/parse_names.go b/steampipeconfig/modconfig/parse_names.go index 4dec35e9c..6827ddc3d 100644 --- a/steampipeconfig/modconfig/parse_names.go +++ b/steampipeconfig/modconfig/parse_names.go @@ -57,6 +57,17 @@ func ParseResourceName(fullName string) (res *ParsedResourceName, err error) { return } +// UnqualifiedResourceName removes the mod prefix from the given name +func UnqualifiedResourceName(fullName string) string { + parts := strings.Split(fullName, ".") + switch len(parts) { + case 3: + return strings.Join(parts[1:], ".") + default: + return fullName + } +} + func ParseResourcePropertyPath(propertyPath string) (res *ParsedPropertyPath, err error) { res = &ParsedPropertyPath{} diff --git a/steampipeconfig/modconfig/plugin_version.go b/steampipeconfig/modconfig/plugin_version.go index 883df6cfc..645010abd 100644 --- a/steampipeconfig/modconfig/plugin_version.go +++ b/steampipeconfig/modconfig/plugin_version.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - goVersion "github.com/hashicorp/go-version" + "github.com/Masterminds/semver" "github.com/hashicorp/hcl/v2" "github.com/turbot/steampipe/ociinstaller" ) @@ -14,7 +14,7 @@ type PluginVersion struct { RawName string `cty:"name" hcl:"name,label"` // the version STREAM, can be either a major or minor version stream i.e. 1 or 1.1 VersionString string `cty:"version" hcl:"version,optional"` - Version *goVersion.Version + Version *semver.Version // the org and name which are parsed from the raw name Org string Name string @@ -39,7 +39,7 @@ func (p *PluginVersion) String() string { // Initialise parses the version and name properties func (p *PluginVersion) Initialise() hcl.Diagnostics { var diags hcl.Diagnostics - if version, err := goVersion.NewVersion(strings.TrimPrefix(p.VersionString, "v")); err != nil { + if version, err := semver.NewVersion(strings.TrimPrefix(p.VersionString, "v")); err != nil { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: fmt.Sprintf("invalid plugin version %s", p.VersionString), diff --git a/steampipeconfig/modconfig/prepared_statement.go b/steampipeconfig/modconfig/prepared_statement.go index b0e8ef78b..361c295cc 100644 --- a/steampipeconfig/modconfig/prepared_statement.go +++ b/steampipeconfig/modconfig/prepared_statement.go @@ -3,6 +3,7 @@ package modconfig import ( "fmt" "log" + "strings" "github.com/turbot/steampipe/utils" ) @@ -25,6 +26,8 @@ func GetPreparedStatementExecuteSQL(source QueryProvider, args *QueryArgs) (stri func preparedStatementName(source QueryProvider) string { var name, suffix string prefix := fmt.Sprintf("%s_", source.ModName()) + prefix = strings.Replace(prefix, ".", "_", -1) + prefix = strings.Replace(prefix, "@", "_", -1) // build suffix using a char to indicate control or query, and the truncated hash switch t := source.(type) { diff --git a/steampipeconfig/modconfig/query.go b/steampipeconfig/modconfig/query.go index 031166f5b..746b9cb1a 100644 --- a/steampipeconfig/modconfig/query.go +++ b/steampipeconfig/modconfig/query.go @@ -35,6 +35,7 @@ type Query struct { DeclRange hcl.Range PreparedStatementName string `column:"prepared_statement_name,text"` metadata *ResourceMetadata + UnqualifiedName string } func (q *Query) Equals(other *Query) bool { @@ -151,11 +152,6 @@ func (q *Query) Name() string { return q.FullName } -// QualifiedName returns the name in format: '.control.' -func (q *Query) QualifiedName() string { - return fmt.Sprintf("%s.%s", q.metadata.ModName, q.FullName) -} - // GetMetadata implements ResourceWithMetadata func (q *Query) GetMetadata() *ResourceMetadata { return q.metadata @@ -177,6 +173,8 @@ func (q *Query) AddReference(ref *ResourceReference) { // SetMod implements HclResource func (q *Query) SetMod(mod *Mod) { q.Mod = mod + q.UnqualifiedName = q.FullName + q.FullName = fmt.Sprintf("%s.%s", mod.ShortName, q.FullName) } // GetMod implements HclResource @@ -205,5 +203,5 @@ func (q *Query) GetPreparedStatementName() string { // ModName implements QueryProvider func (q *Query) ModName() string { - return q.Mod.ShortName + return q.Mod.NameWithVersion() } diff --git a/steampipeconfig/modconfig/report.go b/steampipeconfig/modconfig/report.go index fea400fa5..265551ee2 100644 --- a/steampipeconfig/modconfig/report.go +++ b/steampipeconfig/modconfig/report.go @@ -21,15 +21,17 @@ type Report struct { DeclRange hcl.Range - parents []ModTreeItem - metadata *ResourceMetadata + parents []ModTreeItem + metadata *ResourceMetadata + UnqualifiedName string } func NewReport(block *hcl.Block) *Report { report := &Report{ - ShortName: block.Labels[0], - FullName: fmt.Sprintf("report.%s", block.Labels[0]), - DeclRange: block.DefRange, + ShortName: block.Labels[0], + FullName: fmt.Sprintf("report.%s", block.Labels[0]), + UnqualifiedName: fmt.Sprintf("report.%s", block.Labels[0]), + DeclRange: block.DefRange, } return report } @@ -45,11 +47,6 @@ func (r *Report) Name() string { return r.FullName } -// QualifiedName returns the name in format: '.report.' -func (r *Report) QualifiedName() string { - return fmt.Sprintf("%s.%s", r.metadata.ModName, r.FullName) -} - // OnDecoded implements HclResource func (r *Report) OnDecoded(*hcl.Block) hcl.Diagnostics { return nil } @@ -61,6 +58,8 @@ func (r *Report) AddReference(*ResourceReference) { // SetMod implements HclResource func (r *Report) SetMod(mod *Mod) { r.Mod = mod + r.UnqualifiedName = r.FullName + r.FullName = fmt.Sprintf("%s.%s", mod.ShortName, r.FullName) } // GetMod implements HclResource diff --git a/steampipeconfig/modconfig/requires.go b/steampipeconfig/modconfig/requires.go index 3388382c9..9f51b9397 100644 --- a/steampipeconfig/modconfig/requires.go +++ b/steampipeconfig/modconfig/requires.go @@ -2,36 +2,37 @@ package modconfig import ( "fmt" + "sort" "strings" - goVersion "github.com/hashicorp/go-version" + "github.com/Masterminds/semver" "github.com/hashicorp/hcl/v2" "github.com/turbot/steampipe/version" ) -// Requires is a struct representing mod dependencies -type Requires struct { +// Require is a struct representing mod dependencies +type Require struct { SteampipeVersionString string `hcl:"steampipe,optional"` - SteampipeVersion *goVersion.Version - Plugins []*PluginVersion `hcl:"plugin,block"` - Mods []*ModVersion `hcl:"mod,block"` - DeclRange hcl.Range `json:"-"` + SteampipeVersion *semver.Version + Plugins []*PluginVersion `hcl:"plugin,block"` + Mods []*ModVersionConstraint `hcl:"mod,block"` + DeclRange hcl.Range `json:"-"` + // map keyed by name [and alias] + modMap map[string]*ModVersionConstraint } -func (r *Requires) ValidateSteampipeVersion(modName string) error { - if r.SteampipeVersion != nil { - if version.Version.LessThan(r.SteampipeVersion) { - return fmt.Errorf("steampipe version %s does not satisfy %s which requires version %s", version.String(), modName, r.SteampipeVersion.String()) - } - } - return nil +func newRequire() *Require { + r := &Require{} + r.initialise() + return r } -func (r *Requires) Initialise() hcl.Diagnostics { +func (r *Require) initialise() hcl.Diagnostics { var diags hcl.Diagnostics + r.modMap = make(map[string]*ModVersionConstraint) if r.SteampipeVersionString != "" { - steampipeVersion, err := goVersion.NewVersion(strings.TrimPrefix(r.SteampipeVersionString, "v")) + steampipeVersion, err := semver.NewVersion(strings.TrimPrefix(r.SteampipeVersionString, "v")) if err != nil { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, @@ -49,6 +50,82 @@ func (r *Requires) Initialise() hcl.Diagnostics { for _, m := range r.Mods { moreDiags := m.Initialise() diags = append(diags, moreDiags...) + if !diags.HasErrors() { + // key map entry by name [and alias] + r.modMap[m.Name] = m + } } return diags } + +func (r *Require) ValidateSteampipeVersion(modName string) error { + if r.SteampipeVersion != nil { + if version.SteampipeVersion.LessThan(r.SteampipeVersion) { + return fmt.Errorf("steampipe version %s does not satisfy %s which requires version %s", version.SteampipeVersion.String(), modName, r.SteampipeVersion.String()) + } + } + return nil +} + +// AddModDependencies adds all the mod in newModVersions to our list of mods, using the following logic +// - if a mod with same name, [alias] and constraint exists, it is not added +// - if a mod with same name [and alias] and different constraint exist, it is replaced +func (r *Require) AddModDependencies(newModVersions map[string]*ModVersionConstraint) { + // rebuild the Mods array + + // first rebuild the mod map + for name, newVersion := range newModVersions { + // todo take alias into account + r.modMap[name] = newVersion + } + + // now update the mod array from the map + var newMods = make([]*ModVersionConstraint, len(r.modMap)) + idx := 0 + for _, requiredVersion := range r.modMap { + newMods[idx] = requiredVersion + idx++ + } + // sort by name + sort.Sort(ModVersionConstraintCollection(newMods)) + // write back + r.Mods = newMods +} + +func (r *Require) RemoveModDependencies(versions map[string]*ModVersionConstraint) { + // first rebuild the mod map + for name := range versions { + // todo take alias into account + delete(r.modMap, name) + } + // now update the mod array from the map + var newMods = make([]*ModVersionConstraint, len(r.modMap)) + idx := 0 + for _, requiredVersion := range r.modMap { + newMods[idx] = requiredVersion + idx++ + } + // sort by name + sort.Sort(ModVersionConstraintCollection(newMods)) + // write back + r.Mods = newMods +} + +func (r *Require) RemoveAllModDependencies() { + r.Mods = nil +} + +func (r *Require) GetModDependency(name string /*,alias string*/) *ModVersionConstraint { + return r.modMap[name] +} + +func (r *Require) ContainsMod(requiredModVersion *ModVersionConstraint) bool { + if c := r.GetModDependency(requiredModVersion.Name); c != nil { + return c.Equals(requiredModVersion) + } + return false +} + +func (r *Require) Empty() bool { + return r.SteampipeVersion == nil && len(r.Mods) == 0 && len(r.Plugins) == 0 +} diff --git a/steampipeconfig/modconfig/variable.go b/steampipeconfig/modconfig/variable.go index 40c7d605b..6a5c34280 100644 --- a/steampipeconfig/modconfig/variable.go +++ b/steampipeconfig/modconfig/variable.go @@ -29,7 +29,8 @@ type Variable struct { ParsingMode var_config.VariableParsingMode Mod *Mod - metadata *ResourceMetadata + metadata *ResourceMetadata + UnqualifiedName string } func NewVariable(v *var_config.Variable) *Variable { @@ -58,11 +59,6 @@ func (v *Variable) Name() string { return v.FullName } -// QualifiedName returns the name in format: '.var.' -func (v *Variable) QualifiedName() string { - return fmt.Sprintf("%s.%s", v.metadata.ModName, v.FullName) -} - // GetMetadata implements ResourceWithMetadata func (v *Variable) GetMetadata() *ResourceMetadata { return v.metadata @@ -82,6 +78,8 @@ func (v *Variable) AddReference(*ResourceReference) {} // SetMod implements HclResource func (v *Variable) SetMod(mod *Mod) { v.Mod = mod + v.UnqualifiedName = v.FullName + v.FullName = fmt.Sprintf("%s.%s", mod.ShortName, v.FullName) } // GetMod implements HclResource diff --git a/steampipeconfig/parse/decode.go b/steampipeconfig/parse/decode.go index 1b31fe253..8011058c2 100644 --- a/steampipeconfig/parse/decode.go +++ b/steampipeconfig/parse/decode.go @@ -417,7 +417,6 @@ func decodeControl(block *hcl.Block, runCtx *RunContext) (*modconfig.Control, *d } return c, res - } func decodeControlArgs(attr *hcl.Attribute, evalCtx *hcl.EvalContext, controlName string) (*modconfig.QueryArgs, hcl.Diagnostics) { @@ -540,12 +539,6 @@ func decodeProperty(content *hcl.BodyContent, property string, dest interface{}, func handleDecodeResult(resource modconfig.HclResource, res *decodeResult, block *hcl.Block, runCtx *RunContext) hcl.Diagnostics { var diags hcl.Diagnostics if res.Success() { - // if resource supports metadata, save it - if resourceWithMetadata, ok := resource.(modconfig.ResourceWithMetadata); ok { - body := block.Body.(*hclsyntax.Body) - diags = addResourceMetadata(resourceWithMetadata, body.SrcRange, runCtx) - } - // if resource is NOT a mod, set mod pointer on hcl resource and add resource to current mod if _, ok := resource.(*modconfig.Mod); !ok { resource.SetMod(runCtx.CurrentMod) @@ -553,6 +546,13 @@ func handleDecodeResult(resource modconfig.HclResource, res *decodeResult, block moreDiags := runCtx.CurrentMod.AddResource(resource) diags = append(diags, moreDiags...) } + + // if resource supports metadata, save it + if resourceWithMetadata, ok := resource.(modconfig.ResourceWithMetadata); ok { + body := block.Body.(*hclsyntax.Body) + diags = addResourceMetadata(resourceWithMetadata, body.SrcRange, runCtx) + } + // add resource into the run context moreDiags := runCtx.AddResource(resource) diags = append(diags, moreDiags...) diff --git a/steampipeconfig/parse/installed_mod.go b/steampipeconfig/parse/installed_mod.go index 7f80446d3..ac3c82f7b 100644 --- a/steampipeconfig/parse/installed_mod.go +++ b/steampipeconfig/parse/installed_mod.go @@ -1,11 +1,11 @@ package parse import ( - goVersion "github.com/hashicorp/go-version" + "github.com/Masterminds/semver" "github.com/turbot/steampipe/steampipeconfig/modconfig" ) type InstalledMod struct { Mod *modconfig.Mod - Version *goVersion.Version + Version *semver.Version } diff --git a/steampipeconfig/parse/parser.go b/steampipeconfig/parse/parser.go index 1dfecf9e5..cc3c8a159 100644 --- a/steampipeconfig/parse/parser.go +++ b/steampipeconfig/parse/parser.go @@ -7,10 +7,10 @@ import ( "os" "path/filepath" - "github.com/hashicorp/hcl/v2/gohcl" - "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclparse" + "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/hashicorp/hcl/v2/json" "github.com/turbot/steampipe-plugin-sdk/plugin" "github.com/turbot/steampipe/constants" @@ -75,12 +75,12 @@ func ModfileExists(modPath string) bool { } // ParseModDefinition parses the modfile only -// it is expected the callign code wil lhave verified the existence of the modfile by calling ModfileExists +// it is expected the calling code will have verified the existence of the modfile by calling ModfileExists func ParseModDefinition(modPath string) (*modconfig.Mod, error) { // TODO think about variables // if there is no mod at this location, return error - modFilePath := filepath.Join(modPath, "mod.sp") + modFilePath := constants.ModFilePath(modPath) if _, err := os.Stat(modFilePath); os.IsNotExist(err) { return nil, fmt.Errorf("no mod file found in %s", modPath) } @@ -107,7 +107,11 @@ func ParseModDefinition(modPath string) (*modconfig.Mod, error) { for _, block := range content.Blocks { if block.Type == modconfig.BlockTypeMod { - mod := modconfig.NewMod(block.Labels[0], modPath, block.DefRange) + var defRange = block.DefRange + if hclBody, ok := block.Body.(*hclsyntax.Body); ok { + defRange = hclBody.SrcRange + } + mod := modconfig.NewMod(block.Labels[0], modPath, defRange) diags := gohcl.DecodeBody(block.Body, evalCtx, mod) if diags.HasErrors() { return nil, plugin.DiagsToError("Failed to decode mod hcl file", diags) @@ -156,7 +160,8 @@ func ParseMod(modPath string, fileData map[string][]byte, pseudoResources []modc addPseudoResourcesToMod(pseudoResources, hclResources, mod) // add this mod to run context - this it to ensure all pseudo resources get added - runCtx.AddMod(mod, content, fileData) + runCtx.SetDecodeContent(content, fileData) + runCtx.AddMod(mod) // perform initial decode to get dependencies // (if there are no dependencies, this is all that is needed) @@ -180,7 +185,7 @@ func ParseMod(modPath string, fileData map[string][]byte, pseudoResources []modc // now tell mod to build tree of controls. // NOTE: this also builds the sorted benchmark list - if err := mod.BuildResourceTree(); err != nil { + if err := mod.BuildResourceTree(runCtx.LoadedDependencyMods); err != nil { return nil, err } diff --git a/steampipeconfig/parse/run_context.go b/steampipeconfig/parse/run_context.go index 397bf9c22..becf1b90f 100644 --- a/steampipeconfig/parse/run_context.go +++ b/steampipeconfig/parse/run_context.go @@ -8,9 +8,9 @@ import ( "github.com/stevenle/topsort" filehelpers "github.com/turbot/go-kit/files" "github.com/turbot/go-kit/helpers" - "github.com/turbot/steampipe/constants" "github.com/turbot/steampipe/steampipeconfig/hclhelpers" "github.com/turbot/steampipe/steampipeconfig/modconfig" + "github.com/turbot/steampipe/steampipeconfig/version_map" "github.com/zclconf/go-cty/cty" ) @@ -35,7 +35,9 @@ type ReferenceTypeValueMap map[string]map[string]cty.Value type RunContext struct { // the mod which is currently being parsed - CurrentMod *modconfig.Mod + CurrentMod *modconfig.Mod + // the workspace lock data + WorkspaceLock *version_map.WorkspaceLock UnresolvedBlocks map[string]*unresolvedBlock FileData map[string][]byte // the eval context used to decode references in HCL @@ -44,8 +46,7 @@ type RunContext struct { Flags ParseModFlag ListOptions *filehelpers.ListOptions LoadedDependencyMods modconfig.ModMap - WorkspacePath string - ModInstallationPath string + RootEvalPath string // if set, only decode these blocks BlockTypes []string // if set, exclude these block types @@ -59,11 +60,11 @@ type RunContext struct { Variables map[string]*modconfig.Variable } -func NewRunContext(workspacePath string, flags ParseModFlag, listOptions *filehelpers.ListOptions) *RunContext { +func NewRunContext(workspaceLock *version_map.WorkspaceLock, rootEvalPath string, flags ParseModFlag, listOptions *filehelpers.ListOptions) *RunContext { c := &RunContext{ Flags: flags, - ModInstallationPath: constants.WorkspaceModPath(workspacePath), - WorkspacePath: workspacePath, + RootEvalPath: rootEvalPath, + WorkspaceLock: workspaceLock, ListOptions: listOptions, LoadedDependencyMods: make(modconfig.ModMap), UnresolvedBlocks: make(map[string]*unresolvedBlock), @@ -77,9 +78,20 @@ func NewRunContext(workspacePath string, flags ParseModFlag, listOptions *filehe // add enums to the variables which may be referenced from within the hcl c.addSteampipeEnums() c.buildEvalContext() + return c } +func (r *RunContext) EnsureWorkspaceLock(mod *modconfig.Mod) error { + // if the mod has dependencies, there must a workspace lock object in the run context + // (mod MUST be the workspace mod, not a dependency, as we would hit this error as soon as we parse it) + if mod.HasDependentMods() && (r.WorkspaceLock.Empty() || r.WorkspaceLock.Incomplete()) { + return fmt.Errorf("not all dependencies are installed - run 'steampipe mod install'") + } + + return nil +} + // VariableValueMap converts a map of variables to a map of the underlying cty value func VariableValueMap(variables map[string]*modconfig.Variable) map[string]cty.Value { ret := make(map[string]cty.Value, len(variables)) @@ -101,15 +113,12 @@ func (r *RunContext) AddVariables(inputVariables map[string]*modconfig.Variable) // AddMod is used to add a mod with any pseudo resources to the eval context // - in practice this will be a shell mod with just pseudo resources - other resources will be added as they are parsed -func (r *RunContext) AddMod(mod *modconfig.Mod, content *hcl.BodyContent, fileData map[string][]byte) hcl.Diagnostics { +func (r *RunContext) AddMod(mod *modconfig.Mod) hcl.Diagnostics { if len(r.UnresolvedBlocks) > 0 { // should never happen panic("calling SetContent on runContext but there are unresolved blocks from a previous parse") } - r.FileData = fileData - r.blocks = content.Blocks - var diags hcl.Diagnostics moreDiags := r.storeResourceInCtyMap(mod) @@ -119,20 +128,24 @@ func (r *RunContext) AddMod(mod *modconfig.Mod, content *hcl.BodyContent, fileDa moreDiags := r.storeResourceInCtyMap(q) diags = append(diags, moreDiags...) } - for _, q := range mod.Controls { - moreDiags := r.storeResourceInCtyMap(q) + for _, c := range mod.Controls { + moreDiags := r.storeResourceInCtyMap(c) diags = append(diags, moreDiags...) } - for _, q := range mod.Locals { - moreDiags := r.storeResourceInCtyMap(q) + for _, b := range mod.Benchmarks { + moreDiags := r.storeResourceInCtyMap(b) diags = append(diags, moreDiags...) } - for _, q := range mod.Reports { - moreDiags := r.storeResourceInCtyMap(q) + for _, l := range mod.Locals { + moreDiags := r.storeResourceInCtyMap(l) diags = append(diags, moreDiags...) } - for _, q := range mod.Panels { - moreDiags := r.storeResourceInCtyMap(q) + for _, rpt := range mod.Reports { + moreDiags := r.storeResourceInCtyMap(rpt) + diags = append(diags, moreDiags...) + } + for _, p := range mod.Panels { + moreDiags := r.storeResourceInCtyMap(p) diags = append(diags, moreDiags...) } @@ -141,6 +154,11 @@ func (r *RunContext) AddMod(mod *modconfig.Mod, content *hcl.BodyContent, fileDa return diags } +func (r *RunContext) SetDecodeContent(content *hcl.BodyContent, fileData map[string][]byte) { + r.blocks = content.Blocks + r.FileData = fileData +} + func (r *RunContext) ShouldIncludeBlock(block *hcl.Block) bool { if len(r.BlockTypes) > 0 && !helpers.StringSliceContains(r.BlockTypes, block.Type) { return false @@ -360,10 +378,11 @@ func (r *RunContext) buildEvalContext() { variables[mod] = cty.ObjectVal(refTypeMap) } - //create evaluation context + // create evaluation context r.EvalCtx = &hcl.EvalContext{ Variables: variables, - Functions: ContextFunctions(r.WorkspacePath), + // use the mod path as the file root for functions + Functions: ContextFunctions(r.RootEvalPath), } } @@ -411,11 +430,9 @@ func (r *RunContext) addReferenceValue(resource modconfig.HclResource, value cty typeString := parsedName.ItemType // the resource name will not have a mod - but the run context knows which mod we are parsing - mod := r.CurrentMod - modName := mod.ShortName - if mod.ModPath == r.WorkspacePath { + if mod.ModPath == r.RootEvalPath { modName = "local" } variablesForMod, ok := r.referenceValues[modName] diff --git a/steampipeconfig/test_data/demo/control_demo/mod.sp b/steampipeconfig/test_data/demo/control_demo/mod.sp index cf1da12f6..220957046 100644 --- a/steampipeconfig/test_data/demo/control_demo/mod.sp +++ b/steampipeconfig/test_data/demo/control_demo/mod.sp @@ -13,7 +13,7 @@ mod "m1" { description = "CIS reports, queries, and actions for AWS. Open source CLI. No DB required." } // -// requires { +// require { // steampipe ">0.3.0" {} // // plugin "aws" {} diff --git a/steampipeconfig/test_data/dep_test2/mod.sp b/steampipeconfig/test_data/dep_test2/mod.sp index e8b2b974a..30805a431 100644 --- a/steampipeconfig/test_data/dep_test2/mod.sp +++ b/steampipeconfig/test_data/dep_test2/mod.sp @@ -11,7 +11,7 @@ mod "m1" { labels = ["public cloud", "aws"] # dependencies - requires { + require { steampipe = ">0.3.0" plugin "aws" {} diff --git a/steampipeconfig/version_map/dependency_version_map.go b/steampipeconfig/version_map/dependency_version_map.go new file mode 100644 index 000000000..456c19b2c --- /dev/null +++ b/steampipeconfig/version_map/dependency_version_map.go @@ -0,0 +1,103 @@ +package version_map + +import ( + "github.com/Masterminds/semver" + "github.com/turbot/steampipe/steampipeconfig/modconfig" + "github.com/xlab/treeprint" +) + +type DependencyVersionMap map[string]ResolvedVersionMap + +// Add adds a dependency to the list of items installed for the given parent +func (m DependencyVersionMap) Add(dependencyName string, dependencyVersion *semver.Version, constraintString string, parentName string) { + // get the map for this parent + parentItems := m[parentName] + // create if needed + if parentItems == nil { + parentItems = make(ResolvedVersionMap) + } + // add the dependency + parentItems.Add(dependencyName, &ResolvedVersionConstraint{dependencyName, dependencyVersion, constraintString}) + // save + m[parentName] = parentItems +} + +// FlatMap converts the DependencyVersionMap into a ResolvedVersionMap, keyed by full name +func (m DependencyVersionMap) FlatMap() ResolvedVersionMap { + res := make(ResolvedVersionMap) + for _, deps := range m { + for _, dep := range deps { + res[modconfig.ModVersionFullName(dep.Name, dep.Version)] = dep + } + } + return res +} + +func (m DependencyVersionMap) GetDependencyTree(rootName string) treeprint.Tree { + tree := treeprint.NewWithRoot(rootName) + m.buildTree(rootName, tree) + return tree +} + +func (m DependencyVersionMap) buildTree(name string, tree treeprint.Tree) { + deps := m[name] + for name, version := range deps { + fullName := modconfig.ModVersionFullName(name, version.Version) + child := tree.AddBranch(fullName) + // if there are children add them + m.buildTree(fullName, child) + } +} + +// GetMissingFromOther returns a map of dependencies which exit in this map but not 'other' +func (m DependencyVersionMap) GetMissingFromOther(other DependencyVersionMap) DependencyVersionMap { + res := make(DependencyVersionMap) + for parent, deps := range m { + otherDeps := other[parent] + if otherDeps == nil { + otherDeps = make(ResolvedVersionMap) + } + for name, dep := range deps { + if _, ok := otherDeps[name]; !ok { + res.Add(dep.Name, dep.Version, dep.Constraint, parent) + } + } + } + return res +} + +func (m DependencyVersionMap) GetUpgradedInOther(other DependencyVersionMap) DependencyVersionMap { + res := make(DependencyVersionMap) + for parent, deps := range m { + otherDeps := other[parent] + if otherDeps == nil { + otherDeps = make(ResolvedVersionMap) + } + for name, dep := range deps { + if otherDep, ok := otherDeps[name]; ok { + if otherDep.Version.GreaterThan(dep.Version) { + res.Add(otherDep.Name, otherDep.Version, otherDep.Constraint, parent) + } + } + } + } + return res +} + +func (m DependencyVersionMap) GetDowngradedInOther(other DependencyVersionMap) DependencyVersionMap { + res := make(DependencyVersionMap) + for parent, deps := range m { + otherDeps := other[parent] + if otherDeps == nil { + otherDeps = make(ResolvedVersionMap) + } + for name, dep := range deps { + if otherDep, ok := otherDeps[name]; ok { + if otherDep.Version.LessThan(dep.Version) { + res.Add(otherDep.Name, otherDep.Version, otherDep.Constraint, parent) + } + } + } + } + return res +} diff --git a/steampipeconfig/version_map/resolved_version_constraint.go b/steampipeconfig/version_map/resolved_version_constraint.go new file mode 100644 index 000000000..d01ac788e --- /dev/null +++ b/steampipeconfig/version_map/resolved_version_constraint.go @@ -0,0 +1,20 @@ +package version_map + +import "github.com/Masterminds/semver" + +type ResolvedVersionConstraint struct { + Name string + // Alias string + Version *semver.Version + Constraint string +} + +func (c ResolvedVersionConstraint) Equals(other *ResolvedVersionConstraint) bool { + return c.Name == other.Name && + c.Version.Equal(other.Version) && + c.Constraint == other.Constraint +} + +func (c ResolvedVersionConstraint) IsPrerelease() bool { + return c.Version.Prerelease() != "" || c.Version.Metadata() != "" +} diff --git a/steampipeconfig/version_map/resolved_version_list_map.go b/steampipeconfig/version_map/resolved_version_list_map.go new file mode 100644 index 000000000..7bf08731c --- /dev/null +++ b/steampipeconfig/version_map/resolved_version_list_map.go @@ -0,0 +1,49 @@ +package version_map + +import ( + "github.com/turbot/steampipe/steampipeconfig/modconfig" +) + +// ResolvedVersionListMap represents a map of ResolvedVersionConstraint arrays, keyed by dependency name +type ResolvedVersionListMap map[string][]*ResolvedVersionConstraint + +// Add appends the version constraint to the list for the given name +func (m ResolvedVersionListMap) Add(name string, versionConstraint *ResolvedVersionConstraint) { + // if there is already an entry for the same name, replace it + // TODO handle alias + m[name] = []*ResolvedVersionConstraint{versionConstraint} +} + +// Remove removes the given version constraint from the list for the given name +func (m ResolvedVersionListMap) Remove(name string, constraint *ResolvedVersionConstraint) { + var res []*ResolvedVersionConstraint + for _, c := range m[name] { + if !c.Equals(constraint) { + res = append(res, c) + } + } + m[name] = res +} + +// FlatMap converts the ResolvedVersionListMap map into a map keyed by the FULL dependency name (i.e. including version( +func (m ResolvedVersionListMap) FlatMap() map[string]*ResolvedVersionConstraint { + var res = make(map[string]*ResolvedVersionConstraint) + for name, versions := range m { + for _, version := range versions { + key := modconfig.ModVersionFullName(name, version.Version) + res[key] = version + } + } + return res +} + +// FlatNames converts the ResolvedVersionListMap map into a string array of full names +func (m ResolvedVersionListMap) FlatNames() []string { + var res []string + for name, versions := range m { + for _, version := range versions { + res = append(res, modconfig.ModVersionFullName(name, version.Version)) + } + } + return res +} diff --git a/steampipeconfig/version_map/resolved_version_map.go b/steampipeconfig/version_map/resolved_version_map.go new file mode 100644 index 000000000..d932475a9 --- /dev/null +++ b/steampipeconfig/version_map/resolved_version_map.go @@ -0,0 +1,21 @@ +package version_map + +// ResolvedVersionMap represents a map of ResolvedVersionConstraint, keyed by dependency name +type ResolvedVersionMap map[string]*ResolvedVersionConstraint + +func (m ResolvedVersionMap) Add(name string, constraint *ResolvedVersionConstraint) { + m[name] = constraint +} + +func (m ResolvedVersionMap) Remove(name string) { + delete(m, name) +} + +// ToVersionListMap converts this map into a ResolvedVersionListMap +func (m ResolvedVersionMap) ToVersionListMap() ResolvedVersionListMap { + res := make(ResolvedVersionListMap, len(m)) + for k, v := range m { + res.Add(k, v) + } + return res +} diff --git a/steampipeconfig/version_map/version_constraint_map.go b/steampipeconfig/version_map/version_constraint_map.go new file mode 100644 index 000000000..1ac9f99bc --- /dev/null +++ b/steampipeconfig/version_map/version_constraint_map.go @@ -0,0 +1,5 @@ +package version_map + +import "github.com/turbot/steampipe/steampipeconfig/modconfig" + +type VersionConstraintMap map[string]*modconfig.ModVersionConstraint diff --git a/steampipeconfig/version_map/version_list_map.go b/steampipeconfig/version_map/version_list_map.go new file mode 100644 index 000000000..562faf898 --- /dev/null +++ b/steampipeconfig/version_map/version_list_map.go @@ -0,0 +1,31 @@ +package version_map + +import ( + "sort" + + "github.com/Masterminds/semver" + "github.com/turbot/steampipe/steampipeconfig/modconfig" +) + +// VersionListMap is a map keyed by dependency name storing a list of versions for each dependency +type VersionListMap map[string]semver.Collection + +func (i VersionListMap) Add(name string, version *semver.Version) { + versions := append(i[name], version) + // reverse sort the versions + sort.Sort(sort.Reverse(versions)) + i[name] = versions + +} + +// FlatMap converts the VersionListMap map into a bool map keyed by qualified dependency name +func (m VersionListMap) FlatMap() map[string]bool { + var res = make(map[string]bool) + for name, versions := range m { + for _, version := range versions { + key := modconfig.ModVersionFullName(name, version) + res[key] = true + } + } + return res +} diff --git a/steampipeconfig/version_map/version_map.go b/steampipeconfig/version_map/version_map.go new file mode 100644 index 000000000..7789952e2 --- /dev/null +++ b/steampipeconfig/version_map/version_map.go @@ -0,0 +1,8 @@ +package version_map + +import ( + "github.com/Masterminds/semver" +) + +// VersionMap represents a map of semver versions, keyed by dependency name +type VersionMap map[string]*semver.Version diff --git a/steampipeconfig/version_map/workspace_lock.go b/steampipeconfig/version_map/workspace_lock.go new file mode 100644 index 000000000..90aa6caef --- /dev/null +++ b/steampipeconfig/version_map/workspace_lock.go @@ -0,0 +1,294 @@ +package version_map + +import ( + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + + "github.com/Masterminds/semver" + filehelpers "github.com/turbot/go-kit/files" + "github.com/turbot/go-kit/helpers" + "github.com/turbot/steampipe/constants" + "github.com/turbot/steampipe/steampipeconfig/modconfig" + "github.com/turbot/steampipe/utils" + "github.com/turbot/steampipe/version_helpers" +) + +// WorkspaceLock is a map of ModVersionMaps items keyed by the parent mod whose dependencies are installed +type WorkspaceLock struct { + WorkspacePath string + InstallCache DependencyVersionMap + MissingVersions DependencyVersionMap + + ModInstallationPath string + installedMods VersionListMap +} + +// EmptyWorkspaceLock creates a new empty workspace lock based, +// sharing workspace path and installedMods with 'existingLock' +func EmptyWorkspaceLock(existingLock *WorkspaceLock) *WorkspaceLock { + return &WorkspaceLock{ + WorkspacePath: existingLock.WorkspacePath, + ModInstallationPath: constants.WorkspaceModPath(existingLock.WorkspacePath), + InstallCache: make(DependencyVersionMap), + MissingVersions: make(DependencyVersionMap), + installedMods: existingLock.installedMods, + } +} + +func LoadWorkspaceLock(workspacePath string) (*WorkspaceLock, error) { + var installCache = make(DependencyVersionMap) + lockPath := constants.WorkspaceLockPath(workspacePath) + if helpers.FileExists(lockPath) { + + fileContent, err := os.ReadFile(lockPath) + if err != nil { + log.Printf("[TRACE] error reading %s: %s\n", lockPath, err.Error()) + return nil, err + } + err = json.Unmarshal(fileContent, &installCache) + if err != nil { + log.Printf("[TRACE] failed to unmarshal %s: %s\n", lockPath, err.Error()) + return nil, nil + } + } + res := &WorkspaceLock{ + WorkspacePath: workspacePath, + ModInstallationPath: constants.WorkspaceModPath(workspacePath), + InstallCache: installCache, + MissingVersions: make(DependencyVersionMap), + } + + if err := res.getInstalledMods(); err != nil { + return nil, err + } + + // populate the MissingVersions + // (this removes missing items from the install cache) + res.setMissing() + + return res, nil +} + +// populate MissingVersions and UnreferencedVersions +func (l *WorkspaceLock) validate() error { + + return nil +} + +// getInstalledMods returns a map installed mods, and the versions installed for each +func (l *WorkspaceLock) getInstalledMods() error { + // recursively search for all the mod.sp files under the .steampipe/mods folder, then build the mod name from the file path + modFiles, err := filehelpers.ListFiles(l.ModInstallationPath, &filehelpers.ListOptions{ + Flags: filehelpers.FilesRecursive, + Include: []string{"**/mod.sp"}, + }) + if err != nil { + return err + } + + // create result map - a list of version for each mod + installedMods := make(VersionListMap, len(modFiles)) + // collect errors + var errors []error + + for _, modfilePath := range modFiles { + // try to parse the mon name and version form the parent folder of the modfile + modName, version, err := l.parseModPath(modfilePath) + if err != nil { + // if we fail to parse, just ignore this modfile + // - it's parent is not a valid mod installation folder so it is probably a child folder of a mod + continue + } + // add this mod version to the map + installedMods.Add(modName, version) + } + + if len(errors) > 0 { + return utils.CombineErrors(errors...) + } + l.installedMods = installedMods + return nil +} + +// GetUnreferencedMods returns a map of all installed mods which are not in the lock file +func (l *WorkspaceLock) GetUnreferencedMods() VersionListMap { + var unreferencedVersions = make(VersionListMap) + for name, versions := range l.installedMods { + for _, version := range versions { + if !l.ContainsModVersion(name, version) { + unreferencedVersions.Add(name, version) + } + } + } + return unreferencedVersions +} + +// identify mods which are in InstallCache but not installed +// move them from InstallCache into MissingVersions +func (l *WorkspaceLock) setMissing() { + // create a map of full modname to bool to allow simple checking + flatInstalled := l.installedMods.FlatMap() + + for parent, deps := range l.InstallCache { + // deps is a map of dep name to resolved contraint list + // flatten and iterate + + for name, resolvedConstraint := range deps { + fullName := modconfig.ModVersionFullName(name, resolvedConstraint.Version) + + if !flatInstalled[fullName] { + // get the mod name from the constraint (fullName includes the version) + name := resolvedConstraint.Name + // remove this item from the install cache and add into missing + l.MissingVersions.Add(name, resolvedConstraint.Version, resolvedConstraint.Constraint, parent) + l.InstallCache[parent].Remove(name) + } + } + } +} + +// extract the mod name and version from the modfile path +func (l *WorkspaceLock) parseModPath(modfilePath string) (modName string, modVersion *semver.Version, err error) { + modFullName, err := filepath.Rel(l.ModInstallationPath, filepath.Dir(modfilePath)) + if err != nil { + return + } + return modconfig.ParseModFullName(modFullName) +} + +func (l *WorkspaceLock) Save() error { + if len(l.InstallCache) == 0 { + // ignore error + l.Delete() + return nil + } + content, err := json.MarshalIndent(l.InstallCache, "", " ") + if err != nil { + return err + } + return os.WriteFile(constants.WorkspaceLockPath(l.WorkspacePath), content, 0644) +} + +// Delete deletes the lock file +func (l *WorkspaceLock) Delete() error { + return os.Remove(constants.WorkspaceLockPath(l.WorkspacePath)) +} + +// DeleteMods removes mods from the lock file then, if it is empty, deletes the file +func (l *WorkspaceLock) DeleteMods(mods VersionConstraintMap, parent *modconfig.Mod) { + for modName := range mods { + if parentDependencies := l.InstallCache[parent.GetModDependencyPath()]; parentDependencies != nil { + parentDependencies.Remove(modName) + } + } +} + +// GetMod looks for a lock file entry matching the given mod name +func (l *WorkspaceLock) GetMod(modName string, parent *modconfig.Mod) *ResolvedVersionConstraint { + if parentDependencies := l.InstallCache[parent.GetModDependencyPath()]; parentDependencies != nil { + // look for this mod in the lock file entries for this parent + return parentDependencies[modName] + } + return nil +} + +// GetLockedModVersions builds a ResolvedVersionListMap with the resolved versions +// for each item of the given VersionConstraintMap found in the lock file +func (l *WorkspaceLock) GetLockedModVersions(mods VersionConstraintMap, parent *modconfig.Mod) (ResolvedVersionListMap, error) { + var res = make(ResolvedVersionListMap) + for name, constraint := range mods { + resolvedConstraint, err := l.GetLockedModVersion(constraint, parent) + if err != nil { + return nil, err + } + if resolvedConstraint != nil { + res.Add(name, resolvedConstraint) + } + } + return res, nil +} + +// GetLockedModVersion looks for a lock file entry matching the required constraint and returns nil if not found +func (l *WorkspaceLock) GetLockedModVersion(requiredModVersion *modconfig.ModVersionConstraint, parent *modconfig.Mod) (*ResolvedVersionConstraint, error) { + lockedVersion := l.GetMod(requiredModVersion.Name, parent) + if lockedVersion == nil { + return nil, nil + } + + // verify the locked version satisfies the version constraint + if !requiredModVersion.Constraint.Check(lockedVersion.Version) { + return nil, nil + } + + return lockedVersion, nil +} + +// EnsureLockedModVersion looks for a lock file entry matching the required mod name +func (l *WorkspaceLock) EnsureLockedModVersion(requiredModVersion *modconfig.ModVersionConstraint, parent *modconfig.Mod) (*ResolvedVersionConstraint, error) { + lockedVersion := l.GetMod(requiredModVersion.Name, parent) + if lockedVersion == nil { + return nil, nil + } + + // verify the locked version satisfies the version constraint + if !requiredModVersion.Constraint.Check(lockedVersion.Version) { + return nil, fmt.Errorf("failed to resolvedependencies for %s - locked version %s does not meet the constraint %s", parent.GetModDependencyPath(), modconfig.ModVersionFullName(requiredModVersion.Name, lockedVersion.Version), requiredModVersion.Constraint.Original) + } + + return lockedVersion, nil +} + +// GetLockedModVersionConstraint looks for a lock file entry matching the required mod version and if found, +// returns it in the form of a ModVersionConstraint +func (l *WorkspaceLock) GetLockedModVersionConstraint(requiredModVersion *modconfig.ModVersionConstraint, parent *modconfig.Mod) (*modconfig.ModVersionConstraint, error) { + lockedVersion, err := l.EnsureLockedModVersion(requiredModVersion, parent) + if err != nil { + // EnsureLockedModVersion returns an error if the locked version does not satisfy the requirement + return nil, err + } + if lockedVersion == nil { + // EnsureLockedModVersion returns nil if no locked version is found + return nil, nil + } + // create a new ModVersionConstraint using the locked version + lockedVersionFullName := modconfig.ModVersionFullName(requiredModVersion.Name, lockedVersion.Version) + return modconfig.NewModVersionConstraint(lockedVersionFullName) +} + +// ContainsModVersion returns whether the lockfile contains the given mod version +func (l *WorkspaceLock) ContainsModVersion(modName string, modVersion *semver.Version) bool { + for _, modVersionMap := range l.InstallCache { + for lockName, lockVersion := range modVersionMap { + // TODO consider handling of metadata + if lockName == modName && lockVersion.Version.Equal(modVersion) && lockVersion.Version.Metadata() == modVersion.Metadata() { + return true + } + } + } + return false +} + +func (l *WorkspaceLock) ContainsModConstraint(modName string, constraint *version_helpers.Constraints) bool { + for _, modVersionMap := range l.InstallCache { + for lockName, lockVersion := range modVersionMap { + if lockName == modName && lockVersion.Constraint == constraint.Original { + return true + } + } + } + return false +} + +// Incomplete returned whether there are any missing dependencies +// (i.e. they exist in the lock file but ate not installed) +func (l *WorkspaceLock) Incomplete() bool { + return len(l.MissingVersions) > 0 +} + +// Empty returns whether the install cache is empty +func (l *WorkspaceLock) Empty() bool { + return len(l.InstallCache) == 0 +} diff --git a/steampipeconfig/version_map/workspace_lock_list.go b/steampipeconfig/version_map/workspace_lock_list.go new file mode 100644 index 000000000..00ba0c534 --- /dev/null +++ b/steampipeconfig/version_map/workspace_lock_list.go @@ -0,0 +1,10 @@ +package version_map + +func (l *WorkspaceLock) GetModList(rootName string) string { + if len(l.InstallCache) == 0 { + return "No mods installed" + } + + tree := l.InstallCache.GetDependencyTree(rootName) + return tree.String() +} diff --git a/task/version_checker.go b/task/version_checker.go index 725a150ac..c8b9ac095 100644 --- a/task/version_checker.go +++ b/task/version_checker.go @@ -8,8 +8,8 @@ import ( "net/url" "os" + SemVer "github.com/Masterminds/semver" "github.com/fatih/color" - SemVer "github.com/hashicorp/go-version" "github.com/olekukonko/tablewriter" "github.com/spf13/viper" "github.com/turbot/steampipe/constants" @@ -18,7 +18,7 @@ import ( ) // the current version of the Steampipe CLI application -var currentVersion = version.String() +var currentVersion = version.SteampipeVersion.String() type versionCheckResponse struct { NewVersion string `json:"latest_version,omitempty"` // `json:"current_version"` diff --git a/tests/acceptance/test_data/mod_install/mod-install.txt b/tests/acceptance/test_data/mod_install/mod-install.txt new file mode 100644 index 000000000..954c4c1a2 --- /dev/null +++ b/tests/acceptance/test_data/mod_install/mod-install.txt @@ -0,0 +1 @@ +This is a folder used for acceptance tests. \ No newline at end of file diff --git a/tests/acceptance/test_files/021.mod-install.bats b/tests/acceptance/test_files/021.mod-install.bats new file mode 100644 index 000000000..c43428178 --- /dev/null +++ b/tests/acceptance/test_files/021.mod-install.bats @@ -0,0 +1,80 @@ +load "$LIB_BATS_ASSERT/load.bash" +load "$LIB_BATS_SUPPORT/load.bash" + +@test "list with no mods installed" { + run steampipe mod list + assert_output 'No mods installed' +} + +@test "install latest" { + run steampipe mod install github.com/turbot/steampipe-mod-aws-compliance + assert_output --partial 'Installed 1 mod: + +local +└── github.com/turbot/steampipe-mod-aws-compliance' + # need the check the version from mod.sp file as well +} + +@test "install latest and then run install" { + steampipe mod install github.com/turbot/steampipe-mod-aws-compliance + run steampipe mod install + assert_output 'All mods are up to date' +} + +@test "install mod and list" { + steampipe mod install github.com/turbot/steampipe-mod-aws-compliance@0.10 + run steampipe mod list + assert_output ' +local +└── github.com/turbot/steampipe-mod-aws-compliance@v0.10' +} + +@test "install old version when latest already installed" { + steampipe mod install github.com/turbot/steampipe-mod-aws-compliance + run steampipe mod install github.com/turbot/steampipe-mod-aws-compliance@0.1 + assert_output ' +Downgraded 1 mod: + +local +└── github.com/turbot/steampipe-mod-aws-compliance@v0.1' +} + +@test "install mod version, remove .steampipe folder and then run install" { + # install particular mod version, remove .steampipe folder and run mod install + steampipe mod install github.com/turbot/steampipe-mod-aws-compliance@0.1 + rm -rf .steampipe + run steampipe mod install + + # should install the same cached version + # better message + assert_output ' +Installed 1 mod: + +local +└── github.com/turbot/steampipe-mod-aws-compliance@v0.1' +} + +@test "install mod version, remove .cache file and then run install" { + # install particular mod version, remove .mod.cache.json file and run mod install + steampipe mod install github.com/turbot/steampipe-mod-aws-compliance@0.1 + rm -rf .mod.cache.json + run steampipe mod install + + # should install the same cached version + # better message + assert_output ' +Installed 1 mod: + +local +└── github.com/turbot/steampipe-mod-aws-compliance@v0.1' +} + +function teardown() { + rm -rf .steampipe/ + rm -rf .mod.cache.json + rm -rf mod.sp +} + +function setup() { + cd $FILE_PATH/test_data/mod_install +} diff --git a/utils/useragent.go b/utils/useragent.go index 6097050a2..65bd7299e 100644 --- a/utils/useragent.go +++ b/utils/useragent.go @@ -17,13 +17,13 @@ import ( const httpTimeout = 5 * time.Second func getUserAgent() string { - return fmt.Sprintf("Turbot Steampipe/%s (+https://steampipe.io)", version.String()) + return fmt.Sprintf("Turbot Steampipe/%s (+https://steampipe.io)", version.SteampipeVersion.String()) } // BuildRequestPayload :: merges the provided payload with the standard payload that needs to be sent func BuildRequestPayload(signature string, payload map[string]interface{}) *bytes.Buffer { requestPayload := map[string]interface{}{ - "version": version.String(), + "version": version.SteampipeVersion.String(), "os_platform": runtime.GOOS, "arch": runtime.GOARCH, "signature": signature, diff --git a/version/version.go b/version/version.go index 9ec422623..083fa719e 100644 --- a/version/version.go +++ b/version/version.go @@ -8,7 +8,7 @@ package version import ( "fmt" - goVersion "github.com/hashicorp/go-version" + "github.com/Masterminds/semver" ) /** @@ -24,22 +24,18 @@ var steampipeVersion = "0.11.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -var prerelease = "dev.0" +var prerelease = "dev.6" -// Version is an instance of version.Version. This has the secondary +// SteampipeVersion is an instance of semver.Version. This has the secondary // benefit of verifying during tests and init time that our version is a // proper semantic version, which should always be the case. -var Version *goVersion.Version +var SteampipeVersion *semver.Version func init() { versionString := steampipeVersion if prerelease != "" { versionString = fmt.Sprintf("%s-%s", steampipeVersion, prerelease) } - Version = goVersion.Must(goVersion.NewVersion(versionString)) -} + SteampipeVersion = semver.MustParse(versionString) -// String returns the complete version string, including prerelease -func String() string { - return Version.String() } diff --git a/version_helpers/constraints.go b/version_helpers/constraints.go new file mode 100644 index 000000000..936022883 --- /dev/null +++ b/version_helpers/constraints.go @@ -0,0 +1,46 @@ +package version_helpers + +import ( + "github.com/Masterminds/semver" +) + +// Constraints wraps semver.Constraints type, adding the Original property +type Constraints struct { + constraint *semver.Constraints + Original string +} + +func NewConstraint(c string) (*Constraints, error) { + constraints, err := semver.NewConstraint(c) + if err != nil { + return nil, err + } + return &Constraints{ + constraint: constraints, + Original: c, + }, nil +} + +// Check tests if a version satisfies the constraints. +func (c Constraints) Check(v *semver.Version) bool { + return c.constraint.Check(v) +} + +// Validate checks if a version satisfies a constraint. If not a slice of +// reasons for the failure are returned in addition to a bool. +func (c Constraints) Validate(v *semver.Version) (bool, []error) { + return c.constraint.Validate(v) +} + +func (c Constraints) Equals(other *Constraints) bool { + return c.Original == other.Original +} + +// IsPrerelease determines whether the constraint parses as a specifc version with prerelease or metadata set +func (c Constraints) IsPrerelease() bool { + v, err := semver.NewVersion(c.Original) + if err != nil { + return false + } + return v.Prerelease() != "" || v.Metadata() != "" +} diff --git a/workspace/workspace.go b/workspace/workspace.go index ccbfc6c84..2690e44e8 100644 --- a/workspace/workspace.go +++ b/workspace/workspace.go @@ -11,7 +11,6 @@ import ( "github.com/fsnotify/fsnotify" filehelpers "github.com/turbot/go-kit/files" - "github.com/turbot/go-kit/types" typehelpers "github.com/turbot/go-kit/types" "github.com/turbot/steampipe/constants" "github.com/turbot/steampipe/db/db_common" @@ -19,6 +18,7 @@ import ( "github.com/turbot/steampipe/steampipeconfig" "github.com/turbot/steampipe/steampipeconfig/modconfig" "github.com/turbot/steampipe/steampipeconfig/parse" + "github.com/turbot/steampipe/steampipeconfig/version_map" "github.com/turbot/steampipe/utils" ) @@ -140,7 +140,7 @@ func (w *Workspace) SetOnFileWatcherEventMessages(f func()) { } // access functions -// NOTE: all access functions lock 'loadLock' - this is to avoid conflicts with th efile watcher +// NOTE: all access functions lock 'loadLock' - this is to avoid conflicts with the file watcher func (w *Workspace) Close() { if w.watcher != nil { @@ -219,33 +219,6 @@ func (w *Workspace) GetResourceMaps() *modconfig.WorkspaceResourceMaps { return workspaceMap } -// GetMod attempts to return the mod with a name matching 'modName' -// It first checks the workspace mod, then checks all mod dependencies -func (w *Workspace) GetMod(modName string) *modconfig.Mod { - // is it the workspace mod? - if modName == w.Mod.Name() { - return w.Mod - } - // try workspace mod dependencies - return w.Mods[modName] -} - -// ModList returns a flat list of all mods - the workspace mod and depenfency mods -func (w *Workspace) ModList() []*modconfig.Mod { - var res = []*modconfig.Mod{w.Mod} - for _, m := range w.Mods { - res = append(res, m) - } - return res -} - -// SaveWorkspaceMod searialises the workspace mode to and long name: .control. and long name: .benchmark. and full name: .benchmark. and long name: .benchmark.