jprdonnelly/steampipe
1
0
Fork 0
mirror of https://github.com/turbot/steampipe.git synced 2025-12-19 09:58:53 -05:00
Code Issues Packages Projects Releases Wiki Activity

Update mod require, deprecate steampipe property and add steampipe block. Deprecate plugin version property and add min_version`. Closes #3347. Fix args passed to dependency mods failing to resolve if they reference variables. Closes #3348

Browse Source
This commit is contained in:
kaidaguerre
2023-04-19 12:28:53 +01:00
committed by GitHub
parent a6919cca75
commit f47985f88f
56 changed files with 636 additions and 520 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/release_cli_and_assets.yml vendored
View File

@@ -327,6 +327,7 @@ jobs:
- "cache"
- "mod_install"
- "mod"
- "mod_require"
- "check"
- "performance"
- "workspace"

1
.github/workflows/test.yml vendored
View File

@@ -100,6 +100,7 @@ jobs:
- "cache"
- "mod_install"
- "mod"
- "mod_require"
- "check"
- "performance"
- "workspace"

1
go.mod
View File

@@ -4,7 +4,6 @@ go 1.19
require (
github.com/Machiel/slugify v1.0.1
github.com/Masterminds/semver v1.5.0
github.com/Masterminds/semver/v3 v3.2.1
github.com/alecthomas/chroma v0.10.0
github.com/bgentry/speakeasy v0.1.0

1
go.sum
View File

@@ -85,7 +85,6 @@ github.com/Machiel/slugify v1.0.1/go.mod h1:fTFGn5uWEynW4CUMG7sWkYXOf1UgDxyTM3Db
github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=

2
pkg/constants/config_keys.go
View File

@@ -3,7 +3,7 @@ package constants
// viper config keys
const (
// ConfigKeyDatabaseSearchPath is used to store the search path set in the database config in viper
// the viper value will be set via via a call to getScopedKey in steampipeconfig/steampipeconfig.go
// the viper value will be set via a call to getScopedKey in steampipeconfig/steampipeconfig.go
ConfigKeyDatabaseSearchPath = "database.search-path"
ConfigKeyInteractive = "interactive"
ConfigKeyActiveCommand = "cmd"

2
pkg/modinstaller/git.go
View File

@@ -4,7 +4,7 @@ import (
"fmt"
"sort"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/storage/memory"

2
pkg/modinstaller/helpers.go
View File

@@ -1,7 +1,7 @@
package modinstaller
import (
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
"github.com/turbot/steampipe/pkg/versionhelpers"
)

2
pkg/modinstaller/install_data.go
View File

@@ -3,7 +3,7 @@ package modinstaller
import (
"fmt"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
"github.com/turbot/steampipe/pkg/steampipeconfig/versionmap"
"github.com/turbot/steampipe/pkg/versionhelpers"

36
pkg/modinstaller/mod_installer.go
View File

@@ -8,10 +8,12 @@ import (
"path"
"path/filepath"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
git "github.com/go-git/go-git/v5"
"github.com/hashicorp/hcl/v2"
"github.com/otiai10/copy"
"github.com/spf13/viper"
sdkplugin "github.com/turbot/steampipe-plugin-sdk/v5/plugin"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/error_helpers"
"github.com/turbot/steampipe/pkg/filepaths"
@@ -21,6 +23,7 @@ import (
"github.com/turbot/steampipe/pkg/steampipeconfig/versionmap"
"github.com/turbot/steampipe/pkg/utils"
"github.com/turbot/steampipe/sperr"
"github.com/zclconf/go-cty/cty"
)
type ModInstaller struct {
@@ -170,6 +173,11 @@ func (i *ModInstaller) UninstallWorkspaceDependencies(ctx context.Context) error
func (i *ModInstaller) InstallWorkspaceDependencies(ctx context.Context) (err error) {
workspaceMod := i.workspaceMod
defer func() {
if err != nil && i.force {
// suppress the error since this is a forced install
log.Println("[TRACE] suppressing error in InstallWorkspaceDependencies because force is enabled", err)
err = nil
}
// tidy unused mods
// (put in defer so it still gets called in case of errors)
if viper.GetBool(constants.ArgPrune) && !i.dryRun {
@@ -181,15 +189,11 @@ func (i *ModInstaller) InstallWorkspaceDependencies(ctx context.Context) (err er
}
}()
if !i.force {
// there's no point in checking the requirements if force is set
// since we will ignore it anyway
if err := workspaceMod.Require.ValidateSteampipeVersion(workspaceMod.Name()); err != nil {
return err
}
if err := workspaceMod.Require.ValidatePluginVersions(workspaceMod.Name(), i.installedPlugins); err != nil {
return err
}
if err := workspaceMod.Require.ValidateSteampipeVersion(workspaceMod.Name()); err != nil {
return err
}
if err := workspaceMod.Require.ValidatePluginVersions(workspaceMod.Name(), i.installedPlugins); err != nil {
return err
}
// if mod args have been provided, add them to the the workspace mod requires
@@ -578,9 +582,15 @@ func (i *ModInstaller) loadModfile(ctx context.Context, modPath string, createDe
return nil, nil
}
mod, err := parse.ParseModDefinition(modPath)
if err != nil {
return nil, err
// build an eval context just containing functions
evalCtx := &hcl.EvalContext{
Functions: parse.ContextFunctions(modPath),
Variables: make(map[string]cty.Value),
}
mod, res := parse.ParseModDefinition(modPath, evalCtx)
if res.Diags.HasErrors() {
return nil, sdkplugin.DiagsToError("Failed to load mod", res.Diags)
}
return mod, nil

2
pkg/modinstaller/mod_installer_test.go
View File

@@ -4,7 +4,7 @@ import (
"fmt"
"testing"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
)
func TestModInstaller(t *testing.T) {

2
pkg/modinstaller/resolved_mod_ref.go
View File

@@ -1,7 +1,7 @@
package modinstaller
import (
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
"github.com/go-git/go-git/v5/plumbing"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
"github.com/turbot/steampipe/pkg/versionhelpers"

2
pkg/plugin/installed.go
View File

@@ -3,7 +3,7 @@ package plugin
import (
"fmt"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
"github.com/turbot/steampipe/pkg/ociinstaller"
)

46
pkg/steampipeconfig/hclhelpers/hcl_blocks.go Normal file
View File

@@ -0,0 +1,46 @@
package hclhelpers
import (
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
)
func GetFirstBlockOfType(blocks hcl.Blocks, blockType string) *hcl.Block {
for _, block := range blocks {
if block.Type == blockType {
return block
}
}
return nil
}
func FindChildBlocks(parentBlock *hcl.Block, blockType string) hcl.Blocks {
var res hcl.Blocks
childBlocks := parentBlock.Body.(*hclsyntax.Body).Blocks
for _, b := range childBlocks {
if b.Type == blockType {
res = append(res, b.AsHCLBlock())
}
}
return res
}
func FindFirstChildBlock(parentBlock *hcl.Block, blockType string) *hcl.Block {
childBlocks := FindChildBlocks(parentBlock, blockType)
if len(childBlocks) == 0 {
return nil
}
return childBlocks[0]
}
// BlocksToMap convert an array of blocks to a map keyed by block laabel
// NOTE: this panics if any blocks do not have a label
func BlocksToMap(blocks hcl.Blocks) map[string]*hcl.Block {
res := make(map[string]*hcl.Block, len(blocks))
for _, b := range blocks {
if len(b.Labels) == 0 {
panic("all blocks passed to BlocksToMap must have a label")
}
res[b.Labels[0]] = b
}
return res
}

2
pkg/steampipeconfig/inputvars/collect_variables.go
View File

@@ -150,7 +150,7 @@ func CollectVariableValuesFromModRequire(mod *modconfig.Mod, parseCtx *parse.Mod
return nil, err
}
if depMod == nil {
return nil, fmt.Errorf("depency mod %s is not loaded", depMod.Name())
return nil, fmt.Errorf("dependency mod %s is not loaded", depMod.Name())
}
if args := depModConstraint.Args; args != nil {

40
pkg/steampipeconfig/load_mod.go
View File

@@ -7,7 +7,7 @@ import (
"path/filepath"
"strings"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
filehelpers "github.com/turbot/go-kit/files"
"github.com/turbot/go-kit/helpers"
"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
@@ -21,16 +21,16 @@ import (
// if CreatePseudoResources flag is set, construct hcl resources for files with specific extensions
// NOTE: it is an error if there is more than 1 mod defined, however zero mods is acceptable
// - a default mod will be created assuming there are any resource files
func LoadMod(modPath string, parseCtx *parse.ModParseContext, opts ...LoadModOption) (mod *modconfig.Mod, errAndWarnings *modconfig.ErrorAndWarnings) {
func LoadMod(modPath string, parseCtx *parse.ModParseContext, opts ...LoadModOption) (mod *modconfig.Mod, errorsAndWarnings *modconfig.ErrorAndWarnings) {
defer func() {
if r := recover(); r != nil {
errAndWarnings = modconfig.NewErrorsAndWarning(helpers.ToError(r))
errorsAndWarnings = modconfig.NewErrorsAndWarning(helpers.ToError(r))
}
}()
mod, err := loadModDefinition(modPath, parseCtx)
if err != nil {
return nil, modconfig.NewErrorsAndWarning(err)
mod, loadModResult := loadModDefinition(modPath, parseCtx)
if loadModResult.Error != nil {
return nil, loadModResult
}
// apply opts to mod
@@ -49,34 +49,41 @@ func LoadMod(modPath string, parseCtx *parse.ModParseContext, opts ...LoadModOpt
// populate the resource maps of the current mod using the dependency mods
mod.ResourceMaps = parseCtx.GetResourceMaps()
// now load the mod resource hcl
return loadModResources(modPath, parseCtx)
mod, errorsAndWarnings = loadModResources(modPath, parseCtx)
// add in any warnings from mod load
errorsAndWarnings.AddWarning(loadModResult.Warnings...)
return mod, errorsAndWarnings
}
func loadModDefinition(modPath string, parseCtx *parse.ModParseContext) (*modconfig.Mod, error) {
var mod *modconfig.Mod
func loadModDefinition(modPath string, parseCtx *parse.ModParseContext) (mod *modconfig.Mod, errorsAndWarnings *modconfig.ErrorAndWarnings) {
errorsAndWarnings = &modconfig.ErrorAndWarnings{}
// verify the mod folder exists
_, err := os.Stat(modPath)
if os.IsNotExist(err) {
return nil, fmt.Errorf("mod folder %s does not exist", modPath)
return nil, modconfig.NewErrorsAndWarning(fmt.Errorf("mod folder %s does not exist", modPath))
}
if parse.ModfileExists(modPath) {
// load the mod definition to get the dependencies
mod, err = parse.ParseModDefinition(modPath)
if err != nil {
return nil, err
var res *parse.DecodeResult
mod, res = parse.ParseModDefinition(modPath, parseCtx.EvalCtx)
errorsAndWarnings = modconfig.DiagsToErrorsAndWarnings("mod load failed", res.Diags)
if res.Diags.HasErrors() {
return nil, errorsAndWarnings
}
} else {
// so there is no mod file - should we create a default?
if !parseCtx.ShouldCreateDefaultMod() {
errorsAndWarnings.Error = fmt.Errorf("mod folder %s does not contain a mod resource definition", modPath)
// ShouldCreateDefaultMod flag NOT set - fail
return nil, fmt.Errorf("mod folder %s does not contain a mod resource definition", modPath)
return nil, errorsAndWarnings
}
// just create a default mod
mod = modconfig.CreateDefaultMod(modPath)
}
return mod, nil
return mod, errorsAndWarnings
}
func loadModDependencies(mod *modconfig.Mod, parseCtx *parse.ModParseContext) error {
@@ -90,7 +97,6 @@ func loadModDependencies(mod *modconfig.Mod, parseCtx *parse.ModParseContext) er
}
for _, requiredModVersion := range mod.Require.Mods {
// have we already loaded a mod which satisfied this
loadedMod, err := parseCtx.GetLoadedDependencyMod(requiredModVersion, mod)
if err != nil {
@@ -131,7 +137,7 @@ func loadModDependency(modDependency *modconfig.ModVersionConstraint, parseCtx *
defer func() { parseCtx.ListOptions.Exclude = prevExclusions }()
childParseCtx := parse.NewChildModParseContext(parseCtx, dependencyDir)
// NOTE: pass in the version and dependency path of the mod - these must be set before it loads its depdencies
// NOTE: pass in the version and dependency path of the mod - these must be set before it loads its dependencies
mod, errAndWarnings := LoadMod(dependencyDir, childParseCtx, WithDependencyConfig(modDependency.Name, version))
if errAndWarnings.GetError() != nil {
return errAndWarnings.GetError()

2
pkg/steampipeconfig/load_mod_option.go
View File

@@ -2,7 +2,7 @@ package steampipeconfig
import (
"fmt"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
)

2
pkg/steampipeconfig/modconfig/benchmark.go
View File

@@ -59,7 +59,7 @@ func (b *Benchmark) Equals(other *Benchmark) bool {
}
// OnDecoded implements HclResource
func (b *Benchmark) OnDecoded(block *hcl.Block, resourceMapProvider ResourceMapsProvider) hcl.Diagnostics {
func (b *Benchmark) OnDecoded(block *hcl.Block, _ ResourceMapsProvider) hcl.Diagnostics {
b.setBaseProperties()
return nil
}

6
pkg/steampipeconfig/modconfig/block_type.go
View File

@@ -4,7 +4,11 @@ import "github.com/turbot/go-kit/helpers"
// NOTE: when adding a block type, be sure to update QueryProviderBlocks/ReferenceBlocks/AllBlockTypes as needed
const (
BlockTypeMod = "mod"
// require blocks
BlockTypeSteampipe = "steampipe"
BlockTypeMod = "mod"
BlockTypePlugin = "plugin"
// resource blocks
BlockTypeQuery = "query"
BlockTypeControl = "control"
BlockTypeBenchmark = "benchmark"

2
pkg/steampipeconfig/modconfig/dashboard.go
View File

@@ -139,7 +139,7 @@ func (d *Dashboard) Equals(other *Dashboard) bool {
}
// OnDecoded implements HclResource
func (d *Dashboard) OnDecoded(block *hcl.Block, resourceMapProvider ResourceMapsProvider) hcl.Diagnostics {
func (d *Dashboard) OnDecoded(block *hcl.Block, _ ResourceMapsProvider) hcl.Diagnostics {
d.setBaseProperties()
d.ChildNames = make([]string, len(d.children))

2
pkg/steampipeconfig/modconfig/dashboard_category.go
View File

@@ -49,7 +49,7 @@ func NewDashboardCategory(block *hcl.Block, mod *Mod, shortName string) HclResou
}
// OnDecoded implements HclResource
func (c *DashboardCategory) OnDecoded(block *hcl.Block, resourceMapProvider ResourceMapsProvider) hcl.Diagnostics {
func (c *DashboardCategory) OnDecoded(block *hcl.Block, _ ResourceMapsProvider) hcl.Diagnostics {
c.setBaseProperties()
// populate properties map
if len(c.PropertyList) > 0 {

2
pkg/steampipeconfig/modconfig/dashboard_container.go
View File

@@ -55,7 +55,7 @@ func (c *DashboardContainer) Equals(other *DashboardContainer) bool {
}
// OnDecoded implements HclResource
func (c *DashboardContainer) OnDecoded(block *hcl.Block, resourceMapProvider ResourceMapsProvider) hcl.Diagnostics {
func (c *DashboardContainer) OnDecoded(block *hcl.Block, _ ResourceMapsProvider) hcl.Diagnostics {
c.ChildNames = make([]string, len(c.children))
for i, child := range c.children {
c.ChildNames[i] = child.Name()

2
pkg/steampipeconfig/modconfig/dashboard_with.go
View File

@@ -44,7 +44,7 @@ func (w *DashboardWith) Equals(other *DashboardWith) bool {
}
// OnDecoded implements HclResource
func (w *DashboardWith) OnDecoded(_ *hcl.Block, resourceMapProvider ResourceMapsProvider) hcl.Diagnostics {
func (w *DashboardWith) OnDecoded(_ *hcl.Block, _ ResourceMapsProvider) hcl.Diagnostics {
return nil
}

2
pkg/steampipeconfig/modconfig/hcl_resource_impl.go
View File

@@ -43,7 +43,7 @@ func (b *HclResourceImpl) GetUnqualifiedName() string {
}
// OnDecoded implements HclResource
func (b *HclResourceImpl) OnDecoded(block *hcl.Block, resourceMapProvider ResourceMapsProvider) hcl.Diagnostics {
func (b *HclResourceImpl) OnDecoded(block *hcl.Block, _ ResourceMapsProvider) hcl.Diagnostics {
return nil
}

46
pkg/steampipeconfig/modconfig/mod.go
View File

@@ -6,7 +6,7 @@ import (
"path/filepath"
"strings"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/hashicorp/hcl/v2/hclwrite"
@@ -33,7 +33,7 @@ type Mod struct {
Icon *string `cty:"icon" hcl:"icon" column:"icon,text"`
// blocks
Require *Require
Require *Require `hcl:"require,block"`
LegacyRequire *Require `hcl:"requires,block"`
OpenGraph *OpenGraph `hcl:"opengraph,block" column:"open_graph,jsonb"`
@@ -62,7 +62,6 @@ type Mod struct {
}
func NewMod(shortName, modPath string, defRange hcl.Range) *Mod {
require := NewRequire()
name := fmt.Sprintf("mod.%s", shortName)
mod := &Mod{
ModTreeItemImpl: ModTreeItemImpl{
@@ -75,7 +74,7 @@ func NewMod(shortName, modPath string, defRange hcl.Range) *Mod {
},
},
ModPath: modPath,
Require: require,
Require: NewRequire(),
}
mod.ResourceMaps = NewModResources(mod)
@@ -150,7 +149,7 @@ func (m *Mod) GetPaths() []NodePath {
func (m *Mod) SetPaths() {}
// OnDecoded implements HclResource
func (m *Mod) OnDecoded(block *hcl.Block, resourceMapProvider ResourceMapsProvider) hcl.Diagnostics {
func (m *Mod) OnDecoded(block *hcl.Block, _ ResourceMapsProvider) hcl.Diagnostics {
// handle legacy requires block
if m.LegacyRequire != nil && !m.LegacyRequire.Empty() {
// ensure that both 'require' and 'requires' were not set
@@ -170,16 +169,8 @@ func (m *Mod) OnDecoded(block *hcl.Block, resourceMapProvider ResourceMapsProvid
if m.Require == nil {
return nil
}
err := m.Require.initialise()
if err != nil {
return hcl.Diagnostics{&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: err.Error(),
Subject: &block.DefRange,
}}
}
return nil
return m.Require.initialise(block)
}
// AddReference implements ResourceWithMetadata (overridden from ResourceWithMetadataImpl)
@@ -272,10 +263,12 @@ func (m *Mod) Save() error {
}
// require
if require := m.Require; require != nil && !m.Require.Empty() {
if require := m.Require; require != nil && !require.Empty() {
requiresBody := modBody.AppendNewBlock("require", nil).Body()
if require.SteampipeVersionString != "" {
requiresBody.SetAttributeValue("steampipe", cty.StringVal(require.SteampipeVersionString))
if require.Steampipe != nil && require.Steampipe.MinVersionString != "" {
steampipeRequiresBody := requiresBody.AppendNewBlock("steampipe", nil).Body()
steampipeRequiresBody.SetAttributeValue("min_version", cty.StringVal(require.Steampipe.MinVersionString))
}
if len(require.Plugins) > 0 {
pluginValues := make([]cty.Value, len(require.Plugins))
@@ -357,7 +350,7 @@ func (m *Mod) ValidatePluginVersions(availablePlugins map[string]*semver.Version
if m.Require == nil {
return nil
}
return m.Require.ValidatePluginVersions(m.DependencyName, availablePlugins)
return m.Require.ValidatePluginVersions(m.GetInstallCacheKey(), availablePlugins)
}
// CtyValue implements CtyValueProvider
@@ -387,3 +380,20 @@ func (m *Mod) SetDependencyConfig(dependencyPath string) error {
m.Version = version
return nil
}
// RequireHasUnresolvedArgs returns whether the mod has any mod requirements which have unresolved args
// (this could be because the arg refers to a variable, meanin gwe need an additional parse phase
// to resolve the arg values)
func (m *Mod) RequireHasUnresolvedArgs() bool {
if m.Require == nil {
return false
}
for _, m := range m.Require.Mods {
for _, a := range m.Args {
if !a.IsKnown() {
return true
}
}
}
return false
}

2
pkg/steampipeconfig/modconfig/mod_name.go
View File

@@ -4,7 +4,7 @@ import (
"fmt"
"strings"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
)
// BuildModDependencyPath converts a mod dependency name of form github.com/turbot/steampipe-mod-m2

80
pkg/steampipeconfig/modconfig/mod_version_constraint.go
View File

@@ -19,18 +19,15 @@ type ModVersionConstraint struct {
Name string `cty:"name" hcl:"name,label"`
VersionString string `cty:"version" hcl:"version"`
// variable values to be set on the dependency mod
Args map[string]cty.Value `cty:"args"`
Args map[string]cty.Value `cty:"args" hcl:"args,optional"`
// only one of Constraint, Branch and FilePath will be set
Constraint *versionhelpers.Constraints
// // NOTE: aliases will be supported in the future
//Alias string `cty:"alias" hcl:"alias"`
// the branch to use
Branch string
// the local file location to use
FilePath string
DeclRange hcl.Range
}
// NewModVersionConstraint creates a new ModVersionConstraint - this is called when installing a mod
func NewModVersionConstraint(modFullName string) (*ModVersionConstraint, error) {
m := &ModVersionConstraint{
Args: make(map[string]cty.Value),
@@ -52,13 +49,45 @@ func NewModVersionConstraint(modFullName string) (*ModVersionConstraint, error)
}
// try to convert version into a semver constraint
if err := m.Initialise(); err != nil {
if err := m.Initialise(nil); err != nil {
return nil, err
}
return m, nil
}
func (m *ModVersionConstraint) FullName() string {
// Initialise parses the version and name properties
func (m *ModVersionConstraint) Initialise(block *hcl.Block) hcl.Diagnostics {
if block != nil {
m.DeclRange = block.DefRange
}
if strings.HasPrefix(m.Name, filePrefix) {
m.setFilePath()
return nil
}
// now default the version string to latest
if m.VersionString == "" || m.VersionString == "latest" {
m.VersionString = "*"
}
// does the version parse as a semver version
if c, err := versionhelpers.NewConstraint(m.VersionString); err == nil {
// no error
m.Constraint = c
return nil
}
// so there was an error
return hcl.Diagnostics{&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("invalid mod version %s", m.VersionString),
Subject: &m.DeclRange,
}}
}
func (m *ModVersionConstraint) DependencyPath() string {
if m.HasVersion() {
return fmt.Sprintf("%s@%s", m.Name, m.VersionString)
}
@@ -72,42 +101,7 @@ func (m *ModVersionConstraint) HasVersion() bool {
}
func (m *ModVersionConstraint) String() string {
return m.FullName()
}
// Initialise parses the version and name properties
func (m *ModVersionConstraint) Initialise() hcl.Diagnostics {
if strings.HasPrefix(m.Name, filePrefix) {
m.setFilePath()
return nil
}
var diags hcl.Diagnostics
if m.VersionString == "" {
m.Constraint, _ = versionhelpers.NewConstraint("*")
m.VersionString = "latest"
return diags
}
if m.VersionString == "latest" {
m.Constraint, _ = versionhelpers.NewConstraint("*")
return diags
}
// does the version parse as a semver version
if c, err := versionhelpers.NewConstraint(m.VersionString); err == nil {
// no error
m.Constraint = c
return diags
}
// todo handle branch and commit hash
// so there was an error
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("invalid mod version %s", m.VersionString),
Subject: &m.DeclRange,
})
return diags
return m.DependencyPath()
}
func (m *ModVersionConstraint) setFilePath() {

49
pkg/steampipeconfig/modconfig/plugin_version.go
View File

@@ -4,7 +4,7 @@ import (
"fmt"
"strings"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
"github.com/hashicorp/hcl/v2"
"github.com/turbot/steampipe/pkg/ociinstaller"
)
@@ -12,20 +12,22 @@ import (
type PluginVersion struct {
// the plugin name, as specified in the mod requires block. , e.g. turbot/mod1, aws
RawName string `cty:"name" hcl:"name,label"`
// the version STREAM, can be either a major or minor version stream i.e. 1 or 1.1
// deprecated: use MinVersionString
VersionString string `cty:"version" hcl:"version,optional"`
Version *semver.Version
// the minumum version which satisfies the requirement
MinVersionString string `cty:"min_version" hcl:"min_version,optional"`
Constraint *semver.Constraints
// the org and name which are parsed from the raw name
Org string
Name string
DeclRange hcl.Range `json:"-"`
DeclRange hcl.Range
}
func (p *PluginVersion) FullName() string {
if p.VersionString == "" {
if p.MinVersionString == "" {
return p.ShortName()
}
return fmt.Sprintf("%s@%s", p.ShortName(), p.VersionString)
return fmt.Sprintf("%s@%s", p.ShortName(), p.MinVersionString)
}
func (p *PluginVersion) ShortName() string {
@@ -37,18 +39,41 @@ func (p *PluginVersion) String() string {
}
// Initialise parses the version and name properties
func (p *PluginVersion) Initialise() hcl.Diagnostics {
func (p *PluginVersion) Initialise(block *hcl.Block) hcl.Diagnostics {
var diags hcl.Diagnostics
if version, err := semver.NewVersion(strings.TrimPrefix(p.VersionString, "v")); err != nil {
p.DeclRange = block.DefRange
// handle deprecation warnings/errors
if p.VersionString != "" {
if p.MinVersionString != "" {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Both 'min_version' and deprecated 'version' property are set",
Subject: &p.DeclRange,
})
return diags
}
// raise deprecation warning
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("invalid plugin version %s", p.VersionString),
Severity: hcl.DiagWarning,
Summary: fmt.Sprintf("Property 'version' is deprecated - use 'version instead, in plugin '%s' require block", p.Name),
Subject: &p.DeclRange,
})
} else {
p.Version = version
// copy into new property
p.MinVersionString = p.VersionString
}
// convert min version into constraint (including prereleases)
minVersion, err := semver.NewVersion(strings.TrimPrefix(p.MinVersionString, "v"))
if err == nil {
p.Constraint, err = semver.NewConstraint(fmt.Sprintf(">=%s-0", minVersion))
}
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("Invalid plugin version %s", p.MinVersionString),
Subject: &p.DeclRange,
})
}
// parse plugin name
p.Org, p.Name, _ = ociinstaller.NewSteampipeImageRef(p.RawName).GetOrgNameAndStream()

2
pkg/steampipeconfig/modconfig/query_provider_impl.go
View File

@@ -173,7 +173,7 @@ func (q *QueryProviderImpl) getBaseImpl() *QueryProviderImpl {
return q.base.(QueryProvider).GetQueryProviderImpl()
}
func (q *QueryProviderImpl) OnDecoded(block *hcl.Block, resourceMapProvider ResourceMapsProvider) hcl.Diagnostics {
func (q *QueryProviderImpl) OnDecoded(block *hcl.Block, _ ResourceMapsProvider) hcl.Diagnostics {
q.populateQueryName()
return nil

109
pkg/steampipeconfig/modconfig/require.go
View File

@@ -3,26 +3,25 @@ package modconfig
import (
"fmt"
"sort"
"strings"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
"github.com/hashicorp/hcl/v2"
"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
"github.com/turbot/steampipe/pkg/error_helpers"
"github.com/turbot/steampipe/pkg/ociinstaller"
"github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers"
"github.com/turbot/steampipe/pkg/version"
"github.com/turbot/steampipe/sperr"
)
// Require is a struct representing mod dependencies
type Require struct {
SteampipeVersion *semver.Version
Plugins []*PluginVersion `hcl:"plugin,block"`
Plugins []*PluginVersion `hcl:"plugin,block"`
DeprecatedSteampipeVersionString string `hcl:"steampipe,optional"`
Steampipe *SteampipeRequire `hcl:"steampipe,block"`
Mods []*ModVersionConstraint `hcl:"mod,block"`
DeclRange hcl.Range
// map keyed by name [and alias]
SteampipeVersionString string `hcl:"steampipe,optional"`
Mods []*ModVersionConstraint
modMap map[string]*ModVersionConstraint
DeclRange hcl.Range
modMap map[string]*ModVersionConstraint
}
func NewRequire() *Require {
@@ -31,44 +30,89 @@ func NewRequire() *Require {
}
}
func (r *Require) initialise() error {
func (r *Require) initialise(modBlock *hcl.Block) hcl.Diagnostics {
// This will actually be called twice - once when we load the mod definition,
// and again when we load the mod resources (and set the mod metadata, references etc)
// If we have already initialised, return (we can tell by checking the DeclRange)
if !r.DeclRange.Empty() {
return nil
}
var diags hcl.Diagnostics
// handle deprecated properties
moreDiags := r.handleDeprecations()
diags = append(diags, moreDiags...)
// find the require block
requireBlock := hclhelpers.FindFirstChildBlock(modBlock, BlockTypeRequire)
if requireBlock == nil {
// if none was specified, fall back to parent block
requireBlock = modBlock
}
// build maps of plugin and mod blocks
pluginBlockMap := hclhelpers.BlocksToMap(hclhelpers.FindChildBlocks(requireBlock, BlockTypePlugin))
modBlockMap := hclhelpers.BlocksToMap(hclhelpers.FindChildBlocks(requireBlock, BlockTypeMod))
// set our DecRange
r.DeclRange = requireBlock.DefRange
r.modMap = make(map[string]*ModVersionConstraint)
if r.SteampipeVersionString != "" {
steampipeVersion, err := semver.NewVersion(strings.TrimPrefix(r.SteampipeVersionString, "v"))
if err != nil {
return fmt.Errorf("invalid required steampipe version %s", r.SteampipeVersionString)
}
r.SteampipeVersion = steampipeVersion
if r.Steampipe != nil {
moreDiags := r.Steampipe.initialise(requireBlock)
diags = append(diags, moreDiags...)
}
for _, p := range r.Plugins {
moreDiags := p.Initialise()
moreDiags := p.Initialise(pluginBlockMap[p.RawName])
diags = append(diags, moreDiags...)
}
for _, m := range r.Mods {
moreDiags := m.Initialise()
moreDiags := m.Initialise(modBlockMap[m.Name])
diags = append(diags, moreDiags...)
if !diags.HasErrors() {
// key map entry by name [and alias]
r.modMap[m.Name] = m
}
}
return plugin.DiagsToError("failed to initialise Require struct", diags)
return diags
}
func (r *Require) handleDeprecations() hcl.Diagnostics {
var diags hcl.Diagnostics
// the 'steampipe' property is deprecated and replace with a steampipe block
if r.DeprecatedSteampipeVersionString != "" {
// if there is both a steampipe block and property, fail
if r.Steampipe != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Both 'steampipe' block and deprecated 'steampipe' property are set",
Subject: &r.DeclRange,
})
} else {
r.Steampipe = &SteampipeRequire{MinVersionString: r.DeprecatedSteampipeVersionString}
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagWarning,
Summary: "Property 'steampipe' is deprecated for mod require block - use a steampipe block instead",
Subject: &r.DeclRange,
},
)
}
}
return diags
}
func (r *Require) ValidateSteampipeVersion(modName string) error {
if r.SteampipeVersion != nil {
if version.SteampipeVersion.LessThan(r.SteampipeVersion) {
return fmt.Errorf("steampipe version %s does not satisfy %s which requires version %s", version.SteampipeVersion.String(), modName, r.SteampipeVersion.String())
if steampipeVersionConstraint := r.SteampipeVersionConstraint(); steampipeVersionConstraint != nil {
if !steampipeVersionConstraint.Check(version.SteampipeVersion) {
return fmt.Errorf("steampipe version %s does not satisfy %s which requires version %s", version.SteampipeVersion.String(), modName, r.Steampipe.MinVersionString)
}
}
return nil
}
// validates that for every plugin requirement there's at least one plugin installed
// ValidatePluginVersions validates that for every plugin requirement there's at least one plugin installed
func (r *Require) ValidatePluginVersions(modName string, plugins map[string]*semver.Version) error {
if len(r.Plugins) == 0 {
return nil
@@ -89,11 +133,12 @@ func (r *Require) searchInstalledPluginForRequirement(modName string, requiremen
// no point check - different plugin
continue
}
if requirement.Version.LessThan(installed) || requirement.Version.Equal(installed) {
if requirement.Constraint.Check(installed) {
// constraint is satisfied
return nil
}
}
return sperr.New("could not find plugin which satisfies requirement '%s@%s' in '%s'", requirement.RawName, requirement.VersionString, modName)
return sperr.New("could not find plugin which satisfies requirement '%s@%s' - required by '%s'", requirement.RawName, requirement.MinVersionString, modName)
}
// AddModDependencies adds all the mod in newModVersions to our list of mods, using the following logic
@@ -104,7 +149,6 @@ func (r *Require) AddModDependencies(newModVersions map[string]*ModVersionConstr
// first rebuild the mod map
for name, newVersion := range newModVersions {
// todo take alias into account
r.modMap[name] = newVersion
}
@@ -124,7 +168,6 @@ func (r *Require) AddModDependencies(newModVersions map[string]*ModVersionConstr
func (r *Require) RemoveModDependencies(versions map[string]*ModVersionConstraint) {
// first rebuild the mod map
for name := range versions {
// todo take alias into account
delete(r.modMap, name)
}
// now update the mod array from the map
@@ -156,5 +199,13 @@ func (r *Require) ContainsMod(requiredModVersion *ModVersionConstraint) bool {
}
func (r *Require) Empty() bool {
return r.SteampipeVersion == nil && len(r.Mods) == 0 && len(r.Plugins) == 0
return r.SteampipeVersionConstraint() == nil && len(r.Mods) == 0 && len(r.Plugins) == 0
}
func (r *Require) SteampipeVersionConstraint() *semver.Constraints {
if r.Steampipe == nil {
return nil
}
return r.Steampipe.Constraint
}

46
pkg/steampipeconfig/modconfig/steampipe_require.go Normal file
View File

@@ -0,0 +1,46 @@
package modconfig
import (
"fmt"
"strings"
"github.com/Masterminds/semver/v3"
"github.com/hashicorp/hcl/v2"
"github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers"
)
type SteampipeRequire struct {
MinVersionString string `hcl:"min_version,optional"`
Constraint *semver.Constraints
DeclRange hcl.Range
}
func (r *SteampipeRequire) initialise(requireBlock *hcl.Block) hcl.Diagnostics {
// find the steampipe block
steampipeBlock := hclhelpers.FindFirstChildBlock(requireBlock, BlockTypeSteampipe)
if steampipeBlock == nil {
// can happen if there is a legacy property - just use the parent block
steampipeBlock = requireBlock
}
// set DeclRange
r.DeclRange = steampipeBlock.DefRange
if r.MinVersionString == "" {
return nil
}
// convert min version into constraint (including prereleases)
minVersion, err := semver.NewVersion(strings.TrimPrefix(r.MinVersionString, "v"))
if err == nil {
r.Constraint, err = semver.NewConstraint(fmt.Sprintf(">=%s-0", minVersion))
}
if err != nil {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("invalid required steampipe version %s", r.MinVersionString),
Subject: &r.DeclRange,
}}
}
return nil
}

2
pkg/steampipeconfig/modconfig/variable.go
View File

@@ -77,7 +77,7 @@ func (v *Variable) Equals(other *Variable) bool {
}
// OnDecoded implements HclResource
func (v *Variable) OnDecoded(block *hcl.Block, resourceMapProvider ResourceMapsProvider) hcl.Diagnostics {
func (v *Variable) OnDecoded(block *hcl.Block, _ ResourceMapsProvider) hcl.Diagnostics {
return nil
}

60
pkg/steampipeconfig/parse/decode.go
View File

@@ -88,7 +88,7 @@ func shouldAddToMod(resource modconfig.HclResource, block *hcl.Block, parseCtx *
}
// special case decode logic for locals
func decodeLocalsBlock(block *hcl.Block, parseCtx *ModParseContext) ([]modconfig.HclResource, *decodeResult) {
func decodeLocalsBlock(block *hcl.Block, parseCtx *ModParseContext) ([]modconfig.HclResource, *DecodeResult) {
var resources []modconfig.HclResource
var res = newDecodeResult()
@@ -116,7 +116,7 @@ func decodeLocalsBlock(block *hcl.Block, parseCtx *ModParseContext) ([]modconfig
return resources, res
}
func decodeBlock(block *hcl.Block, parseCtx *ModParseContext) (modconfig.HclResource, *decodeResult) {
func decodeBlock(block *hcl.Block, parseCtx *ModParseContext) (modconfig.HclResource, *DecodeResult) {
var resource modconfig.HclResource
var res = newDecodeResult()
@@ -172,8 +172,16 @@ func decodeBlock(block *hcl.Block, parseCtx *ModParseContext) (modconfig.HclReso
return resource, res
}
func decodeMod(block *hcl.Block, evalCtx *hcl.EvalContext, mod *modconfig.Mod) (*modconfig.Mod, *DecodeResult) {
res := newDecodeResult()
// decode the body
diags := decodeHclBody(block.Body, evalCtx, mod, mod)
res.handleDecodeDiags(diags)
return mod, res
}
// generic decode function for any resource we do not have custom decode logic for
func decodeResource(block *hcl.Block, parseCtx *ModParseContext) (modconfig.HclResource, *decodeResult) {
func decodeResource(block *hcl.Block, parseCtx *ModParseContext) (modconfig.HclResource, *DecodeResult) {
res := newDecodeResult()
// get shell resource
resource, diags := resourceForBlock(block, parseCtx)
@@ -232,7 +240,7 @@ func resourceForBlock(block *hcl.Block, parseCtx *ModParseContext) (modconfig.Hc
return resource, nil
}
func decodeLocals(block *hcl.Block, parseCtx *ModParseContext) ([]*modconfig.Local, *decodeResult) {
func decodeLocals(block *hcl.Block, parseCtx *ModParseContext) ([]*modconfig.Local, *DecodeResult) {
res := newDecodeResult()
attrs, diags := block.Body.JustAttributes()
if len(attrs) == 0 {
@@ -263,7 +271,7 @@ func decodeLocals(block *hcl.Block, parseCtx *ModParseContext) ([]*modconfig.Loc
return locals, res
}
func decodeVariable(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Variable, *decodeResult) {
func decodeVariable(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Variable, *DecodeResult) {
res := newDecodeResult()
var variable *modconfig.Variable
@@ -281,7 +289,7 @@ func decodeVariable(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Var
}
func decodeQueryProvider(block *hcl.Block, parseCtx *ModParseContext) (modconfig.QueryProvider, *decodeResult) {
func decodeQueryProvider(block *hcl.Block, parseCtx *ModParseContext) (modconfig.QueryProvider, *DecodeResult) {
res := newDecodeResult()
// TODO [node_reuse] need raise errors for invalid properties https://github.com/turbot/steampipe/issues/2923
@@ -309,7 +317,7 @@ func decodeQueryProvider(block *hcl.Block, parseCtx *ModParseContext) (modconfig
return resource.(modconfig.QueryProvider), res
}
func decodeQueryProviderBlocks(block *hcl.Block, content *hclsyntax.Body, resource modconfig.HclResource, parseCtx *ModParseContext) *decodeResult {
func decodeQueryProviderBlocks(block *hcl.Block, content *hclsyntax.Body, resource modconfig.HclResource, parseCtx *ModParseContext) *DecodeResult {
var diags hcl.Diagnostics
res := newDecodeResult()
queryProvider, ok := resource.(modconfig.QueryProvider)
@@ -350,7 +358,7 @@ func decodeQueryProviderBlocks(block *hcl.Block, content *hclsyntax.Body, resour
return res
}
func decodeNodeAndEdgeProvider(block *hcl.Block, parseCtx *ModParseContext) (modconfig.HclResource, *decodeResult) {
func decodeNodeAndEdgeProvider(block *hcl.Block, parseCtx *ModParseContext) (modconfig.HclResource, *DecodeResult) {
res := newDecodeResult()
// TODO [node_reuse] need raise errors for invalid properties https://github.com/turbot/steampipe/issues/2923
@@ -393,7 +401,7 @@ func decodeNodeAndEdgeProvider(block *hcl.Block, parseCtx *ModParseContext) (mod
return resource, res
}
func decodeNodeAndEdgeProviderBlocks(content *hclsyntax.Body, nodeAndEdgeProvider modconfig.NodeAndEdgeProvider, parseCtx *ModParseContext) *decodeResult {
func decodeNodeAndEdgeProviderBlocks(content *hclsyntax.Body, nodeAndEdgeProvider modconfig.NodeAndEdgeProvider, parseCtx *ModParseContext) *DecodeResult {
var res = newDecodeResult()
for _, b := range content.Blocks {
@@ -443,7 +451,7 @@ func decodeNodeAndEdgeProviderBlocks(content *hclsyntax.Body, nodeAndEdgeProvide
return res
}
func decodeDashboard(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Dashboard, *decodeResult) {
func decodeDashboard(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Dashboard, *DecodeResult) {
res := newDecodeResult()
dashboard := modconfig.NewDashboard(block, parseCtx.CurrentMod, parseCtx.DetermineBlockName(block)).(*modconfig.Dashboard)
@@ -477,7 +485,7 @@ func decodeDashboard(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Da
return dashboard, res
}
func decodeDashboardBlocks(content *hclsyntax.Body, dashboard *modconfig.Dashboard, parseCtx *ModParseContext) *decodeResult {
func decodeDashboardBlocks(content *hclsyntax.Body, dashboard *modconfig.Dashboard, parseCtx *ModParseContext) *DecodeResult {
var res = newDecodeResult()
// set dashboard as parent on the run context - this is used when generating names for anonymous blocks
parseCtx.PushParent(dashboard)
@@ -512,7 +520,7 @@ func decodeDashboardBlocks(content *hclsyntax.Body, dashboard *modconfig.Dashboa
return res
}
func decodeDashboardContainer(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.DashboardContainer, *decodeResult) {
func decodeDashboardContainer(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.DashboardContainer, *DecodeResult) {
res := newDecodeResult()
container := modconfig.NewDashboardContainer(block, parseCtx.CurrentMod, parseCtx.DetermineBlockName(block)).(*modconfig.DashboardContainer)
@@ -538,7 +546,7 @@ func decodeDashboardContainer(block *hcl.Block, parseCtx *ModParseContext) (*mod
return container, res
}
func decodeDashboardContainerBlocks(content *hclsyntax.Body, dashboardContainer *modconfig.DashboardContainer, parseCtx *ModParseContext) *decodeResult {
func decodeDashboardContainerBlocks(content *hclsyntax.Body, dashboardContainer *modconfig.DashboardContainer, parseCtx *ModParseContext) *DecodeResult {
var res = newDecodeResult()
// set container as parent on the run context - this is used when generating names for anonymous blocks
@@ -574,31 +582,31 @@ func decodeDashboardContainerBlocks(content *hclsyntax.Body, dashboardContainer
return res
}
func decodeBenchmark(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Benchmark, *decodeResult) {
func decodeBenchmark(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Benchmark, *DecodeResult) {
res := newDecodeResult()
benchmark := modconfig.NewBenchmark(block, parseCtx.CurrentMod, parseCtx.DetermineBlockName(block)).(*modconfig.Benchmark)
content, diags := block.Body.Content(BenchmarkBlockSchema)
res.handleDecodeDiags(diags)
diags = decodeProperty(content, "children", &benchmark.ChildNames, parseCtx)
diags = decodeProperty(content, "children", &benchmark.ChildNames, parseCtx.EvalCtx)
res.handleDecodeDiags(diags)
diags = decodeProperty(content, "description", &benchmark.Description, parseCtx)
diags = decodeProperty(content, "description", &benchmark.Description, parseCtx.EvalCtx)
res.handleDecodeDiags(diags)
diags = decodeProperty(content, "documentation", &benchmark.Documentation, parseCtx)
diags = decodeProperty(content, "documentation", &benchmark.Documentation, parseCtx.EvalCtx)
res.handleDecodeDiags(diags)
diags = decodeProperty(content, "tags", &benchmark.Tags, parseCtx)
diags = decodeProperty(content, "tags", &benchmark.Tags, parseCtx.EvalCtx)
res.handleDecodeDiags(diags)
diags = decodeProperty(content, "title", &benchmark.Title, parseCtx)
diags = decodeProperty(content, "title", &benchmark.Title, parseCtx.EvalCtx)
res.handleDecodeDiags(diags)
diags = decodeProperty(content, "type", &benchmark.Type, parseCtx)
diags = decodeProperty(content, "type", &benchmark.Type, parseCtx.EvalCtx)
res.handleDecodeDiags(diags)
diags = decodeProperty(content, "display", &benchmark.Display, parseCtx)
diags = decodeProperty(content, "display", &benchmark.Display, parseCtx.EvalCtx)
res.handleDecodeDiags(diags)
// now add children
@@ -612,7 +620,7 @@ func decodeBenchmark(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Be
benchmark.ChildNameStrings = getChildNameStringsFromModTreeItem(children)
}
diags = decodeProperty(content, "base", &benchmark.Base, parseCtx)
diags = decodeProperty(content, "base", &benchmark.Base, parseCtx.EvalCtx)
res.handleDecodeDiags(diags)
if benchmark.Base != nil && len(benchmark.Base.ChildNames) > 0 {
supportedChildren := []string{modconfig.BlockTypeBenchmark, modconfig.BlockTypeControl}
@@ -621,15 +629,15 @@ func decodeBenchmark(block *hcl.Block, parseCtx *ModParseContext) (*modconfig.Be
children, _ := resolveChildrenFromNames(benchmark.Base.ChildNameStrings, block, supportedChildren, parseCtx)
benchmark.Base.SetChildren(children)
}
diags = decodeProperty(content, "width", &benchmark.Width, parseCtx)
diags = decodeProperty(content, "width", &benchmark.Width, parseCtx.EvalCtx)
res.handleDecodeDiags(diags)
return benchmark, res
}
func decodeProperty(content *hcl.BodyContent, property string, dest interface{}, parseCtx *ModParseContext) hcl.Diagnostics {
func decodeProperty(content *hcl.BodyContent, property string, dest interface{}, evalCtx *hcl.EvalContext) hcl.Diagnostics {
var diags hcl.Diagnostics
if attr, ok := content.Attributes[property]; ok {
diags = gohcl.DecodeExpression(attr.Expr, parseCtx.EvalCtx, dest)
diags = gohcl.DecodeExpression(attr.Expr, evalCtx, dest)
}
return diags
}
@@ -638,7 +646,7 @@ func decodeProperty(content *hcl.BodyContent, property string, dest interface{},
// if decode was successful:
// - generate and set resource metadata
// - add resource to ModParseContext (which adds it to the mod)handleModDecodeResult
func handleModDecodeResult(resource modconfig.HclResource, res *decodeResult, block *hcl.Block, parseCtx *ModParseContext) {
func handleModDecodeResult(resource modconfig.HclResource, res *DecodeResult, block *hcl.Block, parseCtx *ModParseContext) {
if !res.Success() {
if len(res.Depends) > 0 {
moreDiags := parseCtx.AddDependencies(block, resource.GetUnqualifiedName(), res.Depends)

138
pkg/steampipeconfig/parse/decode_mod.go
View File

@@ -1,138 +0,0 @@
package parse
import (
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/gocty"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
)
func decodeMod(block *hcl.Block, evalCtx *hcl.EvalContext, mod *modconfig.Mod) (*modconfig.Mod, *decodeResult) {
res := newDecodeResult()
// retrieve the body content which complies with modBlockSchema
// - this will be used to handle attributes which need manual decoding
// everything else will be implicitly decoded
content, remain, diags := block.Body.PartialContent(ModBlockSchema)
res.handleDecodeDiags(diags)
// decode the body to populate all properties that can be automatically decoded
diags = decodeHclBody(remain, evalCtx, mod, mod)
res.handleDecodeDiags(diags)
if !res.Success() {
return mod, res
}
// now decode the require block
require, requireRes := decodeRequireBlock(content, evalCtx)
res.Merge(requireRes)
if require != nil {
mod.Require = require
}
return mod, res
}
func decodeRequireBlock(content *hcl.BodyContent, evalCtx *hcl.EvalContext) (*modconfig.Require, *decodeResult) {
var res = newDecodeResult()
block := getFirstBlockOfType(content.Blocks, modconfig.BlockTypeRequire)
if block == nil {
return nil, res
}
// retrieve the body content which complies with modBlockSchema
// - this will be used to handle attributes which need manual decoding
// everything else will be implicitly decoded
content, remain, diags := block.Body.PartialContent(RequireBlockSchema)
res.handleDecodeDiags(diags)
// decode the body into 'modContainer' to populate all properties that can be automatically decoded
require := modconfig.NewRequire()
diags = gohcl.DecodeBody(remain, evalCtx, require)
// handle any resulting diags, which may specify dependencies
res.handleDecodeDiags(diags)
modversionConstraints, modRes := decodeRequireModVersionConstraintBlocks(content, evalCtx)
res.Merge(modRes)
if modversionConstraints != nil {
require.Mods = modversionConstraints
}
return require, res
}
func decodeRequireModVersionConstraintBlocks(content *hcl.BodyContent, evalCtx *hcl.EvalContext) ([]*modconfig.ModVersionConstraint, *decodeResult) {
var res = newDecodeResult()
var constraints []*modconfig.ModVersionConstraint
for _, block := range content.Blocks {
// we only expect mod blocks
if block.Type != modconfig.BlockTypeMod {
continue
}
// retrieve the body content which complies with modBlockSchema
// - this will be used to handle attributes which need manual decoding
// everything else will be implicitly decoded
requireModContent, remain, diags := block.Body.PartialContent(RequireModBlockSchema)
res.handleDecodeDiags(diags)
// decode the body into 'modContainer' to populate all properties that can be automatically decoded
constraint, _ := modconfig.NewModVersionConstraint(block.Labels[0])
diags = gohcl.DecodeBody(remain, evalCtx, constraint)
// handle any resulting diags, which may specify dependencies
res.handleDecodeDiags(diags)
args, modRes := decodeRequireModArgs(requireModContent, evalCtx)
res.Merge(modRes)
if args != nil {
constraint.Args = args
}
constraints = append(constraints, constraint)
}
return constraints, res
}
func decodeRequireModArgs(content *hcl.BodyContent, evalCtx *hcl.EvalContext) (map[string]cty.Value, *decodeResult) {
var res = newDecodeResult()
attr, ok := content.Attributes["args"]
if !ok {
return nil, res
}
// try to evaluate expression
val, diags := attr.Expr.Value(evalCtx)
// handle any resulting diags, which may specify dependencies
res.handleDecodeDiags(diags)
if diags.HasErrors() {
return nil, res
}
argMap, _ := ctyObjectToCtyArgMap(val)
return argMap, res
}
func ctyObjectToCtyArgMap(val cty.Value) (map[string]cty.Value, error) {
res := make(map[string]cty.Value)
it := val.ElementIterator()
for it.Next() {
k, v := it.Element()
// decode key
var key string
if err := gocty.FromCtyValue(k, &key); err != nil {
return nil, err
}
if v.IsKnown() {
res[key] = v
}
}
return res, nil
}

14
pkg/steampipeconfig/parse/decode_result.go
View File

@@ -7,17 +7,17 @@ import (
)
// struct to hold the result of a decoding operation
type decodeResult struct {
type DecodeResult struct {
Diags hcl.Diagnostics
Depends map[string]*modconfig.ResourceDependency
}
func newDecodeResult() *decodeResult {
return &decodeResult{Depends: make(map[string]*modconfig.ResourceDependency)}
func newDecodeResult() *DecodeResult {
return &DecodeResult{Depends: make(map[string]*modconfig.ResourceDependency)}
}
// Merge merges this decode result with another
func (p *decodeResult) Merge(other *decodeResult) *decodeResult {
func (p *DecodeResult) Merge(other *DecodeResult) *DecodeResult {
p.Diags = append(p.Diags, other.Diags...)
for k, v := range other.Depends {
p.Depends[k] = v
@@ -27,13 +27,13 @@ func (p *decodeResult) Merge(other *decodeResult) *decodeResult {
}
// Success returns if the was parsing successful - true if there are no errors and no dependencies
func (p *decodeResult) Success() bool {
func (p *DecodeResult) Success() bool {
return !p.Diags.HasErrors() && len(p.Depends) == 0
}
// if the diags contains dependency errors, add dependencies to the result
// otherwise add diags to the result
func (p *decodeResult) handleDecodeDiags(diags hcl.Diagnostics) {
func (p *DecodeResult) handleDecodeDiags(diags hcl.Diagnostics) {
for _, diag := range diags {
if dependency := diagsToDependency(diag); dependency != nil {
p.Depends[dependency.String()] = dependency
@@ -53,6 +53,6 @@ func diagsToDependency(diag *hcl.Diagnostic) *modconfig.ResourceDependency {
return nil
}
func (p *decodeResult) addDiags(diags hcl.Diagnostics) {
func (p *DecodeResult) addDiags(diags hcl.Diagnostics) {
p.Diags = append(p.Diags, diags...)
}

12
pkg/steampipeconfig/parse/hcl_blocks.go
View File

@@ -1,12 +0,0 @@
package parse
import "github.com/hashicorp/hcl/v2"
func getFirstBlockOfType(blocks hcl.Blocks, blockType string) *hcl.Block {
for _, block := range blocks {
if block.Type == blockType {
return block
}
}
return nil
}

2
pkg/steampipeconfig/parse/installed_mod.go
View File

@@ -1,7 +1,7 @@
package parse
import (
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
)

12
pkg/steampipeconfig/parse/mod_parse_context.go
View File

@@ -164,11 +164,10 @@ func (m *ModParseContext) AddInputVariables(inputVariables *modconfig.ModVariabl
m.DependencyVariables = inputVariables.DependencyVariables
}
func (m *ModParseContext) AddVariablesToReferenceMap() {
func (m *ModParseContext) AddVariablesToEvalContext() {
m.addRootVariablesToReferenceMap(m.Variables)
m.addDependencyVariablesToReferenceMap()
// NOTE: we do not rebuild the eval context here as in practice, buildEvalContext will be called after the
// mod definition is parsed
m.buildEvalContext()
}
// addRootVariablesToReferenceMap sets the Variables property
@@ -521,7 +520,7 @@ func (m *ModParseContext) GetLoadedDependencyMod(requiredModVersion *modconfig.M
return nil, fmt.Errorf("not all dependencies are installed - run 'steampipe mod install'")
}
// use the full name of the locked version as key
d, _ := m.LoadedDependencyMods[lockedVersion.FullName()]
d, _ := m.LoadedDependencyMods[lockedVersion.DependencyPath()]
return d, nil
}
@@ -564,8 +563,7 @@ func (m *ModParseContext) SetCurrentMod(mod *modconfig.Mod) {
m.Variables = dependencyVariables
}
// set the root variables from the parent
// now the mod is set we can add variables to the reference map
// now the mod is set we can add variables to the eval context
// ( we cannot do this until mod as set as we need to identify which variables to use if we are a dependency
m.AddVariablesToReferenceMap()
m.buildEvalContext()
m.AddVariablesToEvalContext()
}

49
pkg/steampipeconfig/parse/parser.go
View File

@@ -2,6 +2,7 @@ package parse
import (
"fmt"
"github.com/turbot/steampipe/pkg/steampipeconfig/hclhelpers"
"io"
"log"
"os"
@@ -18,7 +19,6 @@ import (
"github.com/turbot/steampipe/pkg/filepaths"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
"github.com/turbot/steampipe/pkg/utils"
"github.com/zclconf/go-cty/cty"
"sigs.k8s.io/yaml"
)
@@ -97,36 +97,44 @@ func ModfileExists(modPath string) bool {
// ParseModDefinition parses the modfile only
// it is expected the calling code will have verified the existence of the modfile by calling ModfileExists
// this is called before parsing the workspace to, for example, identify dependency mods
func ParseModDefinition(modPath string) (*modconfig.Mod, error) {
func ParseModDefinition(modPath string, evalCtx *hcl.EvalContext) (*modconfig.Mod, *DecodeResult) {
res := newDecodeResult()
// if there is no mod at this location, return error
modFilePath := filepaths.ModFilePath(modPath)
if _, err := os.Stat(modFilePath); os.IsNotExist(err) {
return nil, fmt.Errorf("no mod file found in %s", modPath)
res.Diags = append(res.Diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("no mod file found in %s", modPath),
})
return nil, res
}
fileData, diags := LoadFileData(modFilePath)
res.addDiags(diags)
if diags.HasErrors() {
return nil, plugin.DiagsToError("Failed to load mod files", diags)
return nil, res
}
body, diags := ParseHclFiles(fileData)
res.addDiags(diags)
if diags.HasErrors() {
return nil, plugin.DiagsToError("Failed to load all mod source files", diags)
return nil, res
}
workspaceContent, diags := body.Content(WorkspaceBlockSchema)
res.addDiags(diags)
if diags.HasErrors() {
return nil, plugin.DiagsToError("Failed to load mod", diags)
return nil, res
}
// build an eval context containing functions
evalCtx := &hcl.EvalContext{
Functions: ContextFunctions(modPath),
Variables: make(map[string]cty.Value),
}
block := getFirstBlockOfType(workspaceContent.Blocks, modconfig.BlockTypeMod)
block := hclhelpers.GetFirstBlockOfType(workspaceContent.Blocks, modconfig.BlockTypeMod)
if block == nil {
return nil, fmt.Errorf("no mod definition found in %s", modPath)
res.Diags = append(res.Diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("no mod definition found in %s", modPath),
})
return nil, res
}
var defRange = block.DefRange
if hclBody, ok := block.Body.(*hclsyntax.Body); ok {
@@ -138,19 +146,18 @@ func ParseModDefinition(modPath string) (*modconfig.Mod, error) {
// create a temporary runContext to decode the mod definition
// note - this is not fully populated - the only properties which will be used are
var res *decodeResult
mod, res = decodeMod(block, evalCtx, mod)
if res.Diags.HasErrors() {
return nil, plugin.DiagsToError("Failed to decode mod hcl file", res.Diags)
return nil, res
}
// NOTE: IGNORE DEPENDENCY ERRORS
// TODO verify any dependency errors are for args only
// call decode callback
if err := mod.OnDecoded(block, nil); err != nil {
return nil, err
}
return mod, nil
diags = mod.OnDecoded(block, nil)
res.addDiags(diags)
return mod, res
}
// ParseMod parses all source hcl files for the mod path and associated resources, and returns the mod object

24
pkg/steampipeconfig/parse/schema.go
View File

@@ -142,30 +142,6 @@ var WorkspaceBlockSchema = &hcl.BodySchema{
},
}
// ModBlockSchema contains schema for the mod blocks which must be manually decoded
var ModBlockSchema = &hcl.BodySchema{
Blocks: []hcl.BlockHeaderSchema{
{
Type: modconfig.BlockTypeRequire,
},
},
}
var RequireBlockSchema = &hcl.BodySchema{
Blocks: []hcl.BlockHeaderSchema{
{
Type: modconfig.BlockTypeMod,
LabelNames: []string{"name"},
},
},
}
var RequireModBlockSchema = &hcl.BodySchema{
Attributes: []hcl.AttributeSchema{
{Name: "args"},
},
}
// DashboardBlockSchema is only used to validate the blocks of a Dashboard
var DashboardBlockSchema = &hcl.BodySchema{
Blocks: []hcl.BlockHeaderSchema{

4
pkg/steampipeconfig/parse/workspace_profile.go
View File

@@ -130,7 +130,7 @@ func decodeWorkspaceProfileOption(block *hcl.Block) (options.Options, hcl.Diagno
return DecodeOptions(block, WithOverride(constants.CmdNameDashboard, &options.WorkspaceProfileDashboard{}))
}
func decodeWorkspaceProfile(block *hcl.Block, parseCtx *WorkspaceProfileParseContext) (*modconfig.WorkspaceProfile, *decodeResult) {
func decodeWorkspaceProfile(block *hcl.Block, parseCtx *WorkspaceProfileParseContext) (*modconfig.WorkspaceProfile, *DecodeResult) {
res := newDecodeResult()
// get shell resource
resource := modconfig.NewWorkspaceProfile(block)
@@ -186,7 +186,7 @@ func decodeWorkspaceProfile(block *hcl.Block, parseCtx *WorkspaceProfileParseCon
return resource, res
}
func handleWorkspaceProfileDecodeResult(resource *modconfig.WorkspaceProfile, res *decodeResult, block *hcl.Block, parseCtx *WorkspaceProfileParseContext) {
func handleWorkspaceProfileDecodeResult(resource *modconfig.WorkspaceProfile, res *DecodeResult, block *hcl.Block, parseCtx *WorkspaceProfileParseContext) {
if res.Success() {
// call post decode hook
// NOTE: must do this BEFORE adding resource to run context to ensure we respect the base property

2
pkg/steampipeconfig/versionmap/dependency_version_map.go
View File

@@ -1,7 +1,7 @@
package versionmap
import (
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
"github.com/xlab/treeprint"
)

2
pkg/steampipeconfig/versionmap/resolved_version_constraint.go
View File

@@ -1,7 +1,7 @@
package versionmap
import (
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
)

2
pkg/steampipeconfig/versionmap/version_list_map.go
View File

@@ -3,7 +3,7 @@ package versionmap
import (
"sort"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
)

2
pkg/steampipeconfig/versionmap/version_map.go
View File

@@ -1,7 +1,7 @@
package versionmap
import (
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
)
// VersionMap represents a map of semver versions, keyed by dependency name

2
pkg/steampipeconfig/versionmap/workspace_lock.go
View File

@@ -8,7 +8,7 @@ import (
"path/filepath"
"strings"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
filehelpers "github.com/turbot/go-kit/files"
"github.com/turbot/steampipe/pkg/error_helpers"
"github.com/turbot/steampipe/pkg/filepaths"

2
pkg/version/version.go
View File

@@ -7,7 +7,7 @@ package version
import (
"fmt"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
)
/**

2
pkg/versionhelpers/constraints.go
View File

@@ -1,7 +1,7 @@
package versionhelpers
import (
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
)
// Constraints wraps semver.Constraints type, adding the Original property

34
pkg/workspace/workspace.go
View File

@@ -275,6 +275,7 @@ func (w *Workspace) loadWorkspaceMod(ctx context.Context) *modconfig.ErrorAndWar
m.ResourceMaps.PopulateReferences()
// set the mod
w.Mod = m
// set the child mods
w.Mods = parseCtx.GetTopLevelDependencyMods()
// NOTE: add in the workspace mod to the dependency mods
w.Mods[w.Mod.Name()] = w.Mod
@@ -287,18 +288,43 @@ func (w *Workspace) loadWorkspaceMod(ctx context.Context) *modconfig.ErrorAndWar
func (w *Workspace) getInputVariables(ctx context.Context, validateMissing bool) (*modconfig.ModVariableMap, error) {
// build a run context just to use to load variable definitions
variablesRunCtx, err := w.getParseContext(ctx)
variablesParseCtx, err := w.getParseContext(ctx)
if err != nil {
return nil, err
}
inputVariables, err := w.getVariableValues(ctx, variablesParseCtx, validateMissing)
if err != nil {
return nil, err
}
// if needed, reload
// if a mod require has args which use a variable, this will not have been resolved in the first pass
// - we need to parse again
if variablesParseCtx.CurrentMod.RequireHasUnresolvedArgs() {
// add the variables into the parse context and rebuild the eval context
variablesParseCtx.AddInputVariables(inputVariables)
variablesParseCtx.AddVariablesToEvalContext()
// now try to parse the mod again
inputVariables, err = w.getVariableValues(ctx, variablesParseCtx, validateMissing)
if err != nil {
return nil, err
}
}
return inputVariables, nil
}
func (w *Workspace) getVariableValues(ctx context.Context, variablesParseCtx *parse.ModParseContext, validateMissing bool) (*modconfig.ModVariableMap, error) {
// load variable definitions
variableMap, err := steampipeconfig.LoadVariableDefinitions(w.Path, variablesRunCtx)
variableMap, err := steampipeconfig.LoadVariableDefinitions(w.Path, variablesParseCtx)
if err != nil {
return nil, err
}
return steampipeconfig.GetVariableValues(ctx, variablesRunCtx, variableMap, validateMissing)
// get the values
return steampipeconfig.GetVariableValues(ctx, variablesParseCtx, variableMap, validateMissing)
}
// build options used to load workspace

18
pkg/workspace/workspace_require.go
View File

@@ -6,10 +6,9 @@ import (
"sort"
"strings"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/plugin"
"github.com/turbot/steampipe/pkg/steampipeconfig/versionmap"
"github.com/turbot/steampipe/pkg/utils"
)
@@ -32,7 +31,7 @@ func (w *Workspace) CheckRequiredPluginsInstalled() error {
if installedVersion, found := installedPlugins[name]; found {
req.SetInstalledVersion(installedVersion)
if installedPlugins[name].LessThan(requiredVersion) {
if !requiredVersion.Check(installedPlugins[name]) {
pluginsNotInstalled = append(pluginsNotInstalled, req)
}
} else {
@@ -52,12 +51,12 @@ func (w *Workspace) ValidateSteampipeVersion() error {
return w.Mod.ValidateSteampipeVersion()
}
func (w *Workspace) getRequiredPlugins() map[string]*semver.Version {
func (w *Workspace) getRequiredPlugins() map[string]*semver.Constraints {
if w.Mod.Require != nil {
requiredPluginVersions := w.Mod.Require.Plugins
requiredVersion := make(versionmap.VersionMap)
requiredVersion := make(map[string]*semver.Constraints)
for _, pluginVersion := range requiredPluginVersions {
requiredVersion[pluginVersion.ShortName()] = pluginVersion.Version
requiredVersion[pluginVersion.ShortName()] = pluginVersion.Constraint
}
return requiredVersion
}
@@ -70,11 +69,10 @@ type requiredPluginVersion struct {
installedVersion string
}
func (v *requiredPluginVersion) SetRequiredVersion(requiredVersion *semver.Version) {
func (v *requiredPluginVersion) SetRequiredVersion(requiredVersion *semver.Constraints) {
requiredVersionString := requiredVersion.String()
// if no required version was specified, the version will be 0.0.0
if requiredVersionString == "0.0.0" {
v.requiredVersion = "latest"
if requiredVersion == nil {
v.requiredVersion = "*"
} else {
v.requiredVersion = requiredVersionString
}

4
tests/acceptance/test_data/bad_mod_with_sp_version_require_not_met/mod.sp
View File

@@ -3,6 +3,8 @@ mod "bad_mod_with_sp_version_require_not_met" {
description = "This mod is used to test that the steampipe commands always respect the requirements mentioned in mod.sp require section"
require {
steampipe = "10.99.99"
steampipe {
min_version = "10.99.99"
}
}
}

20
tests/acceptance/test_data/mod_install/mod_require_tests.json Normal file
View File

@@ -0,0 +1,20 @@
[
{
"test_name": "new steampipe struct with old steampipe property",
"modsp": "mod \"test\" {\n title = \"test\"\n require {\n steampipe = \"0.18.0\"\n steampipe {\n min_version = \"0.18.0\"\n }\n }\n}",
"cmd": "steampipe query 'select 1'",
"expected": "Both 'steampipe' block and deprecated 'steampipe' property are set"
},
{
"test_name": "old steampipe property",
"modsp": "mod \"test\" {\n title = \"test\"\n require {\n steampipe = \"0.18.0\"\n }\n}",
"cmd": "steampipe query 'select 1'",
"expected": "Warning: Property 'steampipe' is deprecated for mod require block - use a steampipe block instead"
},
{
"test_name": "new steampipe block with min_version",
"modsp": "mod \"test\" {\n title = \"test\"\n require {\n steampipe {\n min_version = \"0.18.0\"\n }\n }\n}",
"cmd": "steampipe query 'select 1'",
"expected": "1"
}
]

8
tests/acceptance/test_data/mod_require_tests/mod_with_new_steampipe_block/mod.sp Normal file
View File

@@ -0,0 +1,8 @@
mod "mod_with_new_steampipe_block" {
title = "mod_with_new_steampipe_block"
require {
steampipe {
min_version = "0.18.0"
}
}
}

9
tests/acceptance/test_data/mod_require_tests/mod_with_old_steampipe_and_new_steampipe_block_in_require/mod.sp Normal file
View File

@@ -0,0 +1,9 @@
mod "mod_with_old_steampipe_and_new_steampipe_block_in_require" {
title = "mod_with_old_steampipe_and_new_steampipe_block_in_require"
require {
steampipe = "0.18.0"
steampipe {
min_version = "0.18.0"
}
}
}

6
tests/acceptance/test_data/mod_require_tests/mod_with_old_steampipe_in_require/mod.sp Normal file
View File

@@ -0,0 +1,6 @@
mod "mod_with_old_steampipe_in_require" {
title = "mod_with_old_steampipe_in_require"
require {
steampipe = "0.18.0"
}
}

109
tests/acceptance/test_files/mod.bats
View File

@@ -362,104 +362,11 @@ load "$LIB_BATS_SUPPORT/load.bash"
cd -
}
## require
@test "running steampipe query with mod plugin requirement not met" {
cd $FILE_PATH/test_data/bad_mod_with_plugin_require_not_met
run steampipe query "select 1"
assert_output --partial 'Warning: could not find plugin which satisfies requirement'
cd -
}
@test "running steampipe check with mod plugin requirement not met" {
cd $FILE_PATH/test_data/bad_mod_with_plugin_require_not_met
run steampipe check all
assert_output --partial 'Warning: could not find plugin which satisfies requirement'
cd -
}
@test "running steampipe dashboard with mod plugin requirement not met" {
skip "test has been disabled since the new behaviour is to start dashboard with a warning"
cd $FILE_PATH/test_data/bad_mod_with_plugin_require_not_met
run steampipe dashboard
assert_output --partial "[ Wait ] Loading Workspace
Error: could not find plugin which satisfies requirement 'gcp' in 'mod.bad_mod_with_require_not_met'"
cd -
}
@test "running steampipe query with steampipe CLI version requirement not met" {
cd $FILE_PATH/test_data/bad_mod_with_sp_version_require_not_met
run steampipe query "select 1"
assert_output --partial 'does not satisfy mod.bad_mod_with_sp_version_require_not_met which requires version 10.99.99'
cd -
}
@test "running steampipe check with steampipe CLI version requirement not met" {
cd $FILE_PATH/test_data/bad_mod_with_sp_version_require_not_met
run steampipe check all
assert_output --partial 'does not satisfy mod.bad_mod_with_sp_version_require_not_met which requires version 10.99.99'
cd -
}
@test "running steampipe dashboard with steampipe CLI version requirement not met" {
skip "test has been disabled since the new behaviour is to start dashboard with a warning"
cd $FILE_PATH/test_data/bad_mod_with_sp_version_require_not_met
run steampipe dashboard
assert_output --partial 'does not satisfy mod.bad_mod_with_sp_version_require_not_met which requires version 10.99.99'
cd -
}
@test "running steampipe query with dependant mod version requirement not met(not installed)" {
cd $FILE_PATH/test_data/bad_mod_with_dep_mod_version_require_not_met
run steampipe query "select 1"
assert_output --partial 'Error: failed to load workspace: not all dependencies are installed'
run steampipe mod install
assert_output --partial 'Error: 1 dependency failed to install - no version of github.com/turbot/steampipe-mod-aws-compliance found satisfying version constraint: 99.21.0'
cd -
}
@test "running steampipe check with dependant mod version requirement not met(not installed)" {
cd $FILE_PATH/test_data/bad_mod_with_dep_mod_version_require_not_met
run steampipe check all
assert_output --partial 'Error: failed to load workspace: not all dependencies are installed'
run steampipe mod install
assert_output --partial 'Error: 1 dependency failed to install - no version of github.com/turbot/steampipe-mod-aws-compliance found satisfying version constraint: 99.21.0'
cd -
}
@test "running steampipe dashboard with dependant mod version requirement not met(not installed)" {
skip "test has been disabled since the new behaviour is to start dashboard with a warning"
cd $FILE_PATH/test_data/bad_mod_with_dep_mod_version_require_not_met
run steampipe dashboard
assert_output --partial 'Error: failed to load workspace: not all dependencies are installed'
run steampipe mod install
assert_output --partial 'Error: 1 dependency failed to install - no version of github.com/turbot/steampipe-mod-aws-compliance found satisfying version constraint: 99.21.0'
cd -
}
## parsing
@test "mod parsing" {
# install necessary plugins
steampipe plugin install aws
steampipe plugin install ibm
steampipe plugin install oci
steampipe plugin install azure
steampipe plugin install azuread
steampipe plugin install aws oci azure azuread
# create a directory to install the mods
target_directory=$(mktemp -d)
@@ -481,14 +388,6 @@ Error: could not find plugin which satisfies requirement 'gcp' in 'mod.bad_mod_w
assert_success
cd -
# install steampipe-mod-ibm-insights
steampipe mod install github.com/turbot/steampipe-mod-ibm-insights
# go to the mod directory and run steampipe query to verify parsing
cd .steampipe/mods/github.com/turbot/steampipe-mod-ibm-insights@*
run steampipe query "select 1"
assert_success
cd -
# install steampipe-mod-oci-compliance
steampipe mod install github.com/turbot/steampipe-mod-oci-compliance
# go to the mod directory and run steampipe query to verify parsing
@@ -518,11 +417,7 @@ Error: could not find plugin which satisfies requirement 'gcp' in 'mod.bad_mod_w
rm -f $STEAMPIPE_INSTALL_DIR/config/azuread.spc
# uninstall the plugins
steampipe plugin uninstall aws
steampipe plugin uninstall ibm
steampipe plugin uninstall oci
steampipe plugin uninstall azure
steampipe plugin uninstall azuread
steampipe plugin uninstall aws oci azure azuread
# rerun steampipe to make sure they are removed from steampipe
steampipe query "select 1"

121
tests/acceptance/test_files/mod_require.bats Normal file
View File

@@ -0,0 +1,121 @@
load "$LIB_BATS_ASSERT/load.bash"
load "$LIB_BATS_SUPPORT/load.bash"
### require tests ###
@test "running steampipe query with mod plugin requirement not met" {
cd $FILE_PATH/test_data/bad_mod_with_plugin_require_not_met
run steampipe query "select 1"
assert_output --partial 'Warning: could not find plugin which satisfies requirement'
cd -
}
@test "running steampipe check with mod plugin requirement not met" {
cd $FILE_PATH/test_data/bad_mod_with_plugin_require_not_met
run steampipe check all
assert_output --partial 'Warning: could not find plugin which satisfies requirement'
cd -
}
@test "running steampipe dashboard with mod plugin requirement not met" {
skip "test has been disabled since the new behaviour is to start dashboard with a warning"
cd $FILE_PATH/test_data/bad_mod_with_plugin_require_not_met
run steampipe dashboard
assert_output --partial "[ Wait ] Loading Workspace
Error: could not find plugin which satisfies requirement 'gcp' in 'mod.bad_mod_with_require_not_met'"
cd -
}
@test "running steampipe query with steampipe CLI version requirement not met" {
cd $FILE_PATH/test_data/bad_mod_with_sp_version_require_not_met
run steampipe query "select 1"
assert_output --partial 'does not satisfy mod.bad_mod_with_sp_version_require_not_met which requires version 10.99.99'
cd -
}
@test "running steampipe check with steampipe CLI version requirement not met" {
cd $FILE_PATH/test_data/bad_mod_with_sp_version_require_not_met
run steampipe check all
assert_output --partial 'does not satisfy mod.bad_mod_with_sp_version_require_not_met which requires version 10.99.99'
cd -
}
@test "running steampipe dashboard with steampipe CLI version requirement not met" {
skip "test has been disabled since the new behaviour is to start dashboard with a warning"
cd $FILE_PATH/test_data/bad_mod_with_sp_version_require_not_met
run steampipe dashboard
assert_output --partial 'does not satisfy mod.bad_mod_with_sp_version_require_not_met which requires version 10.99.99'
cd -
}
@test "running steampipe query with dependant mod version requirement not met(not installed)" {
cd $FILE_PATH/test_data/bad_mod_with_dep_mod_version_require_not_met
run steampipe query "select 1"
assert_output --partial 'Error: failed to load workspace: not all dependencies are installed'
run steampipe mod install
assert_output --partial 'Error: 1 dependency failed to install - no version of github.com/turbot/steampipe-mod-aws-compliance found satisfying version constraint: 99.21.0'
cd -
}
@test "running steampipe check with dependant mod version requirement not met(not installed)" {
cd $FILE_PATH/test_data/bad_mod_with_dep_mod_version_require_not_met
run steampipe check all
assert_output --partial 'Error: failed to load workspace: not all dependencies are installed'
run steampipe mod install
assert_output --partial 'Error: 1 dependency failed to install - no version of github.com/turbot/steampipe-mod-aws-compliance found satisfying version constraint: 99.21.0'
cd -
}
@test "running steampipe dashboard with dependant mod version requirement not met(not installed)" {
skip "test has been disabled since the new behaviour is to start dashboard with a warning"
cd $FILE_PATH/test_data/bad_mod_with_dep_mod_version_require_not_met
run steampipe dashboard
assert_output --partial 'Error: failed to load workspace: not all dependencies are installed'
run steampipe mod install
assert_output --partial 'Error: 1 dependency failed to install - no version of github.com/turbot/steampipe-mod-aws-compliance found satisfying version constraint: 99.21.0'
cd -
}
### deprecation tests ###
@test "old steampipe property" {
# go to the mod directory and run steampipe to get the deprectaion warning
# or error, and check the output
cd $FILE_PATH/test_data/mod_require_tests/mod_with_old_steampipe_in_require
run steampipe query "select 1"
assert_output --partial "Warning: Property 'steampipe' is deprecated for mod require block - use a steampipe block instead"
}
@test "new steampipe block with old steampipe property" {
# go to the mod directory and run steampipe to get the deprectaion warning
# or error, and check the output
cd $FILE_PATH/test_data/mod_require_tests/mod_with_old_steampipe_and_new_steampipe_block_in_require
run steampipe query "select 1"
assert_output --partial "Both 'steampipe' block and deprecated 'steampipe' property are set"
}
@test "new steampipe block with min_version" {
# go to the mod directory and run steampipe to get the deprectaion warning
# or error, and check the output
cd $FILE_PATH/test_data/mod_require_tests/mod_with_new_steampipe_block
run steampipe query "select 1"
assert_output --partial "1"
}