Support mods requiring different versions of the same mod. Resources from transitive dependencies should not added the mod resource map #3302 (#3303)

This commit is contained in:
kaidaguerre
2023-04-04 15:08:27 +01:00
committed by GitHub
parent 33cfeacaff
commit dd08d9d8a4
24 changed files with 388 additions and 257 deletions

29
design/mod_deps.md Normal file
View File

@@ -0,0 +1,29 @@
ModParseContext has LoadedDependencyMods modconfig.ModMap
currently keyed by mod name - change to key by full name of locked version
GetLockedModVersionConstraint()
FullName()
Usage
1) loadModDependencies
```go
func loadModDependencies(mod *modconfig.Mod, parseCtx *parse.ModParseContext) error {
...
for _, requiredModVersion := range mod.Require.Mods {
// if we have a locked version, update the required version to reflect this
lockedVersion, err := parseCtx.WorkspaceLock.GetLockedModVersionConstraint(requiredModVersion, mod)
if err != nil {
errors = append(errors, err)
continue
}
if lockedVersion != nil {
requiredModVersion = lockedVersion
}
// have we already loaded a mod which satisfied this
if loadedMod, ok := parseCtx.LoadedDependencyMods[requiredModVersion.Name]; ok {
```

View File

@@ -64,17 +64,18 @@ func (d *InstallData) GetAvailableUpdates() (versionmap.DependencyVersionMap, er
// onModInstalled is called when a dependency is satisfied by installing a mod version
func (d *InstallData) onModInstalled(dependency *ResolvedModRef, modDef *modconfig.Mod, parent *modconfig.Mod) {
parentPath := parent.GetModDependencyPath()
parentPath := parent.GetInstallCacheKey()
// get the constraint from the parent (it must be there)
modVersion := parent.Require.GetModDependency(dependency.Name)
modVersionConstraint := parent.Require.GetModDependency(dependency.Name).Constraint.Original
// update lock
d.NewLock.InstallCache.Add(dependency.Name, modDef.ShortName, modDef.Version, modVersion.Constraint.Original, parentPath)
d.NewLock.InstallCache.Add(dependency.Name, modDef.ShortName, modDef.Version, modVersionConstraint, parentPath)
}
// addExisting is called when a dependency is satisfied by a mod which is already installed
func (d *InstallData) addExisting(dependencyName string, existingDep *modconfig.Mod, constraint *versionhelpers.Constraints, parent *modconfig.Mod) {
// update lock
parentPath := parent.GetModDependencyPath()
parentPath := parent.GetInstallCacheKey()
d.NewLock.InstallCache.Add(dependencyName, existingDep.ShortName, existingDep.Version, constraint.Original, parentPath)
}
@@ -107,13 +108,13 @@ func (d *InstallData) onInstallComplete() {
}
func (d *InstallData) GetUpdatedTree() treeprint.Tree {
return d.Upgraded.GetDependencyTree(d.WorkspaceMod.GetModDependencyPath())
return d.Upgraded.GetDependencyTree(d.WorkspaceMod.GetInstallCacheKey())
}
func (d *InstallData) GetInstalledTree() treeprint.Tree {
return d.Installed.GetDependencyTree(d.WorkspaceMod.GetModDependencyPath())
return d.Installed.GetDependencyTree(d.WorkspaceMod.GetInstallCacheKey())
}
func (d *InstallData) GetUninstalledTree() treeprint.Tree {
return d.Uninstalled.GetDependencyTree(d.WorkspaceMod.GetModDependencyPath())
return d.Uninstalled.GetDependencyTree(d.WorkspaceMod.GetInstallCacheKey())
}

View File

@@ -184,7 +184,7 @@ func (i *ModInstaller) InstallWorkspaceDependencies() (err error) {
}
func (i *ModInstaller) GetModList() string {
return i.installData.Lock.GetModList(i.workspaceMod.GetModDependencyPath())
return i.installData.Lock.GetModList(i.workspaceMod.GetInstallCacheKey())
}
func (i *ModInstaller) installMods(mods []*modconfig.ModVersionConstraint, parent *modconfig.Mod) error {
@@ -383,7 +383,7 @@ func (i *ModInstaller) install(dependency *ResolvedModRef, parent *modconfig.Mod
return nil, err
}
// now the mod is installed in it's final location, set mod dependency path
if err := i.setModDependencyPath(modDef, destPath); err != nil {
if err := i.setModDependencyConfig(modDef, destPath); err != nil {
return nil, err
}
}
@@ -428,7 +428,7 @@ func (i *ModInstaller) getDependencyDestPath(dependencyFullName string) string {
}
func (i *ModInstaller) loadDependencyMod(modVersion *versionmap.ResolvedVersionConstraint) (*modconfig.Mod, error) {
modPath := i.getDependencyDestPath(modconfig.ModVersionFullName(modVersion.Name, modVersion.Version))
modPath := i.getDependencyDestPath(modconfig.BuildModDependencyPath(modVersion.Name, modVersion.Version))
modDef, err := i.loadModfile(modPath, false)
if err != nil {
return nil, err
@@ -436,7 +436,7 @@ func (i *ModInstaller) loadDependencyMod(modVersion *versionmap.ResolvedVersionC
if modDef == nil {
return nil, fmt.Errorf("failed to load mod from %s", modPath)
}
if err := i.setModDependencyPath(modDef, modPath); err != nil {
if err := i.setModDependencyConfig(modDef, modPath); err != nil {
return nil, err
}
return modDef, nil
@@ -444,8 +444,15 @@ func (i *ModInstaller) loadDependencyMod(modVersion *versionmap.ResolvedVersionC
}
// set the mod dependency path
func (i *ModInstaller) setModDependencyPath(mod *modconfig.Mod, modPath string) (err error) {
mod.ModDependencyPath, err = filepath.Rel(i.modsPath, modPath)
func (i *ModInstaller) setModDependencyConfig(mod *modconfig.Mod, modPath string) (err error) {
mod.DependencyPath, err = filepath.Rel(i.modsPath, modPath)
// parse the dependency path to get the depdency name and version
dependencyName, version, err := modconfig.ParseModDependencyPath(mod.DependencyPath)
if err != nil {
return err
}
mod.DependencyName = dependencyName
mod.Version = version
return
}
@@ -457,6 +464,7 @@ func (i *ModInstaller) loadModfile(modPath string, createDefault bool) (*modconf
}
return nil, nil
}
mod, err := parse.ParseModDefinition(modPath)
if err != nil {
return nil, err

View File

@@ -13,7 +13,7 @@ func (i *ModInstaller) Prune() (versionmap.VersionListMap, error) {
// now delete any mod folders which are not in the lock file
for name, versions := range unusedMods {
for _, version := range versions {
depPath := i.getDependencyDestPath(modconfig.ModVersionFullName(name, version))
depPath := i.getDependencyDestPath(modconfig.BuildModDependencyPath(name, version))
if err := i.deleteDependencyItem(depPath); err != nil {
return nil, err
}

View File

@@ -45,5 +45,5 @@ func (r *ResolvedModRef) setGitReference() {
// FullName returns name in the format <dependency name>@v<dependencyVersion>
func (r *ResolvedModRef) FullName() string {
return modconfig.ModVersionFullName(r.Name, r.Version)
return modconfig.BuildModDependencyPath(r.Name, r.Version)
}

View File

@@ -34,7 +34,7 @@ func getVerb(verb string) string {
func BuildInstallSummary(installData *InstallData) string {
// for now treat an install as update - we only install deps which are in the mod.sp but missing in the mod folder
modDependencyPath := installData.WorkspaceMod.GetModDependencyPath()
modDependencyPath := installData.WorkspaceMod.GetInstallCacheKey()
installCount, installedTreeString := getInstallationResultString(installData.Installed, modDependencyPath)
uninstallCount, uninstalledTreeString := getInstallationResultString(installData.Uninstalled, modDependencyPath)
upgradeCount, upgradeTreeString := getInstallationResultString(installData.Upgraded, modDependencyPath)

View File

@@ -144,9 +144,12 @@ func CollectVariableValuesFromModRequire(mod *modconfig.Mod, parseCtx *parse.Mod
res := make(InputValues)
if mod.Require != nil {
for _, depModConstraint := range mod.Require.Mods {
// find the short name for this mod
depMod, ok := parseCtx.LoadedDependencyMods[depModConstraint.Name]
if !ok {
// find the loaded dep mod which satisfies this constraint
depMod, err := parseCtx.GetLoadedDependencyMod(depModConstraint, mod)
if err != nil {
return nil, err
}
if depMod == nil {
return nil, fmt.Errorf("depency mod %s is not loaded", depMod.Name())
}

View File

@@ -21,7 +21,7 @@ import (
// if CreatePseudoResources flag is set, construct hcl resources for files with specific extensions
// NOTE: it is an error if there is more than 1 mod defined, however zero mods is acceptable
// - a default mod will be created assuming there are any resource files
func LoadMod(modPath string, parseCtx *parse.ModParseContext) (mod *modconfig.Mod, errAndWarnings *modconfig.ErrorAndWarnings) {
func LoadMod(modPath string, parseCtx *parse.ModParseContext, opts ...LoadModOption) (mod *modconfig.Mod, errAndWarnings *modconfig.ErrorAndWarnings) {
defer func() {
if r := recover(); r != nil {
errAndWarnings = modconfig.NewErrorsAndWarning(helpers.ToError(r))
@@ -32,6 +32,12 @@ func LoadMod(modPath string, parseCtx *parse.ModParseContext) (mod *modconfig.Mo
if err != nil {
return nil, modconfig.NewErrorsAndWarning(err)
}
// apply opts to mod
for _, o := range opts {
o(mod)
}
// load the mod dependencies
if err := loadModDependencies(mod, parseCtx); err != nil {
return nil, modconfig.NewErrorsAndWarning(err)
@@ -88,22 +94,16 @@ func loadModDependencies(mod *modconfig.Mod, parseCtx *parse.ModParseContext) er
}
for _, requiredModVersion := range mod.Require.Mods {
// if we have a locked version, update the required version to reflect this
lockedVersion, err := parseCtx.WorkspaceLock.GetLockedModVersionConstraint(requiredModVersion, mod)
if err != nil {
errors = append(errors, err)
continue
}
if lockedVersion != nil {
requiredModVersion = lockedVersion
}
// have we already loaded a mod which satisfied this
if loadedMod, ok := parseCtx.LoadedDependencyMods[requiredModVersion.Name]; ok {
if requiredModVersion.Constraint.Check(loadedMod.Version) {
continue
}
loadedMod, err := parseCtx.GetLoadedDependencyMod(requiredModVersion, mod)
if err != nil {
return err
}
if loadedMod != nil {
continue
}
if err := loadModDependency(requiredModVersion, parseCtx); err != nil {
errors = append(errors, err)
}
@@ -148,19 +148,16 @@ func loadModDependency(modDependency *modconfig.ModVersionConstraint, parseCtx *
childRunCtx.BlockTypes = parseCtx.BlockTypes
childRunCtx.ParentParseCtx = parseCtx
mod, errAndWarnings := LoadMod(dependencyPath, childRunCtx)
// NOTE: pass in the version and dependency path of the mod - these must be set before it loads its depdencies
mod, errAndWarnings := LoadMod(dependencyPath, childRunCtx, WithDependencyConfig(modDependency.Name, version))
if errAndWarnings.GetError() != nil {
return errAndWarnings.GetError()
}
// set the version and dependency path of the mod
mod.Version = version
mod.ModDependencyPath = modDependency.Name
// update loaded dependency mods
parseCtx.LoadedDependencyMods[modDependency.Name] = mod
parseCtx.AddLoadedDependencyMod(mod)
if parseCtx.ParentParseCtx != nil {
parseCtx.ParentParseCtx.LoadedDependencyMods[modDependency.Name] = mod
parseCtx.ParentParseCtx.AddLoadedDependencyMod(mod)
}
return nil
@@ -192,7 +189,7 @@ func loadModResources(modPath string, parseCtx *parse.ModParseContext) (*modconf
return nil, modconfig.NewErrorsAndWarning(plugin.DiagsToError("Failed to load all mod files", diags))
}
// parse all hcl files.
// parse all hcl files (NOTE - this reads the CurrentMod out of ParseContext and adds to it)
mod, errAndWarnings := parse.ParseMod(fileData, pseudoResources, parseCtx)
if errAndWarnings.GetError() == nil {
// now add fully populated mod to the parent run context
@@ -205,7 +202,7 @@ func loadModResources(modPath string, parseCtx *parse.ModParseContext) (*modconf
return mod, errAndWarnings
}
// search the parent folder for a mod installatio which satisfied the given mod dependency
// search the parent folder for a mod installation which satisfied the given mod dependency
func findInstalledDependency(modDependency *modconfig.ModVersionConstraint, parentFolder string) (string, *semver.Version, error) {
shortDepName := filepath.Base(modDependency.Name)
entries, err := os.ReadDir(parentFolder)

View File

@@ -0,0 +1,18 @@
package steampipeconfig
import (
"fmt"
"github.com/Masterminds/semver"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
)
type LoadModOption = func(mod *modconfig.Mod)
func WithDependencyConfig(modDependencyName string, version *semver.Version) LoadModOption {
return func(mod *modconfig.Mod) {
mod.Version = version
// build the ModDependencyPath from the modDependencyName and the version
mod.DependencyPath = fmt.Sprintf("%s@v%s", modDependencyName, version.String())
mod.DependencyName = modDependencyName
}
}

View File

@@ -23,7 +23,7 @@ func LoadVariableDefinitions(variablePath string, parseCtx *parse.ModParseContex
return nil, errAndWarnings.GetError()
}
variableMap := modconfig.NewModVariableMap(mod, parseCtx.LoadedDependencyMods)
variableMap := modconfig.NewModVariableMap(mod, parseCtx.GetTopLevelDependencyMods())
return variableMap, nil
}
@@ -75,7 +75,7 @@ func getInputVariables(variableMap map[string]*modconfig.Variable, validate bool
return nil, err
}
// build map of depedency mod variable values declared in the mod 'Require' section
// build map of dependency mod variable values declared in the mod 'Require' section
depModVarValues, err := inputvars.CollectVariableValuesFromModRequire(mod, parseCtx)
if err != nil {
return nil, err

View File

@@ -27,11 +27,6 @@ type Mod struct {
// required to allow partial decoding
Remain hcl.Body `hcl:",remain" json:"-"`
// ModDependencyPath is the fully qualified mod name, which can be used to 'require' the mod,
// e.g. github.com/turbot/steampipe-mod-azure-thrifty
// This is only set if the mod is installed as a dependency
ModDependencyPath string `cty:"mod_dependency_path"`
// attributes
Categories []string `cty:"categories" hcl:"categories,optional" column:"categories,jsonb"`
Color *string `cty:"color" hcl:"color" column:"color,text"`
@@ -42,17 +37,27 @@ type Mod struct {
LegacyRequire *Require `hcl:"requires,block"`
OpenGraph *OpenGraph `hcl:"opengraph,block" column:"open_graph,jsonb"`
VersionString string `cty:"version"`
Version *semver.Version
// Depency attributes - set if this mod is loaded as a dependency
// the mod version
Version *semver.Version
// DependencyPath is the fully qualified mod name including version,
// which will by the map key in the workspace lock file
// NOTE: this is the relative path to th emod location from the depdemncy install dir (.steampipe/mods)
// e.g. github.com/turbot/steampipe-mod-azure-thrifty@v1.0.0
DependencyPath string
// DependencyName return the name of the mod as a dependency, i.e. the mod dependency path, _without_ the version
// e.g. github.com/turbot/steampipe-mod-azure-thrifty
DependencyName string
// ModPath is the installation location of the mod
ModPath string
// convenient aggregation of all resources
ResourceMaps *ResourceMaps
// the filepath of the mod.sp file (will be empty for default mod)
modFilePath string
// convenient aggregation of all resources
// NOTE: this resource map object references the same set of resources
ResourceMaps *ResourceMaps
}
func NewMod(shortName, modPath string, defRange hcl.Range) *Mod {
@@ -73,26 +78,9 @@ func NewMod(shortName, modPath string, defRange hcl.Range) *Mod {
}
mod.ResourceMaps = NewModResources(mod)
// try to derive mod version from the path
mod.setVersion()
return mod
}
func (m *Mod) setVersion() {
segments := strings.Split(m.ModPath, "@")
if len(segments) == 1 {
return
}
versionString := segments[len(segments)-1]
// try to set version, ignoring error
version, err := semver.NewVersion(versionString)
if err == nil {
m.Version = version
m.VersionString = fmt.Sprintf("%d.%d", version.Major(), version.Minor())
}
}
func (m *Mod) Equals(other *Mod) bool {
res := m.ShortName == other.ShortName &&
m.FullName == other.FullName &&
@@ -152,21 +140,6 @@ func (m *Mod) IsDefaultMod() bool {
return m.modFilePath == ""
}
func (m *Mod) NameWithVersion() string {
if m.VersionString == "" {
return m.ShortName
}
return fmt.Sprintf("%s@%s", m.ShortName, m.VersionString)
}
// GetModDependencyPath ModDependencyPath if it is set. If not it returns NameWithVersion()
func (m *Mod) GetModDependencyPath() string {
if m.ModDependencyPath != "" {
return m.ModDependencyPath
}
return m.NameWithVersion()
}
// GetPaths implements ModTreeItem (override base functionality)
func (m *Mod) GetPaths() []NodePath {
return []NodePath{{m.Name()}}
@@ -177,11 +150,6 @@ func (m *Mod) SetPaths() {}
// OnDecoded implements HclResource
func (m *Mod) OnDecoded(block *hcl.Block, resourceMapProvider ResourceMapsProvider) hcl.Diagnostics {
// if VersionString is set, set Version
if m.VersionString != "" && m.Version == nil {
m.Version, _ = semver.NewVersion(m.VersionString)
}
// handle legacy requires block
if m.LegacyRequire != nil && !m.LegacyRequire.Empty() {
// ensure that both 'require' and 'requires' were not set
@@ -388,3 +356,13 @@ func (m *Mod) ValidateSteampipeVersion() error {
func (m *Mod) CtyValue() (cty.Value, error) {
return GetCtyValue(m)
}
// GetInstallCacheKey returns the key used to find this mod in a workspace lock InstallCache
func (m *Mod) GetInstallCacheKey() string {
// if the ModDependencyPath is set, this is a dependency mod - use that
if m.DependencyPath != "" {
return m.DependencyPath
}
// otherwise use the short name
return m.ShortName
}

View File

@@ -1,42 +1,37 @@
package modconfig
import (
"bytes"
"fmt"
"strings"
"github.com/Masterminds/semver"
)
func ModVersionFullName(name string, version *semver.Version) string {
// BuildModDependencyPath converts a mod dependency name of form github.com/turbot/steampipe-mod-m2
//
// and a version into a dependency path of form github.com/turbot/steampipe-mod-m2@v1.0.0
func BuildModDependencyPath(dependencyName string, version *semver.Version) string {
if version == nil {
return name
// TODO KAI DOES THIS EVER HAPPEN
return dependencyName
}
versionString := GetMonotonicVersionString(version)
return fmt.Sprintf("%s@v%s", name, versionString)
return fmt.Sprintf("%s@v%s", dependencyName, version.String())
}
func GetMonotonicVersionString(v *semver.Version) string {
var buf bytes.Buffer
fmt.Fprintf(&buf, "%d.%d", v.Major(), v.Minor())
if v.Metadata() != "" {
fmt.Fprintf(&buf, "+%s", v.Metadata())
}
return buf.String()
}
func ParseModFullName(fullName string) (modName string, modVersion *semver.Version, err error) {
// we expect modLongName to be of form github.com/turbot/steampipe-mod-m2@v1.0
// ParseModDependencyPath converts a mod depdency path of form github.com/turbot/steampipe-mod-m2@v1.0.0
// into the dependency name (github.com/turbot/steampipe-mod-m2) and version
func ParseModDependencyPath(fullName string) (modDependencyName string, modVersion *semver.Version, err error) {
// split to get the name and version
parts := strings.Split(fullName, "@")
if len(parts) != 2 {
err = fmt.Errorf("invalid mod full name %s", fullName)
return
}
modName = parts[0]
modDependencyName = parts[0]
versionString := parts[1]
modVersion, err = semver.NewVersion(versionString)
// NOTE: we expect the version to be in format 'vx.x.x', i.e. a smver with a preceding v
// NOTE: we expect the version to be in format 'vx.x.x', i.e. a semver with a preceding v
if !strings.HasPrefix(versionString, "v") || err != nil {
err = fmt.Errorf("mod file %s has invalid version", fullName)
}

View File

@@ -84,7 +84,6 @@ func emptyModResources() *ResourceMaps {
}
// ModResourcesForQueries creates a ResourceMaps object containing just the specified queries
// This is used to just create necessary prepared statements when executing batch queries
func ModResourcesForQueries(queryProviders []QueryProvider, mod *Mod) *ResourceMaps {
res := NewModResources(mod)
for _, p := range queryProviders {
@@ -110,6 +109,22 @@ func (m *ResourceMaps) QueryProviders() []QueryProvider {
return res
}
// TopLevelResources returns a new ResourceMaps containing only top level resources (i.e. no dependencies)
func (m *ResourceMaps) TopLevelResources() *ResourceMaps {
res := NewModResources(m.Mod)
f := func(item HclResource) (bool, error) {
if modTreeItem, ok := item.(ModTreeItem); ok && modTreeItem.GetMod().FullName == m.Mod.FullName {
res.AddResource(item)
}
return true, nil
}
m.WalkResources(f)
return res
}
func (m *ResourceMaps) Equals(other *ResourceMaps) bool {
//TODO use cmp.Equals or similar
if other == nil {

View File

@@ -26,6 +26,7 @@ func resolveChildrenFromNames(childNames []string, block *hcl.Block, supportedCh
}
// now get the resource from the parent mod
// find the mod which owns this resource - it may be either the current mod, or one of it's direct dependencies
var mod = parseCtx.GetMod(parsedName.Mod)
if mod == nil {
diags = append(diags, &hcl.Diagnostic{

View File

@@ -39,9 +39,11 @@ type ModParseContext struct {
// the workspace lock data
WorkspaceLock *versionmap.WorkspaceLock
Flags ParseModFlag
ListOptions *filehelpers.ListOptions
LoadedDependencyMods modconfig.ModMap
Flags ParseModFlag
ListOptions *filehelpers.ListOptions
// map of loaded dependency mods, keyed by DependencyPath (including version)
// there may be multiple versions of same mod in this map
loadedDependencyMods modconfig.ModMap
// Variables are populated in an initial parse pass top we store them on the run context
// so we can set them on the mod when we do the main parse
@@ -69,6 +71,8 @@ type ModParseContext struct {
// map of ReferenceTypeValueMaps keyed by mod
// NOTE: all values from root mod are keyed with "local"
referenceValues map[string]ReferenceTypeValueMap
// a map of just the top level dependencies of the CurrentMod, keyed my full mod DepdencyName (with no version)
topLevelDependencyMods modconfig.ModMap
}
func NewModParseContext(workspaceLock *versionmap.WorkspaceLock, rootEvalPath string, flags ParseModFlag, listOptions *filehelpers.ListOptions) *ModParseContext {
@@ -78,7 +82,7 @@ func NewModParseContext(workspaceLock *versionmap.WorkspaceLock, rootEvalPath st
Flags: flags,
WorkspaceLock: workspaceLock,
ListOptions: listOptions,
LoadedDependencyMods: make(modconfig.ModMap),
loadedDependencyMods: make(modconfig.ModMap),
blockChildMap: make(map[string][]string),
blockNameMap: make(map[string]string),
@@ -95,32 +99,32 @@ func NewModParseContext(workspaceLock *versionmap.WorkspaceLock, rootEvalPath st
return c
}
func (r *ModParseContext) EnsureWorkspaceLock(mod *modconfig.Mod) error {
func (m *ModParseContext) EnsureWorkspaceLock(mod *modconfig.Mod) error {
// if the mod has dependencies, there must a workspace lock object in the run context
// (mod MUST be the workspace mod, not a dependency, as we would hit this error as soon as we parse it)
if mod.HasDependentMods() && (r.WorkspaceLock.Empty() || r.WorkspaceLock.Incomplete()) {
if mod.HasDependentMods() && (m.WorkspaceLock.Empty() || m.WorkspaceLock.Incomplete()) {
return fmt.Errorf("not all dependencies are installed - run 'steampipe mod install'")
}
return nil
}
func (r *ModParseContext) PushParent(parent modconfig.ModTreeItem) {
r.parents = append(r.parents, parent.GetUnqualifiedName())
func (m *ModParseContext) PushParent(parent modconfig.ModTreeItem) {
m.parents = append(m.parents, parent.GetUnqualifiedName())
}
func (r *ModParseContext) PopParent() string {
n := len(r.parents) - 1
res := r.parents[n]
r.parents = r.parents[:n]
func (m *ModParseContext) PopParent() string {
n := len(m.parents) - 1
res := m.parents[n]
m.parents = m.parents[:n]
return res
}
func (r *ModParseContext) PeekParent() string {
if len(r.parents) == 0 {
return r.CurrentMod.Name()
func (m *ModParseContext) PeekParent() string {
if len(m.parents) == 0 {
return m.CurrentMod.Name()
}
return r.parents[len(r.parents)-1]
return m.parents[len(m.parents)-1]
}
// VariableValueMap converts a map of variables to a map of the underlying cty value
@@ -134,59 +138,58 @@ func VariableValueMap(variables map[string]*modconfig.Variable) map[string]cty.V
// AddInputVariables adds variables to the run context.
// This function is called for the root run context after loading all input variables
func (r *ModParseContext) AddInputVariables(inputVariables *modconfig.ModVariableMap) {
r.setRootVariables(inputVariables.RootVariables)
r.setDependencyVariables(inputVariables.DependencyVariables)
func (m *ModParseContext) AddInputVariables(inputVariables *modconfig.ModVariableMap) {
m.setRootVariables(inputVariables.RootVariables)
m.setDependencyVariables(inputVariables.DependencyVariables)
}
// SetVariablesForDependencyMod adds variables to the run context.
// This function is called for dependent mod run context
func (r *ModParseContext) SetVariablesForDependencyMod(mod *modconfig.Mod, dependencyVariablesMap map[string]map[string]*modconfig.Variable) {
r.setRootVariables(dependencyVariablesMap[mod.ShortName])
r.setDependencyVariables(dependencyVariablesMap)
func (m *ModParseContext) SetVariablesForDependencyMod(mod *modconfig.Mod, dependencyVariablesMap map[string]map[string]*modconfig.Variable) {
m.setRootVariables(dependencyVariablesMap[mod.ShortName])
m.setDependencyVariables(dependencyVariablesMap)
}
// setRootVariables sets the Variables property
// and adds the variables to the referenceValues map (used to build the eval context)
func (r *ModParseContext) setRootVariables(variables map[string]*modconfig.Variable) {
r.Variables = variables
func (m *ModParseContext) setRootVariables(variables map[string]*modconfig.Variable) {
m.Variables = variables
// NOTE: we add with the name "var" not "variable" as that is how variables are referenced
r.referenceValues["local"]["var"] = VariableValueMap(variables)
m.referenceValues["local"]["var"] = VariableValueMap(variables)
}
// setDependencyVariables sets the DependencyVariables property
// and adds the dependency variables to the referenceValues map (used to build the eval context
func (r *ModParseContext) setDependencyVariables(dependencyVariables map[string]map[string]*modconfig.Variable) {
r.DependencyVariables = dependencyVariables
func (m *ModParseContext) setDependencyVariables(dependencyVariables map[string]map[string]*modconfig.Variable) {
m.DependencyVariables = dependencyVariables
// NOTE: we add with the name "var" not "variable" as that is how variables are referenced
// add top level variables
// add dependency mod variables, scoped by mod name
for depModName, depVars := range r.DependencyVariables {
for depModName, depVars := range m.DependencyVariables {
// create map for this dependency if needed
if r.referenceValues[depModName] == nil {
r.referenceValues[depModName] = make(ReferenceTypeValueMap)
if m.referenceValues[depModName] == nil {
m.referenceValues[depModName] = make(ReferenceTypeValueMap)
}
r.referenceValues[depModName]["var"] = VariableValueMap(depVars)
m.referenceValues[depModName]["var"] = VariableValueMap(depVars)
}
}
// AddMod is used to add a mod with any pseudo resources to the eval context
// - in practice this will be a shell mod with just pseudo resources - other resources will be added as they are parsed
func (r *ModParseContext) AddMod(mod *modconfig.Mod) hcl.Diagnostics {
if len(r.UnresolvedBlocks) > 0 {
// AddMod is used to add a mod to the eval context
func (m *ModParseContext) AddMod(mod *modconfig.Mod) hcl.Diagnostics {
if len(m.UnresolvedBlocks) > 0 {
// should never happen
panic("calling SetContent on runContext but there are unresolved blocks from a previous parse")
}
var diags hcl.Diagnostics
moreDiags := r.storeResourceInCtyMap(mod)
moreDiags := m.storeResourceInCtyMap(mod)
diags = append(diags, moreDiags...)
resourceFunc := func(item modconfig.HclResource) (bool, error) {
// add all mod resources except variables into cty map
if _, ok := item.(*modconfig.Variable); !ok {
moreDiags := r.storeResourceInCtyMap(item)
moreDiags := m.storeResourceInCtyMap(item)
diags = append(diags, moreDiags...)
}
// continue walking
@@ -195,78 +198,85 @@ func (r *ModParseContext) AddMod(mod *modconfig.Mod) hcl.Diagnostics {
mod.WalkResources(resourceFunc)
// rebuild the eval context
r.buildEvalContext()
m.buildEvalContext()
return diags
}
func (r *ModParseContext) SetDecodeContent(content *hcl.BodyContent, fileData map[string][]byte) {
func (m *ModParseContext) SetDecodeContent(content *hcl.BodyContent, fileData map[string][]byte) {
// put blocks into map as well
r.topLevelBlocks = make(map[*hcl.Block]struct{}, len(r.blocks))
m.topLevelBlocks = make(map[*hcl.Block]struct{}, len(m.blocks))
for _, b := range content.Blocks {
r.topLevelBlocks[b] = struct{}{}
m.topLevelBlocks[b] = struct{}{}
}
r.ParseContext.SetDecodeContent(content, fileData)
m.ParseContext.SetDecodeContent(content, fileData)
}
// AddDependencies :: the block could not be resolved as it has dependencies
// 1) store block as unresolved
// 2) add dependencies to our tree of dependencies
func (r *ModParseContext) AddDependencies(block *hcl.Block, name string, dependencies map[string]*modconfig.ResourceDependency) hcl.Diagnostics {
func (m *ModParseContext) AddDependencies(block *hcl.Block, name string, dependencies map[string]*modconfig.ResourceDependency) hcl.Diagnostics {
// TACTICAL if this is NOT a top level block, add a suffix to the block name
// this is needed to avoid circular dependency errors if a nested block references
// a top level block with the same name
if !r.IsTopLevelBlock(block) {
if !m.IsTopLevelBlock(block) {
name = "nested." + name
}
return r.ParseContext.AddDependencies(block, name, dependencies)
return m.ParseContext.AddDependencies(block, name, dependencies)
}
// ShouldCreateDefaultMod returns whether the flag is set to create a default mod if no mod definition exists
func (r *ModParseContext) ShouldCreateDefaultMod() bool {
return r.Flags&CreateDefaultMod == CreateDefaultMod
func (m *ModParseContext) ShouldCreateDefaultMod() bool {
return m.Flags&CreateDefaultMod == CreateDefaultMod
}
// CreatePseudoResources returns whether the flag is set to create pseudo resources
func (r *ModParseContext) CreatePseudoResources() bool {
return r.Flags&CreatePseudoResources == CreatePseudoResources
func (m *ModParseContext) CreatePseudoResources() bool {
return m.Flags&CreatePseudoResources == CreatePseudoResources
}
// AddResource stores this resource as a variable to be added to the eval context. It alse
func (r *ModParseContext) AddResource(resource modconfig.HclResource) hcl.Diagnostics {
diagnostics := r.storeResourceInCtyMap(resource)
func (m *ModParseContext) AddResource(resource modconfig.HclResource) hcl.Diagnostics {
diagnostics := m.storeResourceInCtyMap(resource)
if diagnostics.HasErrors() {
return diagnostics
}
// rebuild the eval context
r.buildEvalContext()
m.buildEvalContext()
return nil
}
func (r *ModParseContext) GetMod(modShortName string) *modconfig.Mod {
if modShortName == r.CurrentMod.ShortName {
return r.CurrentMod
// GetMod finds the mod with given short name, looking only in first level dependencies
// this is used to resolve resource references
// specifically when the 'children' property of dashboards and benchmarks refers to resource in a dependency mod
func (m *ModParseContext) GetMod(modShortName string) *modconfig.Mod {
if modShortName == m.CurrentMod.ShortName {
return m.CurrentMod
}
// we need to iterate through dependency mods - we cannot use modShortNameas key as it is short name
for _, dep := range r.LoadedDependencyMods {
if dep.ShortName == modShortName {
return dep
// we need to iterate through dependency mods of the current mod
key := m.CurrentMod.GetInstallCacheKey()
deps := m.WorkspaceLock.InstallCache[key]
for _, dep := range deps {
depMod, ok := m.loadedDependencyMods[dep.FullName()]
if ok && depMod.ShortName == modShortName {
return depMod
}
}
return nil
}
func (r *ModParseContext) GetResourceMaps() *modconfig.ResourceMaps {
dependencyResourceMaps := make([]*modconfig.ResourceMaps, len(r.LoadedDependencyMods))
idx := 0
func (m *ModParseContext) GetResourceMaps() *modconfig.ResourceMaps {
// use the current mod as the base resource map
resourceMap := r.CurrentMod.GetResourceMaps()
resourceMap := m.CurrentMod.GetResourceMaps()
// get a map of top level loaded dep mods
deps := m.GetTopLevelDependencyMods()
// merge in the dependency mods
for _, m := range r.LoadedDependencyMods {
dependencyResourceMaps[idx] = m.GetResourceMaps()
idx++
dependencyResourceMaps := make([]*modconfig.ResourceMaps, 0, len(deps))
// merge in the top level resources of the dependency mods
for _, dep := range deps {
dependencyResourceMaps = append(dependencyResourceMaps, dep.GetResourceMaps().TopLevelResources())
}
resourceMap = resourceMap.Merge(dependencyResourceMaps)
@@ -278,12 +288,12 @@ func (m *ModParseContext) GetResource(parsedName *modconfig.ParsedResourceName)
}
// eval functions
func (r *ModParseContext) buildEvalContext() {
func (m *ModParseContext) buildEvalContext() {
// convert variables to cty values
variables := make(map[string]cty.Value)
// now for each mod add all the values
for mod, modMap := range r.referenceValues {
for mod, modMap := range m.referenceValues {
if mod == "local" {
for k, v := range modMap {
variables[k] = cty.ObjectVal(v)
@@ -301,32 +311,32 @@ func (r *ModParseContext) buildEvalContext() {
variables[mod] = cty.ObjectVal(refTypeMap)
}
r.ParseContext.buildEvalContext(variables)
m.ParseContext.buildEvalContext(variables)
}
// update the cached cty value for the given resource, as long as itr does not already exist
func (r *ModParseContext) storeResourceInCtyMap(resource modconfig.HclResource) hcl.Diagnostics {
func (m *ModParseContext) storeResourceInCtyMap(resource modconfig.HclResource) hcl.Diagnostics {
// add resource to variable map
ctyValue, diags := r.getResourceCtyValue(resource)
ctyValue, diags := m.getResourceCtyValue(resource)
if diags.HasErrors() {
return diags
}
// add into the reference value map
if diags := r.addReferenceValue(resource, ctyValue); diags.HasErrors() {
if diags := m.addReferenceValue(resource, ctyValue); diags.HasErrors() {
return diags
}
// remove this resource from unparsed blocks
delete(r.UnresolvedBlocks, resource.Name())
delete(m.UnresolvedBlocks, resource.Name())
return nil
}
func (r *ModParseContext) getResourceCtyValue(resource modconfig.HclResource) (cty.Value, hcl.Diagnostics) {
func (m *ModParseContext) getResourceCtyValue(resource modconfig.HclResource) (cty.Value, hcl.Diagnostics) {
ctyValue, err := resource.(modconfig.CtyValueProvider).CtyValue()
if err != nil {
return cty.Zero, r.errToCtyValueDiags(resource, err)
return cty.Zero, m.errToCtyValueDiags(resource, err)
}
// if this is a value map, merge in the values of base structs
// if it is NOT a value map, the resource must have overridden CtyValue so do not merge base structs
@@ -339,21 +349,21 @@ func (r *ModParseContext) getResourceCtyValue(resource modconfig.HclResource) (c
valueMap = make(map[string]cty.Value)
}
base := resource.GetHclResourceImpl()
if err := r.mergeResourceCtyValue(base, valueMap); err != nil {
return cty.Zero, r.errToCtyValueDiags(resource, err)
if err := m.mergeResourceCtyValue(base, valueMap); err != nil {
return cty.Zero, m.errToCtyValueDiags(resource, err)
}
if qp, ok := resource.(modconfig.QueryProvider); ok {
base := qp.GetQueryProviderImpl()
if err := r.mergeResourceCtyValue(base, valueMap); err != nil {
return cty.Zero, r.errToCtyValueDiags(resource, err)
if err := m.mergeResourceCtyValue(base, valueMap); err != nil {
return cty.Zero, m.errToCtyValueDiags(resource, err)
}
}
if treeItem, ok := resource.(modconfig.ModTreeItem); ok {
base := treeItem.GetModTreeItemImpl()
if err := r.mergeResourceCtyValue(base, valueMap); err != nil {
return cty.Zero, r.errToCtyValueDiags(resource, err)
if err := m.mergeResourceCtyValue(base, valueMap); err != nil {
return cty.Zero, m.errToCtyValueDiags(resource, err)
}
}
return cty.ObjectVal(valueMap), nil
@@ -361,7 +371,7 @@ func (r *ModParseContext) getResourceCtyValue(resource modconfig.HclResource) (c
// merge the cty value of the given interface into valueMap
// (note: this mutates valueMap)
func (r *ModParseContext) mergeResourceCtyValue(resource modconfig.CtyValueProvider, valueMap map[string]cty.Value) (err error) {
func (m *ModParseContext) mergeResourceCtyValue(resource modconfig.CtyValueProvider, valueMap map[string]cty.Value) (err error) {
defer func() {
if r := recover(); r != nil {
fmt.Println(string(debug.Stack()))
@@ -382,7 +392,7 @@ func (r *ModParseContext) mergeResourceCtyValue(resource modconfig.CtyValueProvi
return nil
}
func (r *ModParseContext) errToCtyValueDiags(resource modconfig.HclResource, err error) hcl.Diagnostics {
func (m *ModParseContext) errToCtyValueDiags(resource modconfig.HclResource, err error) hcl.Diagnostics {
return hcl.Diagnostics{&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("failed to convert resource '%s' to its cty value", resource.Name()),
@@ -391,7 +401,7 @@ func (r *ModParseContext) errToCtyValueDiags(resource modconfig.HclResource, err
}}
}
func (r *ModParseContext) addReferenceValue(resource modconfig.HclResource, value cty.Value) hcl.Diagnostics {
func (m *ModParseContext) addReferenceValue(resource modconfig.HclResource, value cty.Value) hcl.Diagnostics {
parsedName, err := modconfig.ParseResourceName(resource.Name())
if err != nil {
return hcl.Diagnostics{&hcl.Diagnostic{
@@ -410,17 +420,17 @@ func (r *ModParseContext) addReferenceValue(resource modconfig.HclResource, valu
typeString := parsedName.ItemType
// the resource name will not have a mod - but the run context knows which mod we are parsing
mod := r.CurrentMod
mod := m.CurrentMod
modName := mod.ShortName
if mod.ModPath == r.RootEvalPath {
if mod.ModPath == m.RootEvalPath {
modName = "local"
}
variablesForMod, ok := r.referenceValues[modName]
variablesForMod, ok := m.referenceValues[modName]
// do we have a map of reference values for this dep mod?
if !ok {
// no - create one
variablesForMod = make(ReferenceTypeValueMap)
r.referenceValues[modName] = variablesForMod
m.referenceValues[modName] = variablesForMod
}
// do we have a map of reference values for this type
variablesForType, ok := variablesForMod[typeString]
@@ -435,21 +445,67 @@ func (r *ModParseContext) addReferenceValue(resource modconfig.HclResource, valu
if _, ok := variablesForType[key]; !ok {
variablesForType[key] = value
variablesForMod[typeString] = variablesForType
r.referenceValues[modName] = variablesForMod
m.referenceValues[modName] = variablesForMod
}
return nil
}
func (r *ModParseContext) AddLoadedDependentMods(mods modconfig.ModMap) {
for k, v := range mods {
if _, alreadyLoaded := r.LoadedDependencyMods[k]; !alreadyLoaded {
r.LoadedDependencyMods[k] = v
}
}
}
//func (m *ModParseContext) `AddLoadedDependentMods`(mods modconfig.ModMap) {
// update to include version in name
// for k, v := range mods {
// if _, alreadyLoaded := m.LoadedDependencyMods[k]; !alreadyLoaded {
// m.LoadedDependencyMods[k] = v
// }
// }
//}
func (r *ModParseContext) IsTopLevelBlock(block *hcl.Block) bool {
_, isTopLevel := r.topLevelBlocks[block]
func (m *ModParseContext) IsTopLevelBlock(block *hcl.Block) bool {
_, isTopLevel := m.topLevelBlocks[block]
return isTopLevel
}
func (m *ModParseContext) GetLoadedDependencyMod(requiredModVersion *modconfig.ModVersionConstraint, mod *modconfig.Mod) (*modconfig.Mod, error) {
// if we have a locked version, update the required version to reflect this
lockedVersion, err := m.WorkspaceLock.GetLockedModVersionConstraint(requiredModVersion, mod)
if err != nil {
return nil, err
}
if lockedVersion == nil {
return nil, fmt.Errorf("not all dependencies are installed - run 'steampipe mod install'")
}
// use the full name of the locked version as key
d, _ := m.loadedDependencyMods[lockedVersion.FullName()]
return d, nil
}
func (m *ModParseContext) AddLoadedDependencyMod(mod *modconfig.Mod) {
// should never happen
if mod.DependencyPath == "" {
return
}
m.loadedDependencyMods[mod.DependencyPath] = mod
}
// GetTopLevelDependencyMods build a mod map of top level loaded dependencies, keyed by mod name
func (m *ModParseContext) GetTopLevelDependencyMods() modconfig.ModMap {
// lazy load m.topLevelDependencyMods
if m.topLevelDependencyMods != nil {
return m.topLevelDependencyMods
}
// get install cache key fpor this mod (short name for top level mod or ModDependencyPath for dep mods)
installCacheKey := m.CurrentMod.GetInstallCacheKey()
deps := m.WorkspaceLock.InstallCache[installCacheKey]
m.topLevelDependencyMods = make(modconfig.ModMap, len(deps))
// merge in the dependency mods
for _, dep := range deps {
key := dep.FullName()
loadedDepMod := m.loadedDependencyMods[key]
if loadedDepMod != nil {
// as key use the ModDependencyPath _without_ the version
m.topLevelDependencyMods[loadedDepMod.DependencyName] = loadedDepMod
}
}
return m.topLevelDependencyMods
}

View File

@@ -9,38 +9,38 @@ import (
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
)
func (r *ModParseContext) DetermineBlockName(block *hcl.Block) string {
func (m *ModParseContext) DetermineBlockName(block *hcl.Block) string {
var shortName string
// have we cached a name for this block (i.e. is this the second decode pass)
if name, ok := r.GetCachedBlockShortName(block); ok {
if name, ok := m.GetCachedBlockShortName(block); ok {
return name
}
// if there is a parent set in the parent stack, this block is a child of that parent
parentName := r.PeekParent()
parentName := m.PeekParent()
anonymous := len(block.Labels) == 0
if anonymous {
shortName = r.getUniqueName(block.Type, parentName)
shortName = m.getUniqueName(block.Type, parentName)
} else {
shortName = block.Labels[0]
}
// build unqualified name
unqualifiedName := fmt.Sprintf("%s.%s", block.Type, shortName)
r.addChildBlockForParent(parentName, unqualifiedName)
m.addChildBlockForParent(parentName, unqualifiedName)
// cache this name for the second decode pass
r.cacheBlockName(block, unqualifiedName)
m.cacheBlockName(block, unqualifiedName)
return shortName
}
func (r *ModParseContext) GetCachedBlockName(block *hcl.Block) (string, bool) {
name, ok := r.blockNameMap[r.blockHash(block)]
func (m *ModParseContext) GetCachedBlockName(block *hcl.Block) (string, bool) {
name, ok := m.blockNameMap[m.blockHash(block)]
return name, ok
}
func (r *ModParseContext) GetCachedBlockShortName(block *hcl.Block) (string, bool) {
unqualifiedName, ok := r.blockNameMap[r.blockHash(block)]
func (m *ModParseContext) GetCachedBlockShortName(block *hcl.Block) (string, bool) {
unqualifiedName, ok := m.blockNameMap[m.blockHash(block)]
if ok {
parsedName, err := modconfig.ParseResourceName(unqualifiedName)
if err != nil {
@@ -51,31 +51,31 @@ func (r *ModParseContext) GetCachedBlockShortName(block *hcl.Block) (string, boo
return "", false
}
func (r *ModParseContext) GetDecodedResourceForBlock(block *hcl.Block) (modconfig.HclResource, bool) {
if name, ok := r.GetCachedBlockName(block); ok {
func (m *ModParseContext) GetDecodedResourceForBlock(block *hcl.Block) (modconfig.HclResource, bool) {
if name, ok := m.GetCachedBlockName(block); ok {
// see whether the mod contains this resource already
parsedName, err := modconfig.ParseResourceName(name)
if err == nil {
return r.CurrentMod.GetResource(parsedName)
return m.CurrentMod.GetResource(parsedName)
}
}
return nil, false
}
func (r *ModParseContext) cacheBlockName(block *hcl.Block, shortName string) {
r.blockNameMap[r.blockHash(block)] = shortName
func (m *ModParseContext) cacheBlockName(block *hcl.Block, shortName string) {
m.blockNameMap[m.blockHash(block)] = shortName
}
func (r *ModParseContext) blockHash(block *hcl.Block) string {
func (m *ModParseContext) blockHash(block *hcl.Block) string {
return helpers.GetMD5Hash(block.DefRange.String())
}
// getUniqueName returns a name unique within the scope of this execution tree
func (r *ModParseContext) getUniqueName(blockType string, parent string) string {
func (m *ModParseContext) getUniqueName(blockType string, parent string) string {
// count how many children of this block type the parent has
childCount := 0
for _, childName := range r.blockChildMap[parent] {
for _, childName := range m.blockChildMap[parent] {
parsedName, err := modconfig.ParseResourceName(childName)
if err != nil {
// we do not expect this
@@ -89,6 +89,6 @@ func (r *ModParseContext) getUniqueName(blockType string, parent string) string
return fmt.Sprintf("%s_anonymous_%s_%d", sanitisedParentName, blockType, childCount)
}
func (r *ModParseContext) addChildBlockForParent(parent, child string) {
r.blockChildMap[parent] = append(r.blockChildMap[parent], child)
func (m *ModParseContext) addChildBlockForParent(parent, child string) {
m.blockChildMap[parent] = append(m.blockChildMap[parent], child)
}

View File

@@ -225,8 +225,8 @@ func ParseMod(fileData map[string][]byte, pseudoResources []modconfig.MappableRe
prevUnresolvedBlocks = unresolvedBlocks
}
// now tell mod to build tree of controls.
res.Error = mod.BuildResourceTree(parseCtx.LoadedDependencyMods)
// now tell mod to build tree of resources
res.Error = mod.BuildResourceTree(parseCtx.GetTopLevelDependencyMods())
return mod, res
}

View File

@@ -22,12 +22,12 @@ func (m DependencyVersionMap) Add(dependencyName, alias string, dependencyVersio
m[parentName] = parentItems
}
// FlatMap converts the DependencyVersionMap into a ResolvedVersionMap, keyed by full name
// FlatMap converts the DependencyVersionMap into a ResolvedVersionMap, keyed by mod dependency path
func (m DependencyVersionMap) FlatMap() ResolvedVersionMap {
res := make(ResolvedVersionMap)
for _, deps := range m {
for _, dep := range deps {
res[modconfig.ModVersionFullName(dep.Name, dep.Version)] = dep
res[modconfig.BuildModDependencyPath(dep.Name, dep.Version)] = dep
}
}
return res
@@ -42,7 +42,7 @@ func (m DependencyVersionMap) GetDependencyTree(rootName string) treeprint.Tree
func (m DependencyVersionMap) buildTree(name string, tree treeprint.Tree) {
deps := m[name]
for name, version := range deps {
fullName := modconfig.ModVersionFullName(name, version.Version)
fullName := modconfig.BuildModDependencyPath(name, version.Version)
child := tree.AddBranch(fullName)
// if there are children add them
m.buildTree(fullName, child)

View File

@@ -1,6 +1,7 @@
package versionmap
import (
"fmt"
"github.com/Masterminds/semver"
)
@@ -31,3 +32,7 @@ func (c ResolvedVersionConstraint) Equals(other *ResolvedVersionConstraint) bool
func (c ResolvedVersionConstraint) IsPrerelease() bool {
return c.Version.Prerelease() != "" || c.Version.Metadata() != ""
}
func (c ResolvedVersionConstraint) FullName() string {
return fmt.Sprintf("%s@v%s", c.Name, c.Version.String())
}

View File

@@ -30,7 +30,7 @@ func (m ResolvedVersionListMap) FlatMap() map[string]*ResolvedVersionConstraint
var res = make(map[string]*ResolvedVersionConstraint)
for name, versions := range m {
for _, version := range versions {
key := modconfig.ModVersionFullName(name, version.Version)
key := modconfig.BuildModDependencyPath(name, version.Version)
res[key] = version
}
}
@@ -42,7 +42,7 @@ func (m ResolvedVersionListMap) FlatNames() []string {
var res []string
for name, versions := range m {
for _, version := range versions {
res = append(res, modconfig.ModVersionFullName(name, version.Version))
res = append(res, modconfig.BuildModDependencyPath(name, version.Version))
}
}
return res

View File

@@ -10,11 +10,11 @@ import (
// VersionListMap is a map keyed by dependency name storing a list of versions for each dependency
type VersionListMap map[string]semver.Collection
func (i VersionListMap) Add(name string, version *semver.Version) {
versions := append(i[name], version)
func (m VersionListMap) Add(name string, version *semver.Version) {
versions := append(m[name], version)
// reverse sort the versions
sort.Sort(sort.Reverse(versions))
i[name] = versions
m[name] = versions
}
@@ -23,7 +23,7 @@ func (m VersionListMap) FlatMap() map[string]bool {
var res = make(map[string]bool)
for name, versions := range m {
for _, version := range versions {
key := modconfig.ModVersionFullName(name, version)
key := modconfig.BuildModDependencyPath(name, version)
res[key] = true
}
}

View File

@@ -6,6 +6,7 @@ import (
"log"
"os"
"path/filepath"
"strings"
"github.com/Masterminds/semver"
filehelpers "github.com/turbot/go-kit/files"
@@ -90,14 +91,21 @@ func (l *WorkspaceLock) getInstalledMods() error {
for _, modfilePath := range modFiles {
// try to parse the mon name and version form the parent folder of the modfile
modName, version, err := l.parseModPath(modfilePath)
modDependencyName, version, err := l.parseModPath(modfilePath)
if err != nil {
// if we fail to parse, just ignore this modfile
// - it's parent is not a valid mod installation folder so it is probably a child folder of a mod
continue
}
// ensure the dependency mod folder is correctly named
// - for old versions of steampipe the folder name would omit the patch number
if err := l.validateAndFixFolderNamingFormat(modDependencyName, version, modfilePath); err != nil {
continue
}
// add this mod version to the map
installedMods.Add(modName, version)
installedMods.Add(modDependencyName, version)
}
if len(errors) > 0 {
@@ -107,6 +115,20 @@ func (l *WorkspaceLock) getInstalledMods() error {
return nil
}
func (l *WorkspaceLock) validateAndFixFolderNamingFormat(modName string, version *semver.Version, modfilePath string) error {
// verify folder name is of correct format (i.e. including patch number)
modDir := filepath.Dir(modfilePath)
parts := strings.Split(modDir, "@")
currentVersionString := parts[1]
desiredVersionString := fmt.Sprintf("v%s", version.String())
if desiredVersionString != currentVersionString {
desiredDir := fmt.Sprintf("%s@%s", parts[0], desiredVersionString)
log.Printf("[TRACE] renaming dependency mod folder %s to %s", modDir, desiredDir)
return os.Rename(modDir, desiredDir)
}
return nil
}
// GetUnreferencedMods returns a map of all installed mods which are not in the lock file
func (l *WorkspaceLock) GetUnreferencedMods() VersionListMap {
var unreferencedVersions = make(VersionListMap)
@@ -131,7 +153,7 @@ func (l *WorkspaceLock) setMissing() {
// flatten and iterate
for name, resolvedConstraint := range deps {
fullName := modconfig.ModVersionFullName(name, resolvedConstraint.Version)
fullName := modconfig.BuildModDependencyPath(name, resolvedConstraint.Version)
if !flatInstalled[fullName] {
// get the mod name from the constraint (fullName includes the version)
@@ -145,12 +167,12 @@ func (l *WorkspaceLock) setMissing() {
}
// extract the mod name and version from the modfile path
func (l *WorkspaceLock) parseModPath(modfilePath string) (modName string, modVersion *semver.Version, err error) {
func (l *WorkspaceLock) parseModPath(modfilePath string) (modDependencyName string, modVersion *semver.Version, err error) {
modFullName, err := filepath.Rel(l.ModInstallationPath, filepath.Dir(modfilePath))
if err != nil {
return
}
return modconfig.ParseModFullName(modFullName)
return modconfig.ParseModDependencyPath(modFullName)
}
func (l *WorkspaceLock) Save() error {
@@ -177,17 +199,20 @@ func (l *WorkspaceLock) Delete() error {
// DeleteMods removes mods from the lock file then, if it is empty, deletes the file
func (l *WorkspaceLock) DeleteMods(mods VersionConstraintMap, parent *modconfig.Mod) {
for modName := range mods {
if parentDependencies := l.InstallCache[parent.GetModDependencyPath()]; parentDependencies != nil {
if parentDependencies := l.InstallCache[parent.GetInstallCacheKey()]; parentDependencies != nil {
parentDependencies.Remove(modName)
}
}
}
// GetMod looks for a lock file entry matching the given mod name
func (l *WorkspaceLock) GetMod(modName string, parent *modconfig.Mod) *ResolvedVersionConstraint {
if parentDependencies := l.InstallCache[parent.GetModDependencyPath()]; parentDependencies != nil {
// GetMod looks for a lock file entry matching the given mod dependency name
// (e.g.github.com/turbot/steampipe-mod-azure-thrifty
func (l *WorkspaceLock) GetMod(modDependencyName string, parent *modconfig.Mod) *ResolvedVersionConstraint {
parentKey := parent.GetInstallCacheKey()
if parentDependencies := l.InstallCache[parentKey]; parentDependencies != nil {
// look for this mod in the lock file entries for this parent
return parentDependencies[modName]
return parentDependencies[modDependencyName]
}
return nil
}
@@ -232,7 +257,7 @@ func (l *WorkspaceLock) EnsureLockedModVersion(requiredModVersion *modconfig.Mod
// verify the locked version satisfies the version constraint
if !requiredModVersion.Constraint.Check(lockedVersion.Version) {
return nil, fmt.Errorf("failed to resolvedependencies for %s - locked version %s does not meet the constraint %s", parent.GetModDependencyPath(), modconfig.ModVersionFullName(requiredModVersion.Name, lockedVersion.Version), requiredModVersion.Constraint.Original)
return nil, fmt.Errorf("failed to resolvedependencies for %s - locked version %s does not meet the constraint %s", parent.GetInstallCacheKey(), modconfig.BuildModDependencyPath(requiredModVersion.Name, lockedVersion.Version), requiredModVersion.Constraint.Original)
}
return lockedVersion, nil
@@ -251,7 +276,7 @@ func (l *WorkspaceLock) GetLockedModVersionConstraint(requiredModVersion *modcon
return nil, nil
}
// create a new ModVersionConstraint using the locked version
lockedVersionFullName := modconfig.ModVersionFullName(requiredModVersion.Name, lockedVersion.Version)
lockedVersionFullName := modconfig.BuildModDependencyPath(requiredModVersion.Name, lockedVersion.Version)
return modconfig.NewModVersionConstraint(lockedVersionFullName)
}

View File

@@ -275,7 +275,7 @@ func (w *Workspace) loadWorkspaceMod(ctx context.Context) *modconfig.ErrorAndWar
m.ResourceMaps.PopulateReferences()
// set the mod
w.Mod = m
w.Mods = parseCtx.LoadedDependencyMods
w.Mods = parseCtx.GetTopLevelDependencyMods()
// NOTE: add in the workspace mod to the dependency mods
w.Mods[w.Mod.Name()] = w.Mod

View File

@@ -26,7 +26,7 @@ local
run steampipe mod list
assert_output '
local
└── github.com/turbot/steampipe-mod-aws-compliance@v0.10'
└── github.com/turbot/steampipe-mod-aws-compliance@v0.10.0'
}
@test "install old version when latest already installed" {
@@ -36,7 +36,7 @@ local
Downgraded 1 mod:
local
└── github.com/turbot/steampipe-mod-aws-compliance@v0.1'
└── github.com/turbot/steampipe-mod-aws-compliance@v0.1.0'
}
@test "install mod version, remove .steampipe folder and then run install" {
@@ -51,7 +51,7 @@ local
Installed 1 mod:
local
└── github.com/turbot/steampipe-mod-aws-compliance@v0.1'
└── github.com/turbot/steampipe-mod-aws-compliance@v0.1.0'
}
@test "install mod version, remove .cache file and then run install" {
@@ -66,7 +66,7 @@ local
Installed 1 mod:
local
└── github.com/turbot/steampipe-mod-aws-compliance@v0.1'
└── github.com/turbot/steampipe-mod-aws-compliance@v0.1.0'
}
@test "install mod version should fail, since dependant mod has a requirement of different steampipe CLI version" {