jprdonnelly/steampipe
1
0
Fork 0
mirror of https://github.com/turbot/steampipe.git synced 2025-12-19 18:12:43 -05:00
Code Issues Packages Projects Releases Wiki Activity

Migrate from GCP to GHCR as CR for Plugins. Closes #4232

Browse Source
This commit is contained in:
Graza
2024-04-08 15:15:18 +01:00
committed by GitHub
parent 6896d1a8d2
commit ecefbcc00c
28 changed files with 422 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
cmd/plugin.go
View File

@@ -241,8 +241,12 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) {
}()
// args to 'plugin install' -- one or more plugins to install
// plugin names can be simple names ('aws') for "standard" plugins,
// or full refs to the OCI image (us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0)
// plugin names can be simple names for "standard" plugins, constraint suffixed names
// or full refs to the OCI image
// - aws
// - aws@0.118.0
// - aws@^0.118
// - ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0.0
plugins := append([]string{}, args...)
showProgress := viper.GetBool(constants.ArgProgress)
installReports := make(display.PluginInstallReports, 0, len(plugins))
@@ -261,6 +265,13 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) {
}
}
state, err := installationstate.Load()
if err != nil {
error_helpers.ShowError(ctx, fmt.Errorf("could not load state"))
exitCode = constants.ExitCodePluginLoadingError
return
}
// a leading blank line - since we always output multiple lines
fmt.Println()
progressBars := uiprogress.New()
@@ -273,7 +284,30 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) {
for _, pluginName := range plugins {
installWaitGroup.Add(1)
bar := createProgressBar(pluginName, progressBars)
go doPluginInstall(ctx, bar, pluginName, installWaitGroup, reportChannel)
ref := ociinstaller.NewSteampipeImageRef(pluginName)
org, name, constraint := ref.GetOrgNameAndConstraint()
orgAndName := fmt.Sprintf("%s/%s", org, name)
var resolved plugin.ResolvedPluginVersion
if ref.IsFromSteampipeHub() {
rpv, err := plugin.GetLatestPluginVersionByConstraint(ctx, state.InstallationID, org, name, constraint)
if err != nil || rpv == nil {
report := &display.PluginInstallReport{
Plugin: pluginName,
Skipped: true,
SkipReason: constants.InstallMessagePluginNotFound,
IsUpdateReport: false,
}
reportChannel <- report
installWaitGroup.Done()
continue
}
resolved = *rpv
} else {
resolved = plugin.NewResolvedPluginVersion(orgAndName, constraint, constraint)
}
go doPluginInstall(ctx, bar, pluginName, resolved, installWaitGroup, reportChannel)
}
go func() {
installWaitGroup.Wait()
@@ -312,7 +346,7 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) {
fmt.Println()
}
func doPluginInstall(ctx context.Context, bar *uiprogress.Bar, pluginName string, wg *sync.WaitGroup, returnChannel chan *display.PluginInstallReport) {
func doPluginInstall(ctx context.Context, bar *uiprogress.Bar, pluginName string, resolvedPlugin plugin.ResolvedPluginVersion, wg *sync.WaitGroup, returnChannel chan *display.PluginInstallReport) {
var report *display.PluginInstallReport
pluginAlreadyInstalled, _ := plugin.Exists(ctx, pluginName)
@@ -343,7 +377,8 @@ func doPluginInstall(ctx context.Context, bar *uiprogress.Bar, pluginName string
return helpers.Resize(pluginInstallSteps[b.Current()-1], 20)
}
})
report = installPlugin(ctx, pluginName, false, bar)
report = installPlugin(ctx, resolvedPlugin, false, bar)
}
returnChannel <- report
wg.Done()
@@ -361,8 +396,12 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
}()
// args to 'plugin update' -- one or more plugins to update
// These can be simple names ('aws') for "standard" plugins,
// or full refs to the OCI image (us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0)
// These can be simple names for "standard" plugins, constraint suffixed names
// or full refs to the OCI image
// - aws
// - aws@0.118.0
// - aws@^0.118
// - ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0.0
plugins, err := resolveUpdatePluginsFromArgs(args)
showProgress := viper.GetBool(constants.ArgProgress)
@@ -376,7 +415,7 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
return
}
if len(plugins) > 0 && !(cmdconfig.Viper().GetBool("all")) && plugins[0] == "all" {
if len(plugins) > 0 && !(cmdconfig.Viper().GetBool(constants.ArgAll)) && plugins[0] == constants.ArgAll {
// improve the response to wrong argument "steampipe plugin update all"
fmt.Println()
exitCode = constants.ExitCodeInsufficientOrWrongInputs
@@ -404,8 +443,8 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
if cmdconfig.Viper().GetBool(constants.ArgAll) {
for k, v := range pluginVersions {
ref := ociinstaller.NewSteampipeImageRef(k)
org, name, stream := ref.GetOrgNameAndStream()
key := fmt.Sprintf("%s/%s@%s", org, name, stream)
org, name, constraint := ref.GetOrgNameAndConstraint()
key := fmt.Sprintf("%s/%s@%s", org, name, constraint)
plugins = append(plugins, key)
runUpdatesFor = append(runUpdatesFor, v)
@@ -469,7 +508,7 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
for _, key := range sorted {
report := reports[key]
updateWaitGroup.Add(1)
bar := createProgressBar(report.ShortNameWithStream(), progressBars)
bar := createProgressBar(report.ShortNameWithConstraint(), progressBars)
go doPluginUpdate(ctx, bar, report, updateWaitGroup, reportChannel)
}
go func() {
@@ -497,20 +536,8 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
func doPluginUpdate(ctx context.Context, bar *uiprogress.Bar, pvr plugin.VersionCheckReport, wg *sync.WaitGroup, returnChannel chan *display.PluginInstallReport) {
var report *display.PluginInstallReport
if skip, skipReason := plugin.SkipUpdate(pvr); skip {
bar.AppendFunc(func(b *uiprogress.Bar) string {
// set the progress bar to append itself with "Already Installed"
return helpers.Resize(skipReason, 30)
})
// set the progress bar to the maximum
bar.Set(len(pluginInstallSteps))
report = &display.PluginInstallReport{
Plugin: fmt.Sprintf("%s@%s", pvr.CheckResponse.Name, pvr.CheckResponse.Stream),
Skipped: true,
SkipReason: skipReason,
IsUpdateReport: true,
}
} else {
if plugin.UpdateRequired(pvr) {
// update required, resolve version and install update
bar.AppendFunc(func(b *uiprogress.Bar) string {
// set the progress bar to append itself with the step underway
if b.Current() == 0 {
@@ -519,8 +546,24 @@ func doPluginUpdate(ctx context.Context, bar *uiprogress.Bar, pvr plugin.Version
}
return helpers.Resize(pluginInstallSteps[b.Current()-1], 20)
})
report = installPlugin(ctx, pvr.Plugin.Name, true, bar)
rp := plugin.NewResolvedPluginVersion(pvr.ShortName(), pvr.CheckResponse.Version, pvr.CheckResponse.Constraint)
report = installPlugin(ctx, rp, true, bar)
} else {
// update NOT required, return already installed report
bar.AppendFunc(func(b *uiprogress.Bar) string {
// set the progress bar to append itself with "Already Installed"
return helpers.Resize(constants.InstallMessagePluginLatestAlreadyInstalled, 30)
})
// set the progress bar to the maximum
bar.Set(len(pluginInstallSteps))
report = &display.PluginInstallReport{
Plugin: fmt.Sprintf("%s@%s", pvr.CheckResponse.Name, pvr.CheckResponse.Constraint),
Skipped: true,
SkipReason: constants.InstallMessagePluginLatestAlreadyInstalled,
IsUpdateReport: true,
}
}
returnChannel <- report
wg.Done()
}
@@ -533,7 +576,7 @@ func createProgressBar(plugin string, parentProgressBars *uiprogress.Progress) *
return bar
}
func installPlugin(ctx context.Context, pluginName string, isUpdate bool, bar *uiprogress.Bar) *display.PluginInstallReport {
func installPlugin(ctx context.Context, resolvedPlugin plugin.ResolvedPluginVersion, isUpdate bool, bar *uiprogress.Bar) *display.PluginInstallReport {
// start a channel for progress publications from plugin.Install
progress := make(chan struct{}, 5)
defer func() {
@@ -549,10 +592,11 @@ func installPlugin(ctx context.Context, pluginName string, isUpdate bool, bar *u
}
}()
image, err := plugin.Install(ctx, pluginName, progress, ociinstaller.WithSkipConfig(viper.GetBool(constants.ArgSkipConfig)))
image, err := plugin.Install(ctx, resolvedPlugin, progress, ociinstaller.WithSkipConfig(viper.GetBool(constants.ArgSkipConfig)))
if err != nil {
msg := ""
_, name, stream := ociinstaller.NewSteampipeImageRef(pluginName).GetOrgNameAndStream()
// used to build data for the plugin install report to be used for display purposes
_, name, constraint := ociinstaller.NewSteampipeImageRef(resolvedPlugin.GetVersionTag()).GetOrgNameAndConstraint()
if isPluginNotFoundErr(err) {
exitCode = constants.ExitCodePluginNotFound
msg = constants.InstallMessagePluginNotFound
@@ -560,14 +604,15 @@ func installPlugin(ctx context.Context, pluginName string, isUpdate bool, bar *u
msg = err.Error()
}
return &display.PluginInstallReport{
Plugin: fmt.Sprintf("%s@%s", name, stream),
Plugin: fmt.Sprintf("%s@%s", name, constraint),
Skipped: true,
SkipReason: msg,
IsUpdateReport: isUpdate,
}
}
org, name, stream := image.ImageRef.GetOrgNameAndStream()
// used to build data for the plugin install report to be used for display purposes
org, name, _ := image.ImageRef.GetOrgNameAndConstraint()
versionString := ""
if image.Config.Plugin.Version != "" {
versionString = " v" + image.Config.Plugin.Version
@@ -577,7 +622,7 @@ func installPlugin(ctx context.Context, pluginName string, isUpdate bool, bar *u
docURL = fmt.Sprintf("https://%s/%s", org, name)
}
return &display.PluginInstallReport{
Plugin: fmt.Sprintf("%s@%s", name, stream),
Plugin: fmt.Sprintf("%s@%s", name, resolvedPlugin.Constraint),
Skipped: false,
Version: versionString,
DocURL: docURL,

4
pkg/display/plugin_install.go
View File

@@ -28,9 +28,9 @@ type PluginInstallReport struct {
func (i *PluginInstallReport) skipString() string {
ref := ociinstaller.NewSteampipeImageRef(i.Plugin)
_, name, stream := ref.GetOrgNameAndStream()
_, name, constraint := ref.GetOrgNameAndConstraint()
return fmt.Sprintf("Plugin: %s\nReason: %s", fmt.Sprintf("%s@%s", name, stream), i.SkipReason)
return fmt.Sprintf("Plugin: %s\nReason: %s", fmt.Sprintf("%s@%s", name, constraint), i.SkipReason)
}
func (i *PluginInstallReport) installString() string {

8
pkg/filepaths/steampipe.go
View File

@@ -57,7 +57,7 @@ func EnsurePluginDir() string {
return ensureSteampipeSubDir("plugins")
}
func EnsurePluginInstallDir(pluginImageDisplayRef string) string {
func EnsurePluginInstallDir(pluginImageDisplayRef string) string {
installDir := PluginInstallDir(pluginImageDisplayRef)
if _, err := os.Stat(installDir); os.IsNotExist(err) {
@@ -68,14 +68,14 @@ func EnsurePluginInstallDir(pluginImageDisplayRef string) string {
return installDir
}
func PluginInstallDir(pluginImageDisplayRef string) string {
osSafePath := filepath.FromSlash(pluginImageDisplayRef )
func PluginInstallDir(pluginImageDisplayRef string) string {
osSafePath := filepath.FromSlash(pluginImageDisplayRef)
fullPath := filepath.Join(EnsurePluginDir(), osSafePath)
return fullPath
}
func PluginBinaryPath(pluginImageDisplayRef, pluginAlias string) string {
func PluginBinaryPath(pluginImageDisplayRef, pluginAlias string) string {
return filepath.Join(PluginInstallDir(pluginImageDisplayRef), PluginAliasToLongName(pluginAlias)+".plugin")
}

36
pkg/ociinstaller/imageref.go
View File

@@ -9,7 +9,7 @@ import (
const (
DefaultImageTag = "latest"
DefaultImageRepoActualURL = "us-docker.pkg.dev/steampipe"
DefaultImageRepoActualURL = "ghcr.io/turbot/steampipe"
DefaultImageRepoDisplayURL = "hub.steampipe.io"
DefaultImageOrg = "turbot"
@@ -30,7 +30,7 @@ func NewSteampipeImageRef(ref string) *SteampipeImageRef {
}
// ActualImageRef returns the actual, physical full image ref
// (us-docker.pkg.dev/steampipe/plugins/turbot/aws:1.0.0)
// (ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0.0)
func (r *SteampipeImageRef) ActualImageRef() string {
ref := r.requestedRef
@@ -63,6 +63,15 @@ func (r *SteampipeImageRef) DisplayImageRef() string {
return fullRef
}
// DisplayImageRefConstraintOverride returns a "friendly" user-facing version of the image ref
// but with the version replaced by provided constraint
// (hub.steampipe.io/plugins/turbot/aws@^1.0)
func (r *SteampipeImageRef) DisplayImageRefConstraintOverride(constraint string) string {
dir := r.DisplayImageRef()
s := strings.Split(dir, "@")
return fmt.Sprintf("%s@%s", s[0], constraint)
}
func isDigestRef(ref string) bool {
return strings.Contains(ref, "@sha256:")
}
@@ -84,19 +93,20 @@ func sanitizeRefStream(ref string) string {
return ref
}
func (r *SteampipeImageRef) IsFromSteampipeHub() (bool) {
func (r *SteampipeImageRef) IsFromSteampipeHub() bool {
return strings.HasPrefix(r.DisplayImageRef(), constants.SteampipeHubOCIBase)
}
// GetOrgNameAndStream splits the full image reference into (org, name, stream)
func (r *SteampipeImageRef) GetOrgNameAndStream() (string, string, string) {
// GetOrgNameAndConstraint splits the full image reference into (org, name, constraint)
// Constraint will be either a SemVer version (1.2.3) or a SemVer constraint (^0.4)
func (r *SteampipeImageRef) GetOrgNameAndConstraint() (string, string, string) {
// plugin.Name looks like `hub.steampipe.io/plugins/turbot/aws@latest`
split := strings.Split(r.DisplayImageRef(), "/")
pluginNameAndStream := strings.Split(split[len(split)-1], "@")
pluginNameAndSuffix := strings.Split(split[len(split)-1], "@")
if r.IsFromSteampipeHub() {
return split[len(split)-2], pluginNameAndStream[0], pluginNameAndStream[1]
return split[len(split)-2], pluginNameAndSuffix[0], pluginNameAndSuffix[1]
}
return strings.Join(split[0:len(split)-1], "/"), pluginNameAndStream[0], pluginNameAndStream[1]
return strings.Join(split[0:len(split)-1], "/"), pluginNameAndSuffix[0], pluginNameAndSuffix[1]
}
// GetFriendlyName returns the minimum friendly name so that the original name can be rebuilt using preset defaults:
@@ -133,16 +143,16 @@ func getCondensedImageRef(imageRef string) string {
}
// possible formats include
// us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0
// us-docker.pkg.dev/steampipe/plugin/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1
// ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0.0
// ghcr.io/turbot/steampipe/plugins/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1
// turbot/aws:1.0.0
// turbot/aws
// dockerhub.org/myimage
// dockerhub.org/myimage:mytag
// aws:1.0.0
// aws
// us-docker.pkg.dev/steampipe/plugin/turbot/aws@1.0.0
// us-docker.pkg.dev/steampipe/plugin/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1
// ghcr.io/turbot/steampipe/plugins/turbot/aws@1.0.0
// ghcr.io/turbot/steampipe/plugins/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1
// turbot/aws@1.0.0
// dockerhub.org/myimage@mytag
// aws@1.0.0
@@ -169,7 +179,7 @@ func getFullImageRef(imagePath string) string {
return fmt.Sprintf("%s:%s", items[0], tag)
}
return fmt.Sprintf("%s/%s/%s/%s:%s", DefaultImageRepoActualURL, DefaultImageType, org, parts[len(parts)-1], tag)
default: //ex: us-docker.pkg.dev/steampipe/plugin/turbot/aws
default: //ex: ghcr.io/turbot/steampipe/plugins/turbot/aws
return fmt.Sprintf("%s:%s", items[0], tag)
}
}

70
pkg/ociinstaller/imageref_test.go
View File

@@ -32,31 +32,31 @@ func TestFriendlyImageRef(t *testing.T) {
func TestActualImageRef(t *testing.T) {
cases := map[string]string{
"us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0": "us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0",
"us-docker.pkg.dev/steampipe/plugin/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1": "us-docker.pkg.dev/steampipe/plugin/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1",
"aws": "us-docker.pkg.dev/steampipe/plugins/turbot/aws:latest",
"aws:1": "us-docker.pkg.dev/steampipe/plugins/turbot/aws:1",
"turbot/aws:1": "us-docker.pkg.dev/steampipe/plugins/turbot/aws:1",
"turbot/aws:1.0": "us-docker.pkg.dev/steampipe/plugins/turbot/aws:1.0",
"turbot/aws:1.1.1": "us-docker.pkg.dev/steampipe/plugins/turbot/aws:1.1.1",
"turbot/aws": "us-docker.pkg.dev/steampipe/plugins/turbot/aws:latest",
"mycompany/my-plugin": "us-docker.pkg.dev/steampipe/plugins/mycompany/my-plugin:latest",
"mycompany/my-plugin:some-random_tag": "us-docker.pkg.dev/steampipe/plugins/mycompany/my-plugin:some-random_tag",
"ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0.0": "ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0.0",
"ghcr.io/turbot/steampipe/plugins/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1": "ghcr.io/turbot/steampipe/plugins/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1",
"aws": "ghcr.io/turbot/steampipe/plugins/turbot/aws:latest",
"aws:1": "ghcr.io/turbot/steampipe/plugins/turbot/aws:1",
"turbot/aws:1": "ghcr.io/turbot/steampipe/plugins/turbot/aws:1",
"turbot/aws:1.0": "ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0",
"turbot/aws:1.1.1": "ghcr.io/turbot/steampipe/plugins/turbot/aws:1.1.1",
"turbot/aws": "ghcr.io/turbot/steampipe/plugins/turbot/aws:latest",
"mycompany/my-plugin": "ghcr.io/turbot/steampipe/plugins/mycompany/my-plugin:latest",
"mycompany/my-plugin:some-random_tag": "ghcr.io/turbot/steampipe/plugins/mycompany/my-plugin:some-random_tag",
"dockerhub.org/myimage:mytag": "dockerhub.org/myimage:mytag",
"us-docker.pkg.dev/steampipe/plugins/turbot/aws:latest": "us-docker.pkg.dev/steampipe/plugins/turbot/aws:latest",
"hub.steampipe.io/plugins/turbot/aws:latest": "us-docker.pkg.dev/steampipe/plugins/turbot/aws:latest",
"hub.steampipe.io/plugins/someoneelse/myimage:mytag": "us-docker.pkg.dev/steampipe/plugins/someoneelse/myimage:mytag",
"ghcr.io/turbot/steampipe/plugins/turbot/aws:latest": "ghcr.io/turbot/steampipe/plugins/turbot/aws:latest",
"hub.steampipe.io/plugins/turbot/aws:latest": "ghcr.io/turbot/steampipe/plugins/turbot/aws:latest",
"hub.steampipe.io/plugins/someoneelse/myimage:mytag": "ghcr.io/turbot/steampipe/plugins/someoneelse/myimage:mytag",
"us-docker.pkg.dev/steampipe/plugin/turbot/aws@1.0.0": "us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0",
"aws@1": "us-docker.pkg.dev/steampipe/plugins/turbot/aws:1",
"turbot/aws@1": "us-docker.pkg.dev/steampipe/plugins/turbot/aws:1",
"turbot/aws@1.0": "us-docker.pkg.dev/steampipe/plugins/turbot/aws:1.0",
"turbot/aws@1.1.1": "us-docker.pkg.dev/steampipe/plugins/turbot/aws:1.1.1",
"mycompany/my-plugin@some-random_tag": "us-docker.pkg.dev/steampipe/plugins/mycompany/my-plugin:some-random_tag",
"ghcr.io/turbot/steampipe/plugins/turbot/aws@1.0.0": "ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0.0",
"aws@1": "ghcr.io/turbot/steampipe/plugins/turbot/aws:1",
"turbot/aws@1": "ghcr.io/turbot/steampipe/plugins/turbot/aws:1",
"turbot/aws@1.0": "ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0",
"turbot/aws@1.1.1": "ghcr.io/turbot/steampipe/plugins/turbot/aws:1.1.1",
"mycompany/my-plugin@some-random_tag": "ghcr.io/turbot/steampipe/plugins/mycompany/my-plugin:some-random_tag",
"dockerhub.org/myimage@mytag": "dockerhub.org/myimage:mytag",
"us-docker.pkg.dev/steampipe/plugins/turbot/aws@latest": "us-docker.pkg.dev/steampipe/plugins/turbot/aws:latest",
"hub.steampipe.io/plugins/turbot/aws@latest": "us-docker.pkg.dev/steampipe/plugins/turbot/aws:latest",
"hub.steampipe.io/plugins/someoneelse/myimage@mytag": "us-docker.pkg.dev/steampipe/plugins/someoneelse/myimage:mytag",
"ghcr.io/turbot/steampipe/plugins/turbot/aws@latest": "ghcr.io/turbot/steampipe/plugins/turbot/aws:latest",
"hub.steampipe.io/plugins/turbot/aws@latest": "ghcr.io/turbot/steampipe/plugins/turbot/aws:latest",
"hub.steampipe.io/plugins/someoneelse/myimage@mytag": "ghcr.io/turbot/steampipe/plugins/someoneelse/myimage:mytag",
}
for testCase, want := range cases {
@@ -74,8 +74,8 @@ func TestActualImageRef(t *testing.T) {
func TestDisplayImageRef(t *testing.T) {
cases := map[string]string{
"us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0": "hub.steampipe.io/plugin/turbot/aws@1.0.0",
"us-docker.pkg.dev/steampipe/plugin/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1": "hub.steampipe.io/plugin/turbot/aws@sha256-766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1",
"ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0.0": "hub.steampipe.io/plugins/turbot/aws@1.0.0",
"ghcr.io/turbot/steampipe/plugins/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1": "hub.steampipe.io/plugins/turbot/aws@sha256-766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1",
"aws": "hub.steampipe.io/plugins/turbot/aws@latest",
"aws:1": "hub.steampipe.io/plugins/turbot/aws@1",
"turbot/aws:1": "hub.steampipe.io/plugins/turbot/aws@1",
@@ -85,20 +85,20 @@ func TestDisplayImageRef(t *testing.T) {
"mycompany/my-plugin": "hub.steampipe.io/plugins/mycompany/my-plugin@latest",
"mycompany/my-plugin:some-random_tag": "hub.steampipe.io/plugins/mycompany/my-plugin@some-random_tag",
"dockerhub.org/myimage:mytag": "dockerhub.org/myimage@mytag",
"us-docker.pkg.dev/steampipe/plugins/turbot/aws:latest": "hub.steampipe.io/plugins/turbot/aws@latest",
"hub.steampipe.io/plugins/turbot/aws:latest": "hub.steampipe.io/plugins/turbot/aws@latest",
"hub.steampipe.io/plugins/someoneelse/myimage:mytag": "hub.steampipe.io/plugins/someoneelse/myimage@mytag",
"ghcr.io/turbot/steampipe/plugins/turbot/aws:latest": "hub.steampipe.io/plugins/turbot/aws@latest",
"hub.steampipe.io/plugins/turbot/aws:latest": "hub.steampipe.io/plugins/turbot/aws@latest",
"hub.steampipe.io/plugins/someoneelse/myimage:mytag": "hub.steampipe.io/plugins/someoneelse/myimage@mytag",
"us-docker.pkg.dev/steampipe/plugin/turbot/aws@1.0.0": "hub.steampipe.io/plugin/turbot/aws@1.0.0",
"ghcr.io/turbot/steampipe/plugins/turbot/aws@1.0.0": "hub.steampipe.io/plugins/turbot/aws@1.0.0",
"aws@1": "hub.steampipe.io/plugins/turbot/aws@1",
"turbot/aws@1": "hub.steampipe.io/plugins/turbot/aws@1",
"turbot/aws@1.0": "hub.steampipe.io/plugins/turbot/aws@1.0",
"turbot/aws@1.1.1": "hub.steampipe.io/plugins/turbot/aws@1.1.1",
"mycompany/my-plugin@some-random_tag": "hub.steampipe.io/plugins/mycompany/my-plugin@some-random_tag",
"dockerhub.org/myimage@mytag": "dockerhub.org/myimage@mytag",
"us-docker.pkg.dev/steampipe/plugins/turbot/aws@latest": "hub.steampipe.io/plugins/turbot/aws@latest",
"hub.steampipe.io/plugins/turbot/aws@latest": "hub.steampipe.io/plugins/turbot/aws@latest",
"hub.steampipe.io/plugins/someoneelse/myimage@mytag": "hub.steampipe.io/plugins/someoneelse/myimage@mytag",
"ghcr.io/turbot/steampipe/plugins/turbot/aws@latest": "hub.steampipe.io/plugins/turbot/aws@latest",
"hub.steampipe.io/plugins/turbot/aws@latest": "hub.steampipe.io/plugins/turbot/aws@latest",
"hub.steampipe.io/plugins/someoneelse/myimage@mytag": "hub.steampipe.io/plugins/someoneelse/myimage@mytag",
"aws@v1": "hub.steampipe.io/plugins/turbot/aws@1",
"turbot/aws@v1": "hub.steampipe.io/plugins/turbot/aws@1",
@@ -119,7 +119,7 @@ func TestDisplayImageRef(t *testing.T) {
}
func TestGetOrgNameAndStream(t *testing.T) {
func TestGetOrgNameAndConstraint(t *testing.T) {
cases := map[string][3]string{
"hub.steampipe.io/plugins/turbot/aws@latest": {"turbot", "aws", "latest"},
"turbot/aws@latest": {"turbot", "aws", "latest"},
@@ -136,10 +136,10 @@ func TestGetOrgNameAndStream(t *testing.T) {
for testCase, want := range cases {
t.Run(testCase, func(t *testing.T) {
r := NewSteampipeImageRef(testCase)
org, name, stream := r.GetOrgNameAndStream()
got := [3]string{org, name, stream}
org, name, constraint := r.GetOrgNameAndConstraint()
got := [3]string{org, name, constraint}
if got != want {
t.Errorf("TestGetOrgNameAndStream failed for case '%s': expected %s, got %s", testCase, want, got)
t.Errorf("TestGetOrgNameAndSuffix failed for case '%s': expected %s, got %s", testCase, want, got)
}
})
}

53
pkg/ociinstaller/plugin.go
View File

@@ -14,14 +14,14 @@ import (
"time"
"github.com/turbot/steampipe/pkg/filepaths"
versionfile "github.com/turbot/steampipe/pkg/ociinstaller/versionfile"
"github.com/turbot/steampipe/pkg/ociinstaller/versionfile"
"github.com/turbot/steampipe/pkg/utils"
)
var versionFileUpdateLock = &sync.Mutex{}
// InstallPlugin installs a plugin from an OCI Image
func InstallPlugin(ctx context.Context, imageRef string, sub chan struct{}, opts ...PluginInstallOption) (*SteampipeImage, error) {
func InstallPlugin(ctx context.Context, imageRef string, constraint string, sub chan struct{}, opts ...PluginInstallOption) (*SteampipeImage, error) {
config := &pluginInstallConfig{}
for _, opt := range opts {
opt(config)
@@ -44,21 +44,25 @@ func InstallPlugin(ctx context.Context, imageRef string, sub chan struct{}, opts
return nil, err
}
// update the image ref to include the constraint and use to get the plugin install path
constraintRef := image.ImageRef.DisplayImageRefConstraintOverride(constraint)
pluginPath := filepaths.EnsurePluginInstallDir(constraintRef)
sub <- struct{}{}
if err = installPluginBinary(image, tempDir.Path); err != nil {
if err = installPluginBinary(image, tempDir.Path, pluginPath); err != nil {
return nil, fmt.Errorf("plugin installation failed: %s", err)
}
sub <- struct{}{}
if err = installPluginDocs(image, tempDir.Path); err != nil {
if err = installPluginDocs(image, tempDir.Path, pluginPath); err != nil {
return nil, fmt.Errorf("plugin installation failed: %s", err)
}
if !config.skipConfigFile {
if err = installPluginConfigFiles(image, tempDir.Path); err != nil {
if err = installPluginConfigFiles(image, tempDir.Path, constraint); err != nil {
return nil, fmt.Errorf("plugin installation failed: %s", err)
}
}
sub <- struct{}{}
if err := updatePluginVersionFiles(ctx, image); err != nil {
if err := updatePluginVersionFiles(ctx, image, constraint); err != nil {
return nil, err
}
return image, nil
@@ -66,7 +70,7 @@ func InstallPlugin(ctx context.Context, imageRef string, sub chan struct{}, opts
// updatePluginVersionFiles updates the global versions.json to add installation of the plugin
// also adds a version file in the plugin installation directory with the information
func updatePluginVersionFiles(ctx context.Context, image *SteampipeImage) error {
func updatePluginVersionFiles(ctx context.Context, image *SteampipeImage, constraint string) error {
versionFileUpdateLock.Lock()
defer versionFileUpdateLock.Unlock()
@@ -76,7 +80,9 @@ func updatePluginVersionFiles(ctx context.Context, image *SteampipeImage) error
return err
}
pluginFullName := image.ImageRef.DisplayImageRef()
// For the full name we want the constraint (^0.4) used, not the resolved version (0.4.1)
// we override the DisplayImageRef with the constraint here.
pluginFullName := image.ImageRef.DisplayImageRefConstraintOverride(constraint)
installedVersion, ok := v.Plugins[pluginFullName]
if !ok {
@@ -106,9 +112,8 @@ func updatePluginVersionFiles(ctx context.Context, image *SteampipeImage) error
return v.Save()
}
func installPluginBinary(image *SteampipeImage, tempdir string) error {
sourcePath := filepath.Join(tempdir, image.Plugin.BinaryFile)
destDir := filepaths.EnsurePluginInstallDir(image.ImageRef.DisplayImageRef())
func installPluginBinary(image *SteampipeImage, tempDir string, destDir string) error {
sourcePath := filepath.Join(tempDir, image.Plugin.BinaryFile)
// check if system is M1 - if so we need some special handling
isM1, err := utils.IsMacM1()
@@ -134,17 +139,15 @@ func installPluginBinary(image *SteampipeImage, tempdir string) error {
return nil
}
func installPluginDocs(image *SteampipeImage, tempdir string) error {
installTo := filepaths.EnsurePluginInstallDir(image.ImageRef.DisplayImageRef())
func installPluginDocs(image *SteampipeImage, tempDir string, destDir string) error {
// if DocsDir is not set, then there are no docs.
if image.Plugin.DocsDir == "" {
return nil
}
// install the docs
sourcePath := filepath.Join(tempdir, image.Plugin.DocsDir)
destPath := filepath.Join(installTo, "docs")
sourcePath := filepath.Join(tempDir, image.Plugin.DocsDir)
destPath := filepath.Join(destDir, "docs")
if fileExists(destPath) {
os.RemoveAll(destPath)
}
@@ -154,7 +157,7 @@ func installPluginDocs(image *SteampipeImage, tempdir string) error {
return nil
}
func installPluginConfigFiles(image *SteampipeImage, tempdir string) error {
func installPluginConfigFiles(image *SteampipeImage, tempdir string, constraint string) error {
installTo := filepaths.EnsureConfigDir()
// if ConfigFileDir is not set, then there are no config files.
@@ -172,7 +175,7 @@ func installPluginConfigFiles(image *SteampipeImage, tempdir string) error {
for _, obj := range objects {
sourceFile := filepath.Join(sourcePath, obj.Name())
destFile := filepath.Join(installTo, obj.Name())
if err := copyConfigFileUnlessExists(sourceFile, destFile, image.ImageRef); err != nil {
if err := copyConfigFileUnlessExists(sourceFile, destFile, constraint); err != nil {
return fmt.Errorf("could not copy config file from %s to %s", sourceFile, destFile)
}
}
@@ -180,7 +183,7 @@ func installPluginConfigFiles(image *SteampipeImage, tempdir string) error {
return nil
}
func copyConfigFileUnlessExists(sourceFile string, destFile string, ref *SteampipeImageRef) error {
func copyConfigFileUnlessExists(sourceFile string, destFile string, constraint string) error {
if fileExists(destFile) {
return nil
}
@@ -193,7 +196,7 @@ func copyConfigFileUnlessExists(sourceFile string, destFile string, ref *Steampi
return fmt.Errorf("couldn't read source file permissions: %s", err)
}
// update the connection config with the correct plugin version
inputData = addPluginStreamToConfig(inputData, ref)
inputData = addPluginConstraintToConfig(inputData, constraint)
if err = os.WriteFile(destFile, inputData, inputStat.Mode()); err != nil {
return fmt.Errorf("writing to output file failed: %s", err)
}
@@ -203,15 +206,14 @@ func copyConfigFileUnlessExists(sourceFile string, destFile string, ref *Steampi
// The default config files have the plugin set to the 'latest' stream (as this is what is installed by default)
// When installing non-latest plugins, that property needs to be adjusted to the stream actually getting installed.
// Otherwise, during plugin resolution, it will resolve to an incorrect plugin instance
// (or none at at all, if 'latest' versions isn't installed)
func addPluginStreamToConfig(src []byte, ref *SteampipeImageRef) []byte {
_, _, stream := ref.GetOrgNameAndStream()
if stream == "latest" {
// (or none at all, if 'latest' versions isn't installed)
func addPluginConstraintToConfig(src []byte, constraint string) []byte {
if constraint == "latest" {
return src
}
regex := regexp.MustCompile(`^(\s*)plugin\s*=\s*"(.*)"\s*$`)
substitution := fmt.Sprintf(`$1 plugin = "$2@%s"`, stream)
substitution := fmt.Sprintf(`$1 plugin = "$2@%s"`, constraint)
srcScanner := bufio.NewScanner(strings.NewReader(string(src)))
srcScanner.Split(bufio.ScanLines)
@@ -228,4 +230,3 @@ func addPluginStreamToConfig(src []byte, ref *SteampipeImageRef) []byte {
}
return destBuffer.Bytes()
}

94
pkg/ociinstaller/plugin_test.go
View File

@@ -2,6 +2,10 @@ package ociinstaller
import (
"bytes"
"fmt"
"github.com/turbot/steampipe/pkg/filepaths"
"os"
"path/filepath"
"testing"
)
@@ -37,16 +41,104 @@ var transformTests = map[string]transformTest{
pluginLineContent: []byte(`plugin = "chaos"`),
expectedTransformedPluginLineContent: []byte(`plugin = "chaos@0.2.0"`),
},
"^0.2": {
ref: NewSteampipeImageRef("chaos@^0.2"),
pluginLineContent: []byte(`plugin = "chaos"`),
expectedTransformedPluginLineContent: []byte(`plugin = "chaos@^0.2"`),
},
">=0.2": {
ref: NewSteampipeImageRef("chaos@>=0.2"),
pluginLineContent: []byte(`plugin = "chaos"`),
expectedTransformedPluginLineContent: []byte(`plugin = "chaos@>=0.2"`),
},
}
func TestAddPluginName(t *testing.T) {
for name, test := range transformTests {
sourcebytes := test.pluginLineContent
expectedBytes := test.expectedTransformedPluginLineContent
transformed := bytes.TrimSpace(addPluginStreamToConfig(sourcebytes, test.ref))
_, _, constraint := test.ref.GetOrgNameAndConstraint()
transformed := bytes.TrimSpace(addPluginConstraintToConfig(sourcebytes, constraint))
if !bytes.Equal(transformed, expectedBytes) {
t.Fatalf("%s failed - expected(%s) - got(%s)", name, test.expectedTransformedPluginLineContent, transformed)
}
}
}
func TestConstraintBasedFilePathsReadWrite(t *testing.T) {
tmpDir, err := os.MkdirTemp(os.TempDir(), "test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
filepaths.SteampipeDir = tmpDir
cases := make(map[string][]string)
fileContent := "test string"
cases["basic_checks"] = []string{
"latest",
"1.2.3",
"1.0",
"1",
}
cases["operators"] = []string{
"!=1.2.3",
">1.2.0",
">1.2",
">1",
">=1.2.0",
"<1.2.0",
"<=1.2.0",
}
cases["hyphen_range"] = []string{
"1.1-1.2.3",
"1.2.1-1.2.3",
}
cases["wild_cards"] = []string{
"*",
"1.x",
"1.*",
"1.1.x",
"1.1.*",
">=1.2.x",
"<=1.1.x",
}
cases["tilde_range"] = []string{
"~1",
"~1.1",
"~1.x",
"~1.1.1",
"~1.1.x",
}
cases["caret_range"] = []string{
"^1",
"^1.1",
"^1.x",
"^1.1.1",
"^1.1.*",
}
for category, testCases := range cases {
for _, testCase := range testCases {
constraintedDir := filepaths.EnsurePluginInstallDir(fmt.Sprintf("constraint-test:%s", testCase))
filePath := filepath.Join(constraintedDir, "test.txt")
// Write Test
err := os.WriteFile(filePath, []byte(fileContent), 0644)
if err != nil {
t.Fatalf("Write failed for constraint %s %s", category, testCase)
}
// Read Test
b, err := os.ReadFile(filePath)
if err != nil || string(b) != fileContent {
t.Fatalf("Read failed for constraint %s %s", category, testCase)
}
// tidy up
if err := os.RemoveAll(constraintedDir); err != nil {
t.Logf("Failed to remove test folder and contents: %s", constraintedDir)
}
}
}
}

11
pkg/plugin/actions.go
View File

@@ -18,12 +18,6 @@ import (
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
)
const (
DefaultImageTag = "latest"
DefaultImageRepoURL = "us-docker.pkg.dev/steampipe/plugin"
DefaultImageOrg = "turbot"
)
// Remove removes an installed plugin
func Remove(ctx context.Context, image string, pluginConnections map[string][]*modconfig.Connection) (*steampipeconfig.PluginRemoveReport, error) {
statushooks.SetStatus(ctx, fmt.Sprintf("Removing plugin %s", image))
@@ -71,8 +65,9 @@ func Exists(ctx context.Context, plugin string) (bool, error) {
}
// Install installs a plugin in the local file system
func Install(ctx context.Context, plugin string, sub chan struct{}, opts ...ociinstaller.PluginInstallOption) (*ociinstaller.SteampipeImage, error) {
image, err := ociinstaller.InstallPlugin(ctx, plugin, sub, opts...)
func Install(ctx context.Context, plugin ResolvedPluginVersion, sub chan struct{}, opts ...ociinstaller.PluginInstallOption) (*ociinstaller.SteampipeImage, error) {
// Note: we pass the plugin info as strings here rather than passing the ResolvedPluginVersion struct as that causes circular dependency
image, err := ociinstaller.InstallPlugin(ctx, plugin.GetVersionTag(), plugin.Constraint, sub, opts...)
return image, err
}

2
pkg/plugin/installed.go
View File

@@ -14,7 +14,7 @@ func GetInstalledPlugins(ctx context.Context) (map[string]*modconfig.PluginVersi
installedPlugins := make(map[string]*modconfig.PluginVersionString)
installedPluginsData, _ := List(ctx, nil)
for _, plugin := range installedPluginsData {
org, name, _ := ociinstaller.NewSteampipeImageRef(plugin.Name).GetOrgNameAndStream()
org, name, _ := ociinstaller.NewSteampipeImageRef(plugin.Name).GetOrgNameAndConstraint()
pluginShortName := fmt.Sprintf("%s/%s", org, name)
installedPlugins[pluginShortName] = plugin.Version
}

22
pkg/plugin/resolved_version.go Normal file
View File

@@ -0,0 +1,22 @@
package plugin
import "fmt"
type ResolvedPluginVersion struct {
PluginName string
Version string
Constraint string
}
func NewResolvedPluginVersion(pluginName string, version string, constraint string) ResolvedPluginVersion {
return ResolvedPluginVersion{
PluginName: pluginName,
Version: version,
Constraint: constraint,
}
}
// GetVersionTag returns the <PluginName>:<Version> (turbot/chaos:0.4.1)
func (r ResolvedPluginVersion) GetVersionTag() string {
return fmt.Sprintf("%s:%s", r.PluginName, r.Version)
}

24
pkg/plugin/update.go
View File

@@ -4,25 +4,24 @@ import (
"runtime"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/ociinstaller"
)
// SkipUpdate determines if the latest version in a "stream"
// UpdateRequired determines if the latest version in a "stream"
// requires the plugin to update.
func SkipUpdate(report VersionCheckReport) (bool, string) {
func UpdateRequired(report VersionCheckReport) bool {
// 1) If there is an updated version ALWAYS update
if report.Plugin.ImageDigest != report.CheckResponse.Digest {
return false, ""
if report.Plugin.Version != report.CheckResponse.Version {
return true
}
// 2) If we are M1, current installed version is AMD, and ARM is available - update
if isRunningAsMacM1() && manifestHasM1Binary(report.CheckResponse.Manifest) && report.Plugin.BinaryArchitecture != constants.ArchARM64 {
return false, ""
if isRunningAsMacM1() && report.Plugin.BinaryArchitecture != constants.ArchARM64 {
return true
}
// 3) Otherwise skip
return true, constants.InstallMessagePluginLatestAlreadyInstalled
return false
}
// check to see if steampipe is running as a Mac/M1 build
@@ -32,12 +31,3 @@ func SkipUpdate(report VersionCheckReport) (bool, string) {
func isRunningAsMacM1() bool {
return runtime.GOOS == constants.OSDarwin && runtime.GOARCH == constants.ArchARM64
}
func manifestHasM1Binary(manifest responseManifest) bool {
for _, rml := range manifest.Layers {
if rml.MediaType == ociinstaller.MediaTypePluginDarwinArm64Layer {
return true
}
}
return false
}

36
pkg/plugin/version_check_payload.go
View File

@@ -7,35 +7,13 @@ type versionCheckPayload interface {
}
// the payload that travels to-and-fro between steampipe and the server
type versionCheckRequestPayload struct {
Org string `json:"org"`
Name string `json:"name"`
Stream string `json:"stream"`
Version string `json:"version"`
Digest string `json:"digest"`
type versionCheckCorePayload struct {
Org string `json:"org"`
Name string `json:"name"`
Constraint string `json:"constraint"`
Version string `json:"version"`
}
func (v *versionCheckRequestPayload) getMapKey() string {
return fmt.Sprintf("%s/%s/%s", v.Org, v.Name, v.Stream)
}
type responseManifestAnnotations map[string]string
type responseManifestConfig struct {
MediaType string `json:"mediaType"`
Digest string `json:"digest"`
Size int `json:"size"`
}
type responseManifestLayer struct {
responseManifestConfig
Annotations responseManifestAnnotations `json:"annotations"`
}
type responseManifest struct {
SchemaVersion int `json:"schemaVersion"`
Config responseManifestConfig `json:"config"`
Layers []responseManifestLayer `json:"layers"`
Annotations responseManifestAnnotations `json:"annotations"`
}
type versionCheckResponsePayload struct {
versionCheckRequestPayload
Manifest responseManifest `json:"manifest"`
func (v *versionCheckCorePayload) getMapKey() string {
return fmt.Sprintf("%s/%s/%s", v.Org, v.Name, v.Constraint)
}

89
pkg/plugin/version_checker.go
View File

@@ -16,21 +16,27 @@ import (
"github.com/turbot/steampipe/pkg/utils"
)
const (
VersionCheckerSchema = "https"
VersionCheckerHost = "hub.steampipe.io"
VersionCheckerEndpoint = "api/plugin/version"
)
// VersionCheckReport ::
type VersionCheckReport struct {
Plugin *versionfile.InstalledVersion
CheckResponse versionCheckResponsePayload
CheckRequest versionCheckRequestPayload
CheckResponse versionCheckCorePayload
CheckRequest versionCheckCorePayload
}
func (vr *VersionCheckReport) ShortName() string {
return fmt.Sprintf("%s/%s", vr.CheckResponse.Org, vr.CheckResponse.Name)
}
func (vr *VersionCheckReport) ShortNameWithStream() string {
// dont show stream names for latest streams
if vr.CheckResponse.Stream != "latest" {
return fmt.Sprintf("%s/%s@%s", vr.CheckResponse.Org, vr.CheckResponse.Name, vr.CheckResponse.Stream)
func (vr *VersionCheckReport) ShortNameWithConstraint() string {
// dont show constraints for latest
if vr.CheckResponse.Constraint != "latest" {
return fmt.Sprintf("%s/%s@%s", vr.CheckResponse.Org, vr.CheckResponse.Name, vr.CheckResponse.Constraint)
}
return fmt.Sprintf("%s/%s", vr.CheckResponse.Org, vr.CheckResponse.Name)
}
@@ -77,7 +83,7 @@ func (v *VersionChecker) reportPluginUpdates(ctx context.Context) map[string]Ver
// retrieve the plugin version data from steampipe config
versionFileData, err := versionfile.LoadPluginVersionFile(ctx)
if err != nil {
log.Println("[TRACE]", "CheckAndReportPluginUpdates", "could not load versionfile")
log.Printf("[TRACE] reportPluginUpdates could not load version file: %s", err.Error())
return nil
}
@@ -103,7 +109,7 @@ func (v *VersionChecker) reportPluginUpdates(ctx context.Context) map[string]Ver
}
if err = versionFileData.Save(); err != nil {
log.Println("[TRACE]", "CheckAndReportPluginUpdates", "could not save versionfile")
log.Printf("[WARN] reportPluginUpdates could not save version file: %s", err.Error())
return nil
}
@@ -112,21 +118,17 @@ func (v *VersionChecker) reportPluginUpdates(ctx context.Context) map[string]Ver
func (v *VersionChecker) getLatestVersionsForPlugins(ctx context.Context, plugins []*versionfile.InstalledVersion) map[string]VersionCheckReport {
getMapKey := func(thisPayload versionCheckRequestPayload) string {
return fmt.Sprintf("%s/%s/%s", thisPayload.Org, thisPayload.Name, thisPayload.Stream)
}
var requestPayload []versionCheckRequestPayload
var requestPayload []versionCheckCorePayload
reports := map[string]VersionCheckReport{}
for _, ref := range plugins {
thisPayload := v.getPayloadFromInstalledData(ref)
requestPayload = append(requestPayload, thisPayload)
reports[getMapKey(thisPayload)] = VersionCheckReport{
reports[thisPayload.getMapKey()] = VersionCheckReport{
Plugin: ref,
CheckRequest: thisPayload,
CheckResponse: versionCheckResponsePayload{},
CheckResponse: versionCheckCorePayload{},
}
}
@@ -146,33 +148,28 @@ func (v *VersionChecker) getLatestVersionsForPlugins(ctx context.Context, plugin
return reports
}
func (v *VersionChecker) getPayloadFromInstalledData(plugin *versionfile.InstalledVersion) versionCheckRequestPayload {
func (v *VersionChecker) getPayloadFromInstalledData(plugin *versionfile.InstalledVersion) versionCheckCorePayload {
ref := ociinstaller.NewSteampipeImageRef(plugin.Name)
org, name, stream := ref.GetOrgNameAndStream()
payload := versionCheckRequestPayload{
Org: org,
Name: name,
Stream: stream,
Version: plugin.Version,
Digest: plugin.ImageDigest,
}
// if Digest field is missing, populate with dummy field
// - this will force and update an in doing so fix the versions.json
if payload.Digest == "" {
payload.Digest = "no digest"
org, name, constraint := ref.GetOrgNameAndConstraint()
payload := versionCheckCorePayload{
Org: org,
Name: name,
Constraint: constraint,
Version: plugin.Version,
}
return payload
}
func (v *VersionChecker) getVersionCheckURL() url.URL {
var u url.URL
u.Scheme = "https"
u.Host = "hub.steampipe.io"
u.Path = "api/plugin/version"
u.Scheme = VersionCheckerSchema
u.Host = VersionCheckerHost
u.Path = VersionCheckerEndpoint
return u
}
func (v *VersionChecker) requestServerForLatest(ctx context.Context, payload []versionCheckRequestPayload) ([]versionCheckResponsePayload, error) {
func (v *VersionChecker) requestServerForLatest(ctx context.Context, payload []versionCheckCorePayload) ([]versionCheckCorePayload, error) {
// Set a default timeout of 3 sec for the check request (in milliseconds)
sendRequestTo := v.getVersionCheckURL()
requestBody := utils.BuildRequestPayload(v.signature, map[string]interface{}{
@@ -197,7 +194,7 @@ func (v *VersionChecker) requestServerForLatest(ctx context.Context, payload []v
}
defer resp.Body.Close()
var responseData []versionCheckResponsePayload
var responseData []versionCheckCorePayload
err = json.Unmarshal(bodyBytes, &responseData)
if err != nil {
@@ -207,3 +204,29 @@ func (v *VersionChecker) requestServerForLatest(ctx context.Context, payload []v
return responseData, nil
}
func GetLatestPluginVersionByConstraint(ctx context.Context, installationID string, org string, name string, constraint string) (*ResolvedPluginVersion, error) {
vc := VersionChecker{signature: installationID}
payload := []versionCheckCorePayload{
{
Org: org,
Name: name,
Constraint: constraint,
Version: "0.0.0", // This is used by installer, version is required by the API, this makes sense as nothing installed.
},
}
orgAndName := fmt.Sprintf("%s/%s", org, name)
vcr, err := vc.requestServerForLatest(ctx, payload)
if err != nil {
return nil, err
}
if len(vcr) == 0 {
return nil, fmt.Errorf("no version found for %s with constraint %s", orgAndName, constraint)
}
v := vcr[0]
rpv := NewResolvedPluginVersion(orgAndName, v.Version, constraint)
return &rpv, nil
}

2
pkg/steampipeconfig/modconfig/plugin_version.go
View File

@@ -55,7 +55,7 @@ func (p *PluginVersion) Initialise(block *hcl.Block) hcl.Diagnostics {
})
}
// parse plugin name
p.Org, p.Name, _ = ociinstaller.NewSteampipeImageRef(p.RawName).GetOrgNameAndStream()
p.Org, p.Name, _ = ociinstaller.NewSteampipeImageRef(p.RawName).GetOrgNameAndConstraint()
return diags
}

2
pkg/steampipeconfig/modconfig/require.go
View File

@@ -152,7 +152,7 @@ func (r *Require) validatePluginVersions(modName string, plugins map[string]*Plu
// the mod requirement. If plugin is found nil error is returned.
func (r *Require) searchInstalledPluginForRequirement(modName string, requirement *PluginVersion, plugins map[string]*PluginVersionString) error {
for installedName, installed := range plugins {
org, name, _ := ociinstaller.NewSteampipeImageRef(installedName).GetOrgNameAndStream()
org, name, _ := ociinstaller.NewSteampipeImageRef(installedName).GetOrgNameAndConstraint()
if org != requirement.Org || name != requirement.Name {
// no point checking - different plugin
continue

2
pkg/steampipeconfig/plugin_remove.go
View File

@@ -25,7 +25,7 @@ func (r PluginRemoveReports) Print() {
if length > 0 {
fmt.Printf("\nUninstalled %s:\n", utils.Pluralize("plugin", length))
for _, report := range r {
org, name, _ := report.Image.GetOrgNameAndStream()
org, name, _ := report.Image.GetOrgNameAndConstraint()
fmt.Printf("* %s/%s\n", org, name)
staleConnections = append(staleConnections, report.Connections...)

8
pkg/steampipeconfig/steampipeconfig.go
View File

@@ -251,15 +251,15 @@ PluginOptions:
func (c *SteampipeConfig) ConnectionsForPlugin(pluginLongName string, pluginVersion *version.Version) []*modconfig.Connection {
var res []*modconfig.Connection
for _, con := range c.Connections {
// extract stream from plugin
// extract constraint from plugin
ref := ociinstaller.NewSteampipeImageRef(con.Plugin)
org, plugin, stream := ref.GetOrgNameAndStream()
org, plugin, constraint := ref.GetOrgNameAndConstraint()
longName := fmt.Sprintf("%s/%s", org, plugin)
if longName == pluginLongName {
if stream == "latest" {
if constraint == "latest" {
res = append(res, con)
} else {
connectionPluginVersion, err := version.NewVersion(stream)
connectionPluginVersion, err := version.NewVersion(constraint)
if err != nil && connectionPluginVersion.LessThanOrEqual(pluginVersion) {
res = append(res, con)
}

11
pkg/task/available_versions.go
View File

@@ -12,7 +12,6 @@ import (
"github.com/olekukonko/tablewriter"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/plugin"
"github.com/turbot/steampipe/pkg/steampipeconfig"
"github.com/turbot/steampipe/pkg/utils"
)
@@ -109,13 +108,9 @@ func ppNotificationLines() []string {
func (av *AvailableVersionCache) pluginNotificationMessage(ctx context.Context) []string {
var pluginsToUpdate []plugin.VersionCheckReport
// retrieve the plugin version data from steampipe config
pluginVersions := steampipeconfig.GlobalConfig.PluginVersions
for _, r := range av.PluginCache {
installedVersion := pluginVersions[r.Plugin.Name]
skip, _ := plugin.SkipUpdate(r)
if !skip && installedVersion.ImageDigest != r.CheckResponse.Digest {
if plugin.UpdateRequired(r) {
pluginsToUpdate = append(pluginsToUpdate, r)
}
}
@@ -153,7 +148,7 @@ func (av *AvailableVersionCache) getPluginNotificationLines(reports []plugin.Ver
line = fmt.Sprintf(
format,
thisName,
report.CheckResponse.Stream,
report.CheckResponse.Constraint,
constants.Bold(report.CheckResponse.Version),
)
} else {
@@ -166,7 +161,7 @@ func (av *AvailableVersionCache) getPluginNotificationLines(reports []plugin.Ver
line = fmt.Sprintf(
format,
thisName,
report.CheckResponse.Stream,
report.CheckResponse.Constraint,
constants.Bold(report.Plugin.Version),
constants.Bold(version),
)

2
pkg/task/runner.go
View File

@@ -97,7 +97,7 @@ func (r *Runner) run(ctx context.Context) {
if r.options.runUpdateCheck {
// check whether an updated version is available
r.runJobAsync(ctx, func(c context.Context) {
availableCliVersion, _ = fetchAvailableCLIVerion(ctx, r.currentState.InstallationID)
availableCliVersion, _ = fetchAvailableCLIVersion(ctx, r.currentState.InstallationID)
}, &waitGroup)
// check whether an updated version is available

2
pkg/task/version_checker.go
View File

@@ -31,7 +31,7 @@ type versionChecker struct {
}
// get the latest available version of the CLI
func fetchAvailableCLIVerion(ctx context.Context, installationId string) (*CLIVersionCheckResponse, error) {
func fetchAvailableCLIVersion(ctx context.Context, installationId string) (*CLIVersionCheckResponse, error) {
v := new(versionChecker)
v.signature = installationId
err := v.doCheckRequest(ctx)

4
pkg/version/version.go
View File

@@ -18,12 +18,12 @@ Also https://www.digitalocean.com/community/tutorials/using-ldflags-to-set-versi
**/
// The main version number that is being run at the moment.
var steampipeVersion = "0.22.2"
var steampipeVersion = "0.23.0"
// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
var prerelease = ""
var prerelease = "alpha.0"
// SteampipeVersion is an instance of semver.Version. This has the secondary
// benefit of verifying during tests and init time that our version is a

8
tests/acceptance/test_data/templates/expected_plugin_list_json.json
View File

@@ -1,15 +1,15 @@
{
"installed": [
{
"name": "hub.steampipe.io/plugins/turbot/bitbucket@0.3.1",
"version": "0.3.1",
"name": "hub.steampipe.io/plugins/turbot/bitbucket@0.7.1",
"version": "0.7.1",
"connections": [
"bitbucket"
]
},
{
"name": "hub.steampipe.io/plugins/turbot/hackernews@0.6.0",
"version": "0.6.0",
"name": "hub.steampipe.io/plugins/turbot/hackernews@0.8.0",
"version": "0.8.0",
"connections": [
"hackernews"
]

6
tests/acceptance/test_data/templates/expected_plugin_list_json_with_failed_plugins.json
View File

@@ -1,8 +1,8 @@
{
"installed": [
{
"name": "hub.steampipe.io/plugins/turbot/bitbucket@0.3.1",
"version": "0.3.1",
"name": "hub.steampipe.io/plugins/turbot/bitbucket@0.7.1",
"version": "0.7.1",
"connections": [
"bitbucket"
]
@@ -10,7 +10,7 @@
],
"failed": [
{
"name": "hub.steampipe.io/plugins/turbot/hackernews@0.6.0",
"name": "hub.steampipe.io/plugins/turbot/hackernews@0.8.0",
"reason": "plugin failed to start",
"connections": [
"hackernews"

6
tests/acceptance/test_data/templates/expected_plugin_list_json_with_missing_plugins.json
View File

@@ -1,8 +1,8 @@
{
"installed": [
{
"name": "hub.steampipe.io/plugins/turbot/bitbucket@0.3.1",
"version": "0.3.1",
"name": "hub.steampipe.io/plugins/turbot/bitbucket@0.7.1",
"version": "0.7.1",
"connections": [
"bitbucket"
]
@@ -10,7 +10,7 @@
],
"failed": [
{
"name": "hub.steampipe.io/plugins/turbot/hackernews@0.6.0",
"name": "hub.steampipe.io/plugins/turbot/hackernews@0.8.0",
"reason": "Not installed",
"connections": [
"hackernews"

4
tests/acceptance/test_data/templates/expected_plugin_list_table.txt
View File

@@ -1,6 +1,6 @@
+--------------------------------------------------+---------+-------------+
| Installed | Version | Connections |
+--------------------------------------------------+---------+-------------+
| hub.steampipe.io/plugins/turbot/bitbucket@0.3.1 | 0.3.1 | bitbucket |
| hub.steampipe.io/plugins/turbot/hackernews@0.6.0 | 0.6.0 | hackernews |
| hub.steampipe.io/plugins/turbot/bitbucket@0.7.1 | 0.7.1 | bitbucket |
| hub.steampipe.io/plugins/turbot/hackernews@0.8.0 | 0.8.0 | hackernews |
+--------------------------------------------------+---------+-------------+

4
tests/acceptance/test_data/templates/expected_plugin_list_table_with_failed_plugins.txt
View File

@@ -1,12 +1,12 @@
+-------------------------------------------------+---------+-------------+
| Installed | Version | Connections |
+-------------------------------------------------+---------+-------------+
| hub.steampipe.io/plugins/turbot/bitbucket@0.3.1 | 0.3.1 | bitbucket |
| hub.steampipe.io/plugins/turbot/bitbucket@0.7.1 | 0.7.1 | bitbucket |
+-------------------------------------------------+---------+-------------+
+--------------------------------------------------+-------------+------------------------+
| Failed | Connections | Reason |
+--------------------------------------------------+-------------+------------------------+
| hub.steampipe.io/plugins/turbot/hackernews@0.6.0 | hackernews | plugin failed to start |
| hub.steampipe.io/plugins/turbot/hackernews@0.8.0 | hackernews | plugin failed to start |
+--------------------------------------------------+-------------+------------------------+

4
tests/acceptance/test_data/templates/expected_plugin_list_table_with_missing_plugins.txt
View File

@@ -1,11 +1,11 @@
+-------------------------------------------------+---------+-------------+
| Installed | Version | Connections |
+-------------------------------------------------+---------+-------------+
| hub.steampipe.io/plugins/turbot/bitbucket@0.3.1 | 0.3.1 | bitbucket |
| hub.steampipe.io/plugins/turbot/bitbucket@0.7.1 | 0.7.1 | bitbucket |
+-------------------------------------------------+---------+-------------+
+--------------------------------------------------+-------------+---------------+
| Failed | Connections | Reason |
+--------------------------------------------------+-------------+---------------+
| hub.steampipe.io/plugins/turbot/hackernews@0.6.0 | hackernews | Not installed |
| hub.steampipe.io/plugins/turbot/hackernews@0.8.0 | hackernews | Not installed |
+--------------------------------------------------+-------------+---------------+

46
tests/acceptance/test_files/plugin.bats
View File

@@ -2,21 +2,45 @@ load "$LIB_BATS_ASSERT/load.bash"
load "$LIB_BATS_SUPPORT/load.bash"
@test "plugin install" {
run steampipe plugin install net
run steampipe plugin install chaos
assert_success
steampipe plugin uninstall net
steampipe plugin uninstall chaos
}
@test "plugin install from stream" {
run steampipe plugin install net@0.2
run steampipe plugin install chaos@0.4
assert_success
steampipe plugin uninstall net@0.2
steampipe plugin uninstall chaos@0.4
}
@test "plugin install from stream (prefixed with v)" {
run steampipe plugin install net@v0.2
run steampipe plugin install chaos@v0.4
assert_success
steampipe plugin uninstall net@0.2
steampipe plugin uninstall chaos@0.4
}
@test "plugin install from caret constraint" {
run steampipe plugin install chaos@^0.4
assert_success
steampipe plugin uninstall chaos@^0.4
}
@test "plugin install from tilde constraint" {
run steampipe plugin install chaos@~0.4.0
assert_success
steampipe plugin uninstall chaos@~0.4.0
}
@test "plugin install from wildcard constraint" {
run steampipe plugin install chaos@0.4.*
assert_success
steampipe plugin uninstall chaos@0.4.*
}
@test "plugin install gte constraint" {
run steampipe plugin install "chaos@>=0.4"
assert_success
steampipe plugin uninstall "chaos@>=0.4"
}
@test "create a local plugin, add connection and query" {
@@ -81,7 +105,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
# Create a copy of the install directory
copy_install_directory
steampipe plugin install hackernews@0.6.0 bitbucket@0.3.1 --progress=false --install-dir $MY_TEST_COPY
steampipe plugin install hackernews@0.8.0 bitbucket@0.7.1 --progress=false --install-dir $MY_TEST_COPY
# check table output
run steampipe plugin list --install-dir $MY_TEST_COPY
@@ -102,9 +126,9 @@ load "$LIB_BATS_SUPPORT/load.bash"
# Create a copy of the install directory
copy_install_directory
steampipe plugin install hackernews@0.6.0 bitbucket@0.3.1 --progress=false --install-dir $MY_TEST_COPY
steampipe plugin install hackernews@0.8.0 bitbucket@0.7.1 --progress=false --install-dir $MY_TEST_COPY
# uninstall a plugin but dont remove the config - to simulate the missing plugin scenario
steampipe plugin uninstall hackernews@0.6.0 --install-dir $MY_TEST_COPY
steampipe plugin uninstall hackernews@0.8.0 --install-dir $MY_TEST_COPY
# check table output
run steampipe plugin list --install-dir $MY_TEST_COPY
@@ -128,9 +152,9 @@ load "$LIB_BATS_SUPPORT/load.bash"
# Create a copy of the install directory
copy_install_directory
steampipe plugin install hackernews@0.6.0 bitbucket@0.3.1 --progress=false --install-dir $MY_TEST_COPY
steampipe plugin install hackernews@0.8.0 bitbucket@0.7.1 --progress=false --install-dir $MY_TEST_COPY
# remove the contents of a plugin execuatable to simulate the failed plugin scenario
cat /dev/null > $MY_TEST_COPY/plugins/hub.steampipe.io/plugins/turbot/hackernews@0.6.0/steampipe-plugin-hackernews.plugin
cat /dev/null > $MY_TEST_COPY/plugins/hub.steampipe.io/plugins/turbot/hackernews@0.8.0/steampipe-plugin-hackernews.plugin
# check table output
run steampipe plugin list --install-dir $MY_TEST_COPY