Migrate from GCP to GHCR as CR for Plugins. Closes #4232

cmd/plugin.go

@@ -241,8 +241,12 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) {
 	}()
 
 	// args to 'plugin install' -- one or more plugins to install
-	// plugin names can be simple names ('aws') for "standard" plugins,
+	// plugin names can be simple names for "standard" plugins, constraint suffixed names
-	// or full refs to the OCI image (us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0)
+	// or full refs to the OCI image
+	// - aws
+	// - aws@0.118.0
+	// - aws@^0.118
+	// - ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0.0
 	plugins := append([]string{}, args...)
 	showProgress := viper.GetBool(constants.ArgProgress)
 	installReports := make(display.PluginInstallReports, 0, len(plugins))
@@ -261,6 +265,13 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) {
 		}
 	}
 
+	state, err := installationstate.Load()
+	if err != nil {
+		error_helpers.ShowError(ctx, fmt.Errorf("could not load state"))
+		exitCode = constants.ExitCodePluginLoadingError
+		return
+	}
+
 	// a leading blank line - since we always output multiple lines
 	fmt.Println()
 	progressBars := uiprogress.New()
@@ -273,7 +284,30 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) {
 	for _, pluginName := range plugins {
 		installWaitGroup.Add(1)
 		bar := createProgressBar(pluginName, progressBars)
-		go doPluginInstall(ctx, bar, pluginName, installWaitGroup, reportChannel)
+
+		ref := ociinstaller.NewSteampipeImageRef(pluginName)
+		org, name, constraint := ref.GetOrgNameAndConstraint()
+		orgAndName := fmt.Sprintf("%s/%s", org, name)
+		var resolved plugin.ResolvedPluginVersion
+		if ref.IsFromSteampipeHub() {
+			rpv, err := plugin.GetLatestPluginVersionByConstraint(ctx, state.InstallationID, org, name, constraint)
+			if err != nil || rpv == nil {
+				report := &display.PluginInstallReport{
+					Plugin:         pluginName,
+					Skipped:        true,
+					SkipReason:     constants.InstallMessagePluginNotFound,
+					IsUpdateReport: false,
+				}
+				reportChannel <- report
+				installWaitGroup.Done()
+				continue
+			}
+			resolved = *rpv
+		} else {
+			resolved = plugin.NewResolvedPluginVersion(orgAndName, constraint, constraint)
+		}
+
+		go doPluginInstall(ctx, bar, pluginName, resolved, installWaitGroup, reportChannel)
 	}
 	go func() {
 		installWaitGroup.Wait()
@@ -312,7 +346,7 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) {
 	fmt.Println()
 }
 
-func doPluginInstall(ctx context.Context, bar *uiprogress.Bar, pluginName string, wg *sync.WaitGroup, returnChannel chan *display.PluginInstallReport) {
+func doPluginInstall(ctx context.Context, bar *uiprogress.Bar, pluginName string, resolvedPlugin plugin.ResolvedPluginVersion, wg *sync.WaitGroup, returnChannel chan *display.PluginInstallReport) {
 	var report *display.PluginInstallReport
 
 	pluginAlreadyInstalled, _ := plugin.Exists(ctx, pluginName)
@@ -343,7 +377,8 @@ func doPluginInstall(ctx context.Context, bar *uiprogress.Bar, pluginName string
 				return helpers.Resize(pluginInstallSteps[b.Current()-1], 20)
 			}
 		})
-		report = installPlugin(ctx, pluginName, false, bar)
+
+		report = installPlugin(ctx, resolvedPlugin, false, bar)
 	}
 	returnChannel <- report
 	wg.Done()
@@ -361,8 +396,12 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
 	}()
 
 	// args to 'plugin update' -- one or more plugins to update
-	// These can be simple names ('aws') for "standard" plugins,
+	// These can be simple names for "standard" plugins, constraint suffixed names
-	// or full refs to the OCI image (us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0)
+	// or full refs to the OCI image
+	// - aws
+	// - aws@0.118.0
+	// - aws@^0.118
+	// - ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0.0
 	plugins, err := resolveUpdatePluginsFromArgs(args)
 	showProgress := viper.GetBool(constants.ArgProgress)
 
@@ -376,7 +415,7 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
 		return
 	}
 
-	if len(plugins) > 0 && !(cmdconfig.Viper().GetBool("all")) && plugins[0] == "all" {
+	if len(plugins) > 0 && !(cmdconfig.Viper().GetBool(constants.ArgAll)) && plugins[0] == constants.ArgAll {
 		// improve the response to wrong argument "steampipe plugin update all"
 		fmt.Println()
 		exitCode = constants.ExitCodeInsufficientOrWrongInputs
@@ -404,8 +443,8 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
 	if cmdconfig.Viper().GetBool(constants.ArgAll) {
 		for k, v := range pluginVersions {
 			ref := ociinstaller.NewSteampipeImageRef(k)
-			org, name, stream := ref.GetOrgNameAndStream()
+			org, name, constraint := ref.GetOrgNameAndConstraint()
-			key := fmt.Sprintf("%s/%s@%s", org, name, stream)
+			key := fmt.Sprintf("%s/%s@%s", org, name, constraint)
 
 			plugins = append(plugins, key)
 			runUpdatesFor = append(runUpdatesFor, v)
@@ -469,7 +508,7 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
 	for _, key := range sorted {
 		report := reports[key]
 		updateWaitGroup.Add(1)
-		bar := createProgressBar(report.ShortNameWithStream(), progressBars)
+		bar := createProgressBar(report.ShortNameWithConstraint(), progressBars)
 		go doPluginUpdate(ctx, bar, report, updateWaitGroup, reportChannel)
 	}
 	go func() {
@@ -497,20 +536,8 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
 func doPluginUpdate(ctx context.Context, bar *uiprogress.Bar, pvr plugin.VersionCheckReport, wg *sync.WaitGroup, returnChannel chan *display.PluginInstallReport) {
 	var report *display.PluginInstallReport
 
-	if skip, skipReason := plugin.SkipUpdate(pvr); skip {
+	if plugin.UpdateRequired(pvr) {
-		bar.AppendFunc(func(b *uiprogress.Bar) string {
+		// update required, resolve version and install update
-			// set the progress bar to append itself with "Already Installed"
-			return helpers.Resize(skipReason, 30)
-		})
-		// set the progress bar to the maximum
-		bar.Set(len(pluginInstallSteps))
-		report = &display.PluginInstallReport{
-			Plugin:         fmt.Sprintf("%s@%s", pvr.CheckResponse.Name, pvr.CheckResponse.Stream),
-			Skipped:        true,
-			SkipReason:     skipReason,
-			IsUpdateReport: true,
-		}
-	} else {
 		bar.AppendFunc(func(b *uiprogress.Bar) string {
 			// set the progress bar to append itself  with the step underway
 			if b.Current() == 0 {
@@ -519,8 +546,24 @@ func doPluginUpdate(ctx context.Context, bar *uiprogress.Bar, pvr plugin.Version
 			}
 			return helpers.Resize(pluginInstallSteps[b.Current()-1], 20)
 		})
-		report = installPlugin(ctx, pvr.Plugin.Name, true, bar)
+		rp := plugin.NewResolvedPluginVersion(pvr.ShortName(), pvr.CheckResponse.Version, pvr.CheckResponse.Constraint)
+		report = installPlugin(ctx, rp, true, bar)
+	} else {
+		// update NOT required, return already installed report
+		bar.AppendFunc(func(b *uiprogress.Bar) string {
+			// set the progress bar to append itself with "Already Installed"
+			return helpers.Resize(constants.InstallMessagePluginLatestAlreadyInstalled, 30)
+		})
+		// set the progress bar to the maximum
+		bar.Set(len(pluginInstallSteps))
+		report = &display.PluginInstallReport{
+			Plugin:         fmt.Sprintf("%s@%s", pvr.CheckResponse.Name, pvr.CheckResponse.Constraint),
+			Skipped:        true,
+			SkipReason:     constants.InstallMessagePluginLatestAlreadyInstalled,
+			IsUpdateReport: true,
+		}
 	}
+
 	returnChannel <- report
 	wg.Done()
 }
@@ -533,7 +576,7 @@ func createProgressBar(plugin string, parentProgressBars *uiprogress.Progress) *
 	return bar
 }
 
-func installPlugin(ctx context.Context, pluginName string, isUpdate bool, bar *uiprogress.Bar) *display.PluginInstallReport {
+func installPlugin(ctx context.Context, resolvedPlugin plugin.ResolvedPluginVersion, isUpdate bool, bar *uiprogress.Bar) *display.PluginInstallReport {
 	// start a channel for progress publications from plugin.Install
 	progress := make(chan struct{}, 5)
 	defer func() {
@@ -549,10 +592,11 @@ func installPlugin(ctx context.Context, pluginName string, isUpdate bool, bar *u
 		}
 	}()
 
-	image, err := plugin.Install(ctx, pluginName, progress, ociinstaller.WithSkipConfig(viper.GetBool(constants.ArgSkipConfig)))
+	image, err := plugin.Install(ctx, resolvedPlugin, progress, ociinstaller.WithSkipConfig(viper.GetBool(constants.ArgSkipConfig)))
 	if err != nil {
 		msg := ""
-		_, name, stream := ociinstaller.NewSteampipeImageRef(pluginName).GetOrgNameAndStream()
+		// used to build data for the plugin install report to be used for display purposes
+		_, name, constraint := ociinstaller.NewSteampipeImageRef(resolvedPlugin.GetVersionTag()).GetOrgNameAndConstraint()
 		if isPluginNotFoundErr(err) {
 			exitCode = constants.ExitCodePluginNotFound
 			msg = constants.InstallMessagePluginNotFound
@@ -560,14 +604,15 @@ func installPlugin(ctx context.Context, pluginName string, isUpdate bool, bar *u
 			msg = err.Error()
 		}
 		return &display.PluginInstallReport{
-			Plugin:         fmt.Sprintf("%s@%s", name, stream),
+			Plugin:         fmt.Sprintf("%s@%s", name, constraint),
 			Skipped:        true,
 			SkipReason:     msg,
 			IsUpdateReport: isUpdate,
 		}
 	}
 
-	org, name, stream := image.ImageRef.GetOrgNameAndStream()
+	// used to build data for the plugin install report to be used for display purposes
+	org, name, _ := image.ImageRef.GetOrgNameAndConstraint()
 	versionString := ""
 	if image.Config.Plugin.Version != "" {
 		versionString = " v" + image.Config.Plugin.Version
@@ -577,7 +622,7 @@ func installPlugin(ctx context.Context, pluginName string, isUpdate bool, bar *u
 		docURL = fmt.Sprintf("https://%s/%s", org, name)
 	}
 	return &display.PluginInstallReport{
-		Plugin:         fmt.Sprintf("%s@%s", name, stream),
+		Plugin:         fmt.Sprintf("%s@%s", name, resolvedPlugin.Constraint),
 		Skipped:        false,
 		Version:        versionString,
 		DocURL:         docURL,

pkg/display/plugin_install.go

@@ -28,9 +28,9 @@ type PluginInstallReport struct {
 
 func (i *PluginInstallReport) skipString() string {
 	ref := ociinstaller.NewSteampipeImageRef(i.Plugin)
-	_, name, stream := ref.GetOrgNameAndStream()
+	_, name, constraint := ref.GetOrgNameAndConstraint()
 
-	return fmt.Sprintf("Plugin:   %s\nReason:   %s", fmt.Sprintf("%s@%s", name, stream), i.SkipReason)
+	return fmt.Sprintf("Plugin:   %s\nReason:   %s", fmt.Sprintf("%s@%s", name, constraint), i.SkipReason)
 }
 
 func (i *PluginInstallReport) installString() string {

pkg/filepaths/steampipe.go

@@ -57,7 +57,7 @@ func EnsurePluginDir() string {
 	return ensureSteampipeSubDir("plugins")
 }
 
-func EnsurePluginInstallDir(pluginImageDisplayRef  string) string {
+func EnsurePluginInstallDir(pluginImageDisplayRef string) string {
 	installDir := PluginInstallDir(pluginImageDisplayRef)
 
 	if _, err := os.Stat(installDir); os.IsNotExist(err) {
@@ -68,14 +68,14 @@ func EnsurePluginInstallDir(pluginImageDisplayRef  string) string {
 	return installDir
 }
 
-func PluginInstallDir(pluginImageDisplayRef  string) string {
+func PluginInstallDir(pluginImageDisplayRef string) string {
-	osSafePath := filepath.FromSlash(pluginImageDisplayRef )
+	osSafePath := filepath.FromSlash(pluginImageDisplayRef)
 
 	fullPath := filepath.Join(EnsurePluginDir(), osSafePath)
 	return fullPath
 }
 
-func PluginBinaryPath(pluginImageDisplayRef, pluginAlias  string) string {
+func PluginBinaryPath(pluginImageDisplayRef, pluginAlias string) string {
 	return filepath.Join(PluginInstallDir(pluginImageDisplayRef), PluginAliasToLongName(pluginAlias)+".plugin")
 }
 

pkg/ociinstaller/imageref.go

@@ -9,7 +9,7 @@ import (
 
 const (
 	DefaultImageTag            = "latest"
-	DefaultImageRepoActualURL  = "us-docker.pkg.dev/steampipe"
+	DefaultImageRepoActualURL  = "ghcr.io/turbot/steampipe"
 	DefaultImageRepoDisplayURL = "hub.steampipe.io"
 
 	DefaultImageOrg  = "turbot"
@@ -30,7 +30,7 @@ func NewSteampipeImageRef(ref string) *SteampipeImageRef {
 }
 
 // ActualImageRef returns the actual, physical full image ref
-// (us-docker.pkg.dev/steampipe/plugins/turbot/aws:1.0.0)
+// (ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0.0)
 func (r *SteampipeImageRef) ActualImageRef() string {
 	ref := r.requestedRef
 
@@ -63,6 +63,15 @@ func (r *SteampipeImageRef) DisplayImageRef() string {
 	return fullRef
 }
 
+// DisplayImageRefConstraintOverride returns a "friendly" user-facing version of the image ref
+// but with the version replaced by provided constraint
+// (hub.steampipe.io/plugins/turbot/aws@^1.0)
+func (r *SteampipeImageRef) DisplayImageRefConstraintOverride(constraint string) string {
+	dir := r.DisplayImageRef()
+	s := strings.Split(dir, "@")
+	return fmt.Sprintf("%s@%s", s[0], constraint)
+}
+
 func isDigestRef(ref string) bool {
 	return strings.Contains(ref, "@sha256:")
 }
@@ -84,19 +93,20 @@ func sanitizeRefStream(ref string) string {
 	return ref
 }
 
-func (r *SteampipeImageRef) IsFromSteampipeHub() (bool) {
+func (r *SteampipeImageRef) IsFromSteampipeHub() bool {
 	return strings.HasPrefix(r.DisplayImageRef(), constants.SteampipeHubOCIBase)
 }
 
-// GetOrgNameAndStream splits the full image reference into (org, name, stream)
+// GetOrgNameAndConstraint splits the full image reference into (org, name, constraint)
-func (r *SteampipeImageRef) GetOrgNameAndStream() (string, string, string) {
+// Constraint will be either a SemVer version (1.2.3) or a SemVer constraint (^0.4)
+func (r *SteampipeImageRef) GetOrgNameAndConstraint() (string, string, string) {
 	// plugin.Name looks like `hub.steampipe.io/plugins/turbot/aws@latest`
 	split := strings.Split(r.DisplayImageRef(), "/")
-	pluginNameAndStream := strings.Split(split[len(split)-1], "@")
+	pluginNameAndSuffix := strings.Split(split[len(split)-1], "@")
 	if r.IsFromSteampipeHub() {
-		return split[len(split)-2], pluginNameAndStream[0], pluginNameAndStream[1]
+		return split[len(split)-2], pluginNameAndSuffix[0], pluginNameAndSuffix[1]
 	}
-	return strings.Join(split[0:len(split)-1], "/"), pluginNameAndStream[0], pluginNameAndStream[1]
+	return strings.Join(split[0:len(split)-1], "/"), pluginNameAndSuffix[0], pluginNameAndSuffix[1]
 }
 
 // GetFriendlyName returns the minimum friendly name so that the original name can be rebuilt using preset defaults:
@@ -133,16 +143,16 @@ func getCondensedImageRef(imageRef string) string {
 }
 
 // possible formats include
-//		us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0
+//		ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0.0
-//		us-docker.pkg.dev/steampipe/plugin/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1
+//		ghcr.io/turbot/steampipe/plugins/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1
 //		turbot/aws:1.0.0
 //		turbot/aws
 //      dockerhub.org/myimage
 //      dockerhub.org/myimage:mytag
 //		aws:1.0.0
 //		aws
-//		us-docker.pkg.dev/steampipe/plugin/turbot/aws@1.0.0
+//		ghcr.io/turbot/steampipe/plugins/turbot/aws@1.0.0
-//		us-docker.pkg.dev/steampipe/plugin/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1
+//		ghcr.io/turbot/steampipe/plugins/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1
 //		turbot/aws@1.0.0
 //      dockerhub.org/myimage@mytag
 //		aws@1.0.0
@@ -169,7 +179,7 @@ func getFullImageRef(imagePath string) string {
 			return fmt.Sprintf("%s:%s", items[0], tag)
 		}
 		return fmt.Sprintf("%s/%s/%s/%s:%s", DefaultImageRepoActualURL, DefaultImageType, org, parts[len(parts)-1], tag)
-	default: //ex: us-docker.pkg.dev/steampipe/plugin/turbot/aws
+	default: //ex: ghcr.io/turbot/steampipe/plugins/turbot/aws
 		return fmt.Sprintf("%s:%s", items[0], tag)
 	}
 }

pkg/ociinstaller/imageref_test.go

@@ -32,31 +32,31 @@ func TestFriendlyImageRef(t *testing.T) {
 
 func TestActualImageRef(t *testing.T) {
 	cases := map[string]string{
-		"us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0":                                                                   "us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0",
+		"ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0.0":                                                                   "ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0.0",
-		"us-docker.pkg.dev/steampipe/plugin/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1": "us-docker.pkg.dev/steampipe/plugin/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1",
+		"ghcr.io/turbot/steampipe/plugins/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1": "ghcr.io/turbot/steampipe/plugins/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1",
-		"aws":                                 "us-docker.pkg.dev/steampipe/plugins/turbot/aws:latest",
+		"aws":                                 "ghcr.io/turbot/steampipe/plugins/turbot/aws:latest",
-		"aws:1":                               "us-docker.pkg.dev/steampipe/plugins/turbot/aws:1",
+		"aws:1":                               "ghcr.io/turbot/steampipe/plugins/turbot/aws:1",
-		"turbot/aws:1":                        "us-docker.pkg.dev/steampipe/plugins/turbot/aws:1",
+		"turbot/aws:1":                        "ghcr.io/turbot/steampipe/plugins/turbot/aws:1",
-		"turbot/aws:1.0":                      "us-docker.pkg.dev/steampipe/plugins/turbot/aws:1.0",
+		"turbot/aws:1.0":                      "ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0",
-		"turbot/aws:1.1.1":                    "us-docker.pkg.dev/steampipe/plugins/turbot/aws:1.1.1",
+		"turbot/aws:1.1.1":                    "ghcr.io/turbot/steampipe/plugins/turbot/aws:1.1.1",
-		"turbot/aws":                          "us-docker.pkg.dev/steampipe/plugins/turbot/aws:latest",
+		"turbot/aws":                          "ghcr.io/turbot/steampipe/plugins/turbot/aws:latest",
-		"mycompany/my-plugin":                 "us-docker.pkg.dev/steampipe/plugins/mycompany/my-plugin:latest",
+		"mycompany/my-plugin":                 "ghcr.io/turbot/steampipe/plugins/mycompany/my-plugin:latest",
-		"mycompany/my-plugin:some-random_tag": "us-docker.pkg.dev/steampipe/plugins/mycompany/my-plugin:some-random_tag",
+		"mycompany/my-plugin:some-random_tag": "ghcr.io/turbot/steampipe/plugins/mycompany/my-plugin:some-random_tag",
 		"dockerhub.org/myimage:mytag":         "dockerhub.org/myimage:mytag",
-		"us-docker.pkg.dev/steampipe/plugins/turbot/aws:latest": "us-docker.pkg.dev/steampipe/plugins/turbot/aws:latest",
+		"ghcr.io/turbot/steampipe/plugins/turbot/aws:latest": "ghcr.io/turbot/steampipe/plugins/turbot/aws:latest",
-		"hub.steampipe.io/plugins/turbot/aws:latest":            "us-docker.pkg.dev/steampipe/plugins/turbot/aws:latest",
+		"hub.steampipe.io/plugins/turbot/aws:latest":         "ghcr.io/turbot/steampipe/plugins/turbot/aws:latest",
-		"hub.steampipe.io/plugins/someoneelse/myimage:mytag":    "us-docker.pkg.dev/steampipe/plugins/someoneelse/myimage:mytag",
+		"hub.steampipe.io/plugins/someoneelse/myimage:mytag": "ghcr.io/turbot/steampipe/plugins/someoneelse/myimage:mytag",
 
-		"us-docker.pkg.dev/steampipe/plugin/turbot/aws@1.0.0": "us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0",
+		"ghcr.io/turbot/steampipe/plugins/turbot/aws@1.0.0": "ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0.0",
-		"aws@1":                               "us-docker.pkg.dev/steampipe/plugins/turbot/aws:1",
+		"aws@1":                               "ghcr.io/turbot/steampipe/plugins/turbot/aws:1",
-		"turbot/aws@1":                        "us-docker.pkg.dev/steampipe/plugins/turbot/aws:1",
+		"turbot/aws@1":                        "ghcr.io/turbot/steampipe/plugins/turbot/aws:1",
-		"turbot/aws@1.0":                      "us-docker.pkg.dev/steampipe/plugins/turbot/aws:1.0",
+		"turbot/aws@1.0":                      "ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0",
-		"turbot/aws@1.1.1":                    "us-docker.pkg.dev/steampipe/plugins/turbot/aws:1.1.1",
+		"turbot/aws@1.1.1":                    "ghcr.io/turbot/steampipe/plugins/turbot/aws:1.1.1",
-		"mycompany/my-plugin@some-random_tag": "us-docker.pkg.dev/steampipe/plugins/mycompany/my-plugin:some-random_tag",
+		"mycompany/my-plugin@some-random_tag": "ghcr.io/turbot/steampipe/plugins/mycompany/my-plugin:some-random_tag",
 		"dockerhub.org/myimage@mytag":         "dockerhub.org/myimage:mytag",
-		"us-docker.pkg.dev/steampipe/plugins/turbot/aws@latest": "us-docker.pkg.dev/steampipe/plugins/turbot/aws:latest",
+		"ghcr.io/turbot/steampipe/plugins/turbot/aws@latest": "ghcr.io/turbot/steampipe/plugins/turbot/aws:latest",
-		"hub.steampipe.io/plugins/turbot/aws@latest":            "us-docker.pkg.dev/steampipe/plugins/turbot/aws:latest",
+		"hub.steampipe.io/plugins/turbot/aws@latest":         "ghcr.io/turbot/steampipe/plugins/turbot/aws:latest",
-		"hub.steampipe.io/plugins/someoneelse/myimage@mytag":    "us-docker.pkg.dev/steampipe/plugins/someoneelse/myimage:mytag",
+		"hub.steampipe.io/plugins/someoneelse/myimage@mytag": "ghcr.io/turbot/steampipe/plugins/someoneelse/myimage:mytag",
 	}
 
 	for testCase, want := range cases {
@@ -74,8 +74,8 @@ func TestActualImageRef(t *testing.T) {
 
 func TestDisplayImageRef(t *testing.T) {
 	cases := map[string]string{
-		"us-docker.pkg.dev/steampipe/plugin/turbot/aws:1.0.0":                                                                   "hub.steampipe.io/plugin/turbot/aws@1.0.0",
+		"ghcr.io/turbot/steampipe/plugins/turbot/aws:1.0.0":                                                                   "hub.steampipe.io/plugins/turbot/aws@1.0.0",
-		"us-docker.pkg.dev/steampipe/plugin/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1": "hub.steampipe.io/plugin/turbot/aws@sha256-766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1",
+		"ghcr.io/turbot/steampipe/plugins/turbot/aws@sha256:766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1": "hub.steampipe.io/plugins/turbot/aws@sha256-766389c9dd892132c7e7b9124f446b9599a80863d466cd1d333a167dedf2c2b1",
 		"aws":                                 "hub.steampipe.io/plugins/turbot/aws@latest",
 		"aws:1":                               "hub.steampipe.io/plugins/turbot/aws@1",
 		"turbot/aws:1":                        "hub.steampipe.io/plugins/turbot/aws@1",
@@ -85,20 +85,20 @@ func TestDisplayImageRef(t *testing.T) {
 		"mycompany/my-plugin":                 "hub.steampipe.io/plugins/mycompany/my-plugin@latest",
 		"mycompany/my-plugin:some-random_tag": "hub.steampipe.io/plugins/mycompany/my-plugin@some-random_tag",
 		"dockerhub.org/myimage:mytag":         "dockerhub.org/myimage@mytag",
-		"us-docker.pkg.dev/steampipe/plugins/turbot/aws:latest": "hub.steampipe.io/plugins/turbot/aws@latest",
+		"ghcr.io/turbot/steampipe/plugins/turbot/aws:latest": "hub.steampipe.io/plugins/turbot/aws@latest",
-		"hub.steampipe.io/plugins/turbot/aws:latest":            "hub.steampipe.io/plugins/turbot/aws@latest",
+		"hub.steampipe.io/plugins/turbot/aws:latest":         "hub.steampipe.io/plugins/turbot/aws@latest",
-		"hub.steampipe.io/plugins/someoneelse/myimage:mytag":    "hub.steampipe.io/plugins/someoneelse/myimage@mytag",
+		"hub.steampipe.io/plugins/someoneelse/myimage:mytag": "hub.steampipe.io/plugins/someoneelse/myimage@mytag",
 
-		"us-docker.pkg.dev/steampipe/plugin/turbot/aws@1.0.0": "hub.steampipe.io/plugin/turbot/aws@1.0.0",
+		"ghcr.io/turbot/steampipe/plugins/turbot/aws@1.0.0": "hub.steampipe.io/plugins/turbot/aws@1.0.0",
 		"aws@1":                               "hub.steampipe.io/plugins/turbot/aws@1",
 		"turbot/aws@1":                        "hub.steampipe.io/plugins/turbot/aws@1",
 		"turbot/aws@1.0":                      "hub.steampipe.io/plugins/turbot/aws@1.0",
 		"turbot/aws@1.1.1":                    "hub.steampipe.io/plugins/turbot/aws@1.1.1",
 		"mycompany/my-plugin@some-random_tag": "hub.steampipe.io/plugins/mycompany/my-plugin@some-random_tag",
 		"dockerhub.org/myimage@mytag":         "dockerhub.org/myimage@mytag",
-		"us-docker.pkg.dev/steampipe/plugins/turbot/aws@latest": "hub.steampipe.io/plugins/turbot/aws@latest",
+		"ghcr.io/turbot/steampipe/plugins/turbot/aws@latest": "hub.steampipe.io/plugins/turbot/aws@latest",
-		"hub.steampipe.io/plugins/turbot/aws@latest":            "hub.steampipe.io/plugins/turbot/aws@latest",
+		"hub.steampipe.io/plugins/turbot/aws@latest":         "hub.steampipe.io/plugins/turbot/aws@latest",
-		"hub.steampipe.io/plugins/someoneelse/myimage@mytag":    "hub.steampipe.io/plugins/someoneelse/myimage@mytag",
+		"hub.steampipe.io/plugins/someoneelse/myimage@mytag": "hub.steampipe.io/plugins/someoneelse/myimage@mytag",
 
 		"aws@v1":            "hub.steampipe.io/plugins/turbot/aws@1",
 		"turbot/aws@v1":     "hub.steampipe.io/plugins/turbot/aws@1",
@@ -119,7 +119,7 @@ func TestDisplayImageRef(t *testing.T) {
 
 }
 
-func TestGetOrgNameAndStream(t *testing.T) {
+func TestGetOrgNameAndConstraint(t *testing.T) {
 	cases := map[string][3]string{
 		"hub.steampipe.io/plugins/turbot/aws@latest":   {"turbot", "aws", "latest"},
 		"turbot/aws@latest":                            {"turbot", "aws", "latest"},
@@ -136,10 +136,10 @@ func TestGetOrgNameAndStream(t *testing.T) {
 	for testCase, want := range cases {
 		t.Run(testCase, func(t *testing.T) {
 			r := NewSteampipeImageRef(testCase)
-			org, name, stream := r.GetOrgNameAndStream()
+			org, name, constraint := r.GetOrgNameAndConstraint()
-			got := [3]string{org, name, stream}
+			got := [3]string{org, name, constraint}
 			if got != want {
-				t.Errorf("TestGetOrgNameAndStream failed for case '%s': expected %s, got %s", testCase, want, got)
+				t.Errorf("TestGetOrgNameAndSuffix failed for case '%s': expected %s, got %s", testCase, want, got)
 			}
 		})
 	}

pkg/ociinstaller/plugin.go

@@ -14,14 +14,14 @@ import (
 	"time"
 
 	"github.com/turbot/steampipe/pkg/filepaths"
-	versionfile "github.com/turbot/steampipe/pkg/ociinstaller/versionfile"
+	"github.com/turbot/steampipe/pkg/ociinstaller/versionfile"
 	"github.com/turbot/steampipe/pkg/utils"
 )
 
 var versionFileUpdateLock = &sync.Mutex{}
 
 // InstallPlugin installs a plugin from an OCI Image
-func InstallPlugin(ctx context.Context, imageRef string, sub chan struct{}, opts ...PluginInstallOption) (*SteampipeImage, error) {
+func InstallPlugin(ctx context.Context, imageRef string, constraint string, sub chan struct{}, opts ...PluginInstallOption) (*SteampipeImage, error) {
 	config := &pluginInstallConfig{}
 	for _, opt := range opts {
 		opt(config)
@@ -44,21 +44,25 @@ func InstallPlugin(ctx context.Context, imageRef string, sub chan struct{}, opts
 		return nil, err
 	}
 
+	// update the image ref to include the constraint and use to get the plugin install path
+	constraintRef := image.ImageRef.DisplayImageRefConstraintOverride(constraint)
+	pluginPath := filepaths.EnsurePluginInstallDir(constraintRef)
+
 	sub <- struct{}{}
-	if err = installPluginBinary(image, tempDir.Path); err != nil {
+	if err = installPluginBinary(image, tempDir.Path, pluginPath); err != nil {
 		return nil, fmt.Errorf("plugin installation failed: %s", err)
 	}
 	sub <- struct{}{}
-	if err = installPluginDocs(image, tempDir.Path); err != nil {
+	if err = installPluginDocs(image, tempDir.Path, pluginPath); err != nil {
 		return nil, fmt.Errorf("plugin installation failed: %s", err)
 	}
 	if !config.skipConfigFile {
-		if err = installPluginConfigFiles(image, tempDir.Path); err != nil {
+		if err = installPluginConfigFiles(image, tempDir.Path, constraint); err != nil {
 			return nil, fmt.Errorf("plugin installation failed: %s", err)
 		}
 	}
 	sub <- struct{}{}
-	if err := updatePluginVersionFiles(ctx, image); err != nil {
+	if err := updatePluginVersionFiles(ctx, image, constraint); err != nil {
 		return nil, err
 	}
 	return image, nil
@@ -66,7 +70,7 @@ func InstallPlugin(ctx context.Context, imageRef string, sub chan struct{}, opts
 
 // updatePluginVersionFiles updates the global versions.json to add installation of the plugin
 // also adds a version file in the plugin installation directory with the information
-func updatePluginVersionFiles(ctx context.Context, image *SteampipeImage) error {
+func updatePluginVersionFiles(ctx context.Context, image *SteampipeImage, constraint string) error {
 	versionFileUpdateLock.Lock()
 	defer versionFileUpdateLock.Unlock()
 
@@ -76,7 +80,9 @@ func updatePluginVersionFiles(ctx context.Context, image *SteampipeImage) error
 		return err
 	}
 
-	pluginFullName := image.ImageRef.DisplayImageRef()
+	// For the full name we want the constraint (^0.4) used, not the resolved version (0.4.1)
+	// we override the DisplayImageRef with the constraint here.
+	pluginFullName := image.ImageRef.DisplayImageRefConstraintOverride(constraint)
 
 	installedVersion, ok := v.Plugins[pluginFullName]
 	if !ok {
@@ -106,9 +112,8 @@ func updatePluginVersionFiles(ctx context.Context, image *SteampipeImage) error
 	return v.Save()
 }
 
-func installPluginBinary(image *SteampipeImage, tempdir string) error {
+func installPluginBinary(image *SteampipeImage, tempDir string, destDir string) error {
-	sourcePath := filepath.Join(tempdir, image.Plugin.BinaryFile)
+	sourcePath := filepath.Join(tempDir, image.Plugin.BinaryFile)
-	destDir := filepaths.EnsurePluginInstallDir(image.ImageRef.DisplayImageRef())
 
 	// check if system is M1 - if so we need some special handling
 	isM1, err := utils.IsMacM1()
@@ -134,17 +139,15 @@ func installPluginBinary(image *SteampipeImage, tempdir string) error {
 	return nil
 }
 
-func installPluginDocs(image *SteampipeImage, tempdir string) error {
+func installPluginDocs(image *SteampipeImage, tempDir string, destDir string) error {
-	installTo := filepaths.EnsurePluginInstallDir(image.ImageRef.DisplayImageRef())
-
 	// if DocsDir is not set, then there are no docs.
 	if image.Plugin.DocsDir == "" {
 		return nil
 	}
 
 	// install the docs
-	sourcePath := filepath.Join(tempdir, image.Plugin.DocsDir)
+	sourcePath := filepath.Join(tempDir, image.Plugin.DocsDir)
-	destPath := filepath.Join(installTo, "docs")
+	destPath := filepath.Join(destDir, "docs")
 	if fileExists(destPath) {
 		os.RemoveAll(destPath)
 	}
@@ -154,7 +157,7 @@ func installPluginDocs(image *SteampipeImage, tempdir string) error {
 	return nil
 }
 
-func installPluginConfigFiles(image *SteampipeImage, tempdir string) error {
+func installPluginConfigFiles(image *SteampipeImage, tempdir string, constraint string) error {
 	installTo := filepaths.EnsureConfigDir()
 
 	// if ConfigFileDir is not set, then there are no config files.
@@ -172,7 +175,7 @@ func installPluginConfigFiles(image *SteampipeImage, tempdir string) error {
 	for _, obj := range objects {
 		sourceFile := filepath.Join(sourcePath, obj.Name())
 		destFile := filepath.Join(installTo, obj.Name())
-		if err := copyConfigFileUnlessExists(sourceFile, destFile, image.ImageRef); err != nil {
+		if err := copyConfigFileUnlessExists(sourceFile, destFile, constraint); err != nil {
 			return fmt.Errorf("could not copy config file from %s to %s", sourceFile, destFile)
 		}
 	}
@@ -180,7 +183,7 @@ func installPluginConfigFiles(image *SteampipeImage, tempdir string) error {
 	return nil
 }
 
-func copyConfigFileUnlessExists(sourceFile string, destFile string, ref *SteampipeImageRef) error {
+func copyConfigFileUnlessExists(sourceFile string, destFile string, constraint string) error {
 	if fileExists(destFile) {
 		return nil
 	}
@@ -193,7 +196,7 @@ func copyConfigFileUnlessExists(sourceFile string, destFile string, ref *Steampi
 		return fmt.Errorf("couldn't read source file permissions: %s", err)
 	}
 	// update the connection config with the correct plugin version
-	inputData = addPluginStreamToConfig(inputData, ref)
+	inputData = addPluginConstraintToConfig(inputData, constraint)
 	if err = os.WriteFile(destFile, inputData, inputStat.Mode()); err != nil {
 		return fmt.Errorf("writing to output file failed: %s", err)
 	}
@@ -203,15 +206,14 @@ func copyConfigFileUnlessExists(sourceFile string, destFile string, ref *Steampi
 // The default config files have the plugin set to the 'latest' stream (as this is what is installed by default)
 // When installing non-latest plugins, that property needs to be adjusted to the stream actually getting installed.
 // Otherwise, during plugin resolution, it will resolve to an incorrect plugin instance
-// (or none at at all, if  'latest' versions isn't installed)
+// (or none at all, if  'latest' versions isn't installed)
-func addPluginStreamToConfig(src []byte, ref *SteampipeImageRef) []byte {
+func addPluginConstraintToConfig(src []byte, constraint string) []byte {
-	_, _, stream := ref.GetOrgNameAndStream()
+	if constraint == "latest" {
-	if stream == "latest" {
 		return src
 	}
 
 	regex := regexp.MustCompile(`^(\s*)plugin\s*=\s*"(.*)"\s*$`)
-	substitution := fmt.Sprintf(`$1 plugin = "$2@%s"`, stream)
+	substitution := fmt.Sprintf(`$1 plugin = "$2@%s"`, constraint)
 
 	srcScanner := bufio.NewScanner(strings.NewReader(string(src)))
 	srcScanner.Split(bufio.ScanLines)
@@ -228,4 +230,3 @@ func addPluginStreamToConfig(src []byte, ref *SteampipeImageRef) []byte {
 	}
 	return destBuffer.Bytes()
 }
-

pkg/ociinstaller/plugin_test.go

@@ -2,6 +2,10 @@ package ociinstaller
 
 import (
 	"bytes"
+	"fmt"
+	"github.com/turbot/steampipe/pkg/filepaths"
+	"os"
+	"path/filepath"
 	"testing"
 )
 
@@ -37,16 +41,104 @@ var transformTests = map[string]transformTest{
 		pluginLineContent:                    []byte(`plugin = "chaos"`),
 		expectedTransformedPluginLineContent: []byte(`plugin = "chaos@0.2.0"`),
 	},
+	"^0.2": {
+		ref:                                  NewSteampipeImageRef("chaos@^0.2"),
+		pluginLineContent:                    []byte(`plugin = "chaos"`),
+		expectedTransformedPluginLineContent: []byte(`plugin = "chaos@^0.2"`),
+	},
+	">=0.2": {
+		ref:                                  NewSteampipeImageRef("chaos@>=0.2"),
+		pluginLineContent:                    []byte(`plugin = "chaos"`),
+		expectedTransformedPluginLineContent: []byte(`plugin = "chaos@>=0.2"`),
+	},
 }
 
 func TestAddPluginName(t *testing.T) {
 	for name, test := range transformTests {
 		sourcebytes := test.pluginLineContent
 		expectedBytes := test.expectedTransformedPluginLineContent
-		transformed := bytes.TrimSpace(addPluginStreamToConfig(sourcebytes, test.ref))
+		_, _, constraint := test.ref.GetOrgNameAndConstraint()
+		transformed := bytes.TrimSpace(addPluginConstraintToConfig(sourcebytes, constraint))
 
 		if !bytes.Equal(transformed, expectedBytes) {
 			t.Fatalf("%s failed - expected(%s) - got(%s)", name, test.expectedTransformedPluginLineContent, transformed)
 		}
 	}
 }
+
+func TestConstraintBasedFilePathsReadWrite(t *testing.T) {
+	tmpDir, err := os.MkdirTemp(os.TempDir(), "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+	filepaths.SteampipeDir = tmpDir
+
+	cases := make(map[string][]string)
+	fileContent := "test string"
+
+	cases["basic_checks"] = []string{
+		"latest",
+		"1.2.3",
+		"1.0",
+		"1",
+	}
+	cases["operators"] = []string{
+		"!=1.2.3",
+		">1.2.0",
+		">1.2",
+		">1",
+		">=1.2.0",
+		"<1.2.0",
+		"<=1.2.0",
+	}
+	cases["hyphen_range"] = []string{
+		"1.1-1.2.3",
+		"1.2.1-1.2.3",
+	}
+	cases["wild_cards"] = []string{
+		"*",
+		"1.x",
+		"1.*",
+		"1.1.x",
+		"1.1.*",
+		">=1.2.x",
+		"<=1.1.x",
+	}
+	cases["tilde_range"] = []string{
+		"~1",
+		"~1.1",
+		"~1.x",
+		"~1.1.1",
+		"~1.1.x",
+	}
+	cases["caret_range"] = []string{
+		"^1",
+		"^1.1",
+		"^1.x",
+		"^1.1.1",
+		"^1.1.*",
+	}
+
+	for category, testCases := range cases {
+		for _, testCase := range testCases {
+			constraintedDir := filepaths.EnsurePluginInstallDir(fmt.Sprintf("constraint-test:%s", testCase))
+			filePath := filepath.Join(constraintedDir, "test.txt")
+
+			// Write Test
+			err := os.WriteFile(filePath, []byte(fileContent), 0644)
+			if err != nil {
+				t.Fatalf("Write failed for constraint %s %s", category, testCase)
+			}
+			// Read Test
+			b, err := os.ReadFile(filePath)
+			if err != nil || string(b) != fileContent {
+				t.Fatalf("Read failed for constraint %s %s", category, testCase)
+			}
+			// tidy up
+			if err := os.RemoveAll(constraintedDir); err != nil {
+				t.Logf("Failed to remove test folder and contents: %s", constraintedDir)
+			}
+		}
+	}
+}

pkg/plugin/actions.go

@@ -18,12 +18,6 @@ import (
 	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
 )
 
-const (
-	DefaultImageTag     = "latest"
-	DefaultImageRepoURL = "us-docker.pkg.dev/steampipe/plugin"
-	DefaultImageOrg     = "turbot"
-)
-
 // Remove removes an installed plugin
 func Remove(ctx context.Context, image string, pluginConnections map[string][]*modconfig.Connection) (*steampipeconfig.PluginRemoveReport, error) {
 	statushooks.SetStatus(ctx, fmt.Sprintf("Removing plugin %s", image))
@@ -71,8 +65,9 @@ func Exists(ctx context.Context, plugin string) (bool, error) {
 }
 
 // Install installs a plugin in the local file system
-func Install(ctx context.Context, plugin string, sub chan struct{}, opts ...ociinstaller.PluginInstallOption) (*ociinstaller.SteampipeImage, error) {
+func Install(ctx context.Context, plugin ResolvedPluginVersion, sub chan struct{}, opts ...ociinstaller.PluginInstallOption) (*ociinstaller.SteampipeImage, error) {
-	image, err := ociinstaller.InstallPlugin(ctx, plugin, sub, opts...)
+	// Note: we pass the plugin info as strings here rather than passing the ResolvedPluginVersion struct as that causes circular dependency
+	image, err := ociinstaller.InstallPlugin(ctx, plugin.GetVersionTag(), plugin.Constraint, sub, opts...)
 	return image, err
 }
 

pkg/plugin/installed.go

@@ -14,7 +14,7 @@ func GetInstalledPlugins(ctx context.Context) (map[string]*modconfig.PluginVersi
 	installedPlugins := make(map[string]*modconfig.PluginVersionString)
 	installedPluginsData, _ := List(ctx, nil)
 	for _, plugin := range installedPluginsData {
-		org, name, _ := ociinstaller.NewSteampipeImageRef(plugin.Name).GetOrgNameAndStream()
+		org, name, _ := ociinstaller.NewSteampipeImageRef(plugin.Name).GetOrgNameAndConstraint()
 		pluginShortName := fmt.Sprintf("%s/%s", org, name)
 		installedPlugins[pluginShortName] = plugin.Version
 	}

pkg/plugin/resolved_version.go

@@ -0,0 +1,22 @@
+package plugin
+
+import "fmt"
+
+type ResolvedPluginVersion struct {
+	PluginName string
+	Version    string
+	Constraint string
+}
+
+func NewResolvedPluginVersion(pluginName string, version string, constraint string) ResolvedPluginVersion {
+	return ResolvedPluginVersion{
+		PluginName: pluginName,
+		Version:    version,
+		Constraint: constraint,
+	}
+}
+
+// GetVersionTag returns the <PluginName>:<Version> (turbot/chaos:0.4.1)
+func (r ResolvedPluginVersion) GetVersionTag() string {
+	return fmt.Sprintf("%s:%s", r.PluginName, r.Version)
+}

pkg/plugin/update.go

@@ -4,25 +4,24 @@ import (
 	"runtime"
 
 	"github.com/turbot/steampipe/pkg/constants"
-	"github.com/turbot/steampipe/pkg/ociinstaller"
 )
 
-// SkipUpdate determines if the latest version in a "stream"
+// UpdateRequired determines if the latest version in a "stream"
 // requires the plugin to update.
-func SkipUpdate(report VersionCheckReport) (bool, string) {
+func UpdateRequired(report VersionCheckReport) bool {
 
 	// 1) If there is an updated version ALWAYS update
-	if report.Plugin.ImageDigest != report.CheckResponse.Digest {
+	if report.Plugin.Version != report.CheckResponse.Version {
-		return false, ""
+		return true
 	}
 
 	// 2) If we are M1, current installed version is AMD, and ARM is available - update
-	if isRunningAsMacM1() && manifestHasM1Binary(report.CheckResponse.Manifest) && report.Plugin.BinaryArchitecture != constants.ArchARM64 {
+	if isRunningAsMacM1() && report.Plugin.BinaryArchitecture != constants.ArchARM64 {
-		return false, ""
+		return true
 	}
 
 	// 3) Otherwise skip
-	return true, constants.InstallMessagePluginLatestAlreadyInstalled
+	return false
 }
 
 // check to see if steampipe is running as a Mac/M1 build
@@ -32,12 +31,3 @@ func SkipUpdate(report VersionCheckReport) (bool, string) {
 func isRunningAsMacM1() bool {
 	return runtime.GOOS == constants.OSDarwin && runtime.GOARCH == constants.ArchARM64
 }
-
-func manifestHasM1Binary(manifest responseManifest) bool {
-	for _, rml := range manifest.Layers {
-		if rml.MediaType == ociinstaller.MediaTypePluginDarwinArm64Layer {
-			return true
-		}
-	}
-	return false
-}

pkg/plugin/version_check_payload.go

@@ -7,35 +7,13 @@ type versionCheckPayload interface {
 }
 
 // the payload that travels to-and-fro between steampipe and the server
-type versionCheckRequestPayload struct {
+type versionCheckCorePayload struct {
-	Org     string `json:"org"`
+	Org        string `json:"org"`
-	Name    string `json:"name"`
+	Name       string `json:"name"`
-	Stream  string `json:"stream"`
+	Constraint string `json:"constraint"`
-	Version string `json:"version"`
+	Version    string `json:"version"`
-	Digest  string `json:"digest"`
 }
 
-func (v *versionCheckRequestPayload) getMapKey() string {
+func (v *versionCheckCorePayload) getMapKey() string {
-	return fmt.Sprintf("%s/%s/%s", v.Org, v.Name, v.Stream)
+	return fmt.Sprintf("%s/%s/%s", v.Org, v.Name, v.Constraint)
-}
-
-type responseManifestAnnotations map[string]string
-type responseManifestConfig struct {
-	MediaType string `json:"mediaType"`
-	Digest    string `json:"digest"`
-	Size      int    `json:"size"`
-}
-type responseManifestLayer struct {
-	responseManifestConfig
-	Annotations responseManifestAnnotations `json:"annotations"`
-}
-type responseManifest struct {
-	SchemaVersion int                         `json:"schemaVersion"`
-	Config        responseManifestConfig      `json:"config"`
-	Layers        []responseManifestLayer     `json:"layers"`
-	Annotations   responseManifestAnnotations `json:"annotations"`
-}
-type versionCheckResponsePayload struct {
-	versionCheckRequestPayload
-	Manifest responseManifest `json:"manifest"`
 }

pkg/plugin/version_checker.go

@@ -16,21 +16,27 @@ import (
 	"github.com/turbot/steampipe/pkg/utils"
 )
 
+const (
+	VersionCheckerSchema   = "https"
+	VersionCheckerHost     = "hub.steampipe.io"
+	VersionCheckerEndpoint = "api/plugin/version"
+)
+
 // VersionCheckReport ::
 type VersionCheckReport struct {
 	Plugin        *versionfile.InstalledVersion
-	CheckResponse versionCheckResponsePayload
+	CheckResponse versionCheckCorePayload
-	CheckRequest  versionCheckRequestPayload
+	CheckRequest  versionCheckCorePayload
 }
 
 func (vr *VersionCheckReport) ShortName() string {
 	return fmt.Sprintf("%s/%s", vr.CheckResponse.Org, vr.CheckResponse.Name)
 }
 
-func (vr *VersionCheckReport) ShortNameWithStream() string {
+func (vr *VersionCheckReport) ShortNameWithConstraint() string {
-	// dont show stream names for latest streams
+	// dont show constraints for latest
-	if vr.CheckResponse.Stream != "latest" {
+	if vr.CheckResponse.Constraint != "latest" {
-		return fmt.Sprintf("%s/%s@%s", vr.CheckResponse.Org, vr.CheckResponse.Name, vr.CheckResponse.Stream)
+		return fmt.Sprintf("%s/%s@%s", vr.CheckResponse.Org, vr.CheckResponse.Name, vr.CheckResponse.Constraint)
 	}
 	return fmt.Sprintf("%s/%s", vr.CheckResponse.Org, vr.CheckResponse.Name)
 }
@@ -77,7 +83,7 @@ func (v *VersionChecker) reportPluginUpdates(ctx context.Context) map[string]Ver
 	// retrieve the plugin version data from steampipe config
 	versionFileData, err := versionfile.LoadPluginVersionFile(ctx)
 	if err != nil {
-		log.Println("[TRACE]", "CheckAndReportPluginUpdates", "could not load versionfile")
+		log.Printf("[TRACE] reportPluginUpdates could not load version file: %s", err.Error())
 		return nil
 	}
 
@@ -103,7 +109,7 @@ func (v *VersionChecker) reportPluginUpdates(ctx context.Context) map[string]Ver
 	}
 
 	if err = versionFileData.Save(); err != nil {
-		log.Println("[TRACE]", "CheckAndReportPluginUpdates", "could not save versionfile")
+		log.Printf("[WARN] reportPluginUpdates could not save version file: %s", err.Error())
 		return nil
 	}
 
@@ -112,21 +118,17 @@ func (v *VersionChecker) reportPluginUpdates(ctx context.Context) map[string]Ver
 
 func (v *VersionChecker) getLatestVersionsForPlugins(ctx context.Context, plugins []*versionfile.InstalledVersion) map[string]VersionCheckReport {
 
-	getMapKey := func(thisPayload versionCheckRequestPayload) string {
+	var requestPayload []versionCheckCorePayload
-		return fmt.Sprintf("%s/%s/%s", thisPayload.Org, thisPayload.Name, thisPayload.Stream)
-	}
-
-	var requestPayload []versionCheckRequestPayload
 	reports := map[string]VersionCheckReport{}
 
 	for _, ref := range plugins {
 		thisPayload := v.getPayloadFromInstalledData(ref)
 		requestPayload = append(requestPayload, thisPayload)
 
-		reports[getMapKey(thisPayload)] = VersionCheckReport{
+		reports[thisPayload.getMapKey()] = VersionCheckReport{
 			Plugin:        ref,
 			CheckRequest:  thisPayload,
-			CheckResponse: versionCheckResponsePayload{},
+			CheckResponse: versionCheckCorePayload{},
 		}
 	}
 
@@ -146,33 +148,28 @@ func (v *VersionChecker) getLatestVersionsForPlugins(ctx context.Context, plugin
 	return reports
 }
 
-func (v *VersionChecker) getPayloadFromInstalledData(plugin *versionfile.InstalledVersion) versionCheckRequestPayload {
+func (v *VersionChecker) getPayloadFromInstalledData(plugin *versionfile.InstalledVersion) versionCheckCorePayload {
 	ref := ociinstaller.NewSteampipeImageRef(plugin.Name)
-	org, name, stream := ref.GetOrgNameAndStream()
+	org, name, constraint := ref.GetOrgNameAndConstraint()
-	payload := versionCheckRequestPayload{
+	payload := versionCheckCorePayload{
-		Org:     org,
+		Org:        org,
-		Name:    name,
+		Name:       name,
-		Stream:  stream,
+		Constraint: constraint,
-		Version: plugin.Version,
+		Version:    plugin.Version,
-		Digest:  plugin.ImageDigest,
-	}
-	// if Digest field is missing, populate with dummy field
-	// - this will force and update an in doing so fix the versions.json
-	if payload.Digest == "" {
-		payload.Digest = "no digest"
 	}
+
 	return payload
 }
 
 func (v *VersionChecker) getVersionCheckURL() url.URL {
 	var u url.URL
-	u.Scheme = "https"
+	u.Scheme = VersionCheckerSchema
-	u.Host = "hub.steampipe.io"
+	u.Host = VersionCheckerHost
-	u.Path = "api/plugin/version"
+	u.Path = VersionCheckerEndpoint
 	return u
 }
 
-func (v *VersionChecker) requestServerForLatest(ctx context.Context, payload []versionCheckRequestPayload) ([]versionCheckResponsePayload, error) {
+func (v *VersionChecker) requestServerForLatest(ctx context.Context, payload []versionCheckCorePayload) ([]versionCheckCorePayload, error) {
 	// Set a default timeout of 3 sec for the check request (in milliseconds)
 	sendRequestTo := v.getVersionCheckURL()
 	requestBody := utils.BuildRequestPayload(v.signature, map[string]interface{}{
@@ -197,7 +194,7 @@ func (v *VersionChecker) requestServerForLatest(ctx context.Context, payload []v
 	}
 	defer resp.Body.Close()
 
-	var responseData []versionCheckResponsePayload
+	var responseData []versionCheckCorePayload
 
 	err = json.Unmarshal(bodyBytes, &responseData)
 	if err != nil {
@@ -207,3 +204,29 @@ func (v *VersionChecker) requestServerForLatest(ctx context.Context, payload []v
 
 	return responseData, nil
 }
+
+func GetLatestPluginVersionByConstraint(ctx context.Context, installationID string, org string, name string, constraint string) (*ResolvedPluginVersion, error) {
+	vc := VersionChecker{signature: installationID}
+	payload := []versionCheckCorePayload{
+		{
+			Org:        org,
+			Name:       name,
+			Constraint: constraint,
+			Version:    "0.0.0", // This is used by installer, version is required by the API, this makes sense as nothing installed.
+		},
+	}
+	orgAndName := fmt.Sprintf("%s/%s", org, name)
+
+	vcr, err := vc.requestServerForLatest(ctx, payload)
+	if err != nil {
+		return nil, err
+	}
+	if len(vcr) == 0 {
+		return nil, fmt.Errorf("no version found for %s with constraint %s", orgAndName, constraint)
+	}
+
+	v := vcr[0]
+	rpv := NewResolvedPluginVersion(orgAndName, v.Version, constraint)
+
+	return &rpv, nil
+}

pkg/steampipeconfig/modconfig/plugin_version.go

@@ -55,7 +55,7 @@ func (p *PluginVersion) Initialise(block *hcl.Block) hcl.Diagnostics {
 		})
 	}
 	// parse plugin name
-	p.Org, p.Name, _ = ociinstaller.NewSteampipeImageRef(p.RawName).GetOrgNameAndStream()
+	p.Org, p.Name, _ = ociinstaller.NewSteampipeImageRef(p.RawName).GetOrgNameAndConstraint()
 
 	return diags
 }

pkg/steampipeconfig/modconfig/require.go

@@ -152,7 +152,7 @@ func (r *Require) validatePluginVersions(modName string, plugins map[string]*Plu
 // the mod requirement. If plugin is found nil error is returned.
 func (r *Require) searchInstalledPluginForRequirement(modName string, requirement *PluginVersion, plugins map[string]*PluginVersionString) error {
 	for installedName, installed := range plugins {
-		org, name, _ := ociinstaller.NewSteampipeImageRef(installedName).GetOrgNameAndStream()
+		org, name, _ := ociinstaller.NewSteampipeImageRef(installedName).GetOrgNameAndConstraint()
 		if org != requirement.Org || name != requirement.Name {
 			// no point checking - different plugin
 			continue

pkg/steampipeconfig/plugin_remove.go

@@ -25,7 +25,7 @@ func (r PluginRemoveReports) Print() {
 	if length > 0 {
 		fmt.Printf("\nUninstalled %s:\n", utils.Pluralize("plugin", length))
 		for _, report := range r {
-			org, name, _ := report.Image.GetOrgNameAndStream()
+			org, name, _ := report.Image.GetOrgNameAndConstraint()
 			fmt.Printf("* %s/%s\n", org, name)
 			staleConnections = append(staleConnections, report.Connections...)
 

pkg/steampipeconfig/steampipeconfig.go

@@ -251,15 +251,15 @@ PluginOptions:
 func (c *SteampipeConfig) ConnectionsForPlugin(pluginLongName string, pluginVersion *version.Version) []*modconfig.Connection {
 	var res []*modconfig.Connection
 	for _, con := range c.Connections {
-		// extract stream from plugin
+		// extract constraint from plugin
 		ref := ociinstaller.NewSteampipeImageRef(con.Plugin)
-		org, plugin, stream := ref.GetOrgNameAndStream()
+		org, plugin, constraint := ref.GetOrgNameAndConstraint()
 		longName := fmt.Sprintf("%s/%s", org, plugin)
 		if longName == pluginLongName {
-			if stream == "latest" {
+			if constraint == "latest" {
 				res = append(res, con)
 			} else {
-				connectionPluginVersion, err := version.NewVersion(stream)
+				connectionPluginVersion, err := version.NewVersion(constraint)
 				if err != nil && connectionPluginVersion.LessThanOrEqual(pluginVersion) {
 					res = append(res, con)
 				}

pkg/task/available_versions.go

@@ -12,7 +12,6 @@ import (
 	"github.com/olekukonko/tablewriter"
 	"github.com/turbot/steampipe/pkg/constants"
 	"github.com/turbot/steampipe/pkg/plugin"
-	"github.com/turbot/steampipe/pkg/steampipeconfig"
 	"github.com/turbot/steampipe/pkg/utils"
 )
 
@@ -109,13 +108,9 @@ func ppNotificationLines() []string {
 
 func (av *AvailableVersionCache) pluginNotificationMessage(ctx context.Context) []string {
 	var pluginsToUpdate []plugin.VersionCheckReport
-	// retrieve the plugin version data from steampipe config
-	pluginVersions := steampipeconfig.GlobalConfig.PluginVersions
 
 	for _, r := range av.PluginCache {
-		installedVersion := pluginVersions[r.Plugin.Name]
+		if plugin.UpdateRequired(r) {
-		skip, _ := plugin.SkipUpdate(r)
-		if !skip && installedVersion.ImageDigest != r.CheckResponse.Digest {
 			pluginsToUpdate = append(pluginsToUpdate, r)
 		}
 	}
@@ -153,7 +148,7 @@ func (av *AvailableVersionCache) getPluginNotificationLines(reports []plugin.Ver
 			line = fmt.Sprintf(
 				format,
 				thisName,
-				report.CheckResponse.Stream,
+				report.CheckResponse.Constraint,
 				constants.Bold(report.CheckResponse.Version),
 			)
 		} else {
@@ -166,7 +161,7 @@ func (av *AvailableVersionCache) getPluginNotificationLines(reports []plugin.Ver
 			line = fmt.Sprintf(
 				format,
 				thisName,
-				report.CheckResponse.Stream,
+				report.CheckResponse.Constraint,
 				constants.Bold(report.Plugin.Version),
 				constants.Bold(version),
 			)

pkg/task/runner.go

@@ -97,7 +97,7 @@ func (r *Runner) run(ctx context.Context) {
 	if r.options.runUpdateCheck {
 		// check whether an updated version is available
 		r.runJobAsync(ctx, func(c context.Context) {
-			availableCliVersion, _ = fetchAvailableCLIVerion(ctx, r.currentState.InstallationID)
+			availableCliVersion, _ = fetchAvailableCLIVersion(ctx, r.currentState.InstallationID)
 		}, &waitGroup)
 
 		// check whether an updated version is available

pkg/task/version_checker.go

@@ -31,7 +31,7 @@ type versionChecker struct {
 }
 
 // get the latest available version of the CLI
-func fetchAvailableCLIVerion(ctx context.Context, installationId string) (*CLIVersionCheckResponse, error) {
+func fetchAvailableCLIVersion(ctx context.Context, installationId string) (*CLIVersionCheckResponse, error) {
 	v := new(versionChecker)
 	v.signature = installationId
 	err := v.doCheckRequest(ctx)

pkg/version/version.go

@@ -18,12 +18,12 @@ Also https://www.digitalocean.com/community/tutorials/using-ldflags-to-set-versi
 **/
 
 // The main version number that is being run at the moment.
-var steampipeVersion = "0.22.2"
+var steampipeVersion = "0.23.0"
 
 // A pre-release marker for the version. If this is "" (empty string)
 // then it means that it is a final release. Otherwise, this is a pre-release
 // such as "dev" (in development), "beta", "rc1", etc.
-var prerelease = ""
+var prerelease = "alpha.0"
 
 // SteampipeVersion is an instance of semver.Version. This has the secondary
 // benefit of verifying during tests and init time that our version is a

tests/acceptance/test_data/templates/expected_plugin_list_json.json

@@ -1,15 +1,15 @@
 {
   "installed": [
     {
-      "name": "hub.steampipe.io/plugins/turbot/bitbucket@0.3.1",
+      "name": "hub.steampipe.io/plugins/turbot/bitbucket@0.7.1",
-      "version": "0.3.1",
+      "version": "0.7.1",
       "connections": [
         "bitbucket"
       ]
     },
     {
-      "name": "hub.steampipe.io/plugins/turbot/hackernews@0.6.0",
+      "name": "hub.steampipe.io/plugins/turbot/hackernews@0.8.0",
-      "version": "0.6.0",
+      "version": "0.8.0",
       "connections": [
         "hackernews"
       ]

tests/acceptance/test_data/templates/expected_plugin_list_json_with_failed_plugins.json

@@ -1,8 +1,8 @@
 {
   "installed": [
     {
-      "name": "hub.steampipe.io/plugins/turbot/bitbucket@0.3.1",
+      "name": "hub.steampipe.io/plugins/turbot/bitbucket@0.7.1",
-      "version": "0.3.1",
+      "version": "0.7.1",
       "connections": [
         "bitbucket"
       ]
@@ -10,7 +10,7 @@
   ],
   "failed": [
     {
-      "name": "hub.steampipe.io/plugins/turbot/hackernews@0.6.0",
+      "name": "hub.steampipe.io/plugins/turbot/hackernews@0.8.0",
       "reason": "plugin failed to start",
       "connections": [
         "hackernews"

tests/acceptance/test_data/templates/expected_plugin_list_json_with_missing_plugins.json

@@ -1,8 +1,8 @@
 {
   "installed": [
     {
-      "name": "hub.steampipe.io/plugins/turbot/bitbucket@0.3.1",
+      "name": "hub.steampipe.io/plugins/turbot/bitbucket@0.7.1",
-      "version": "0.3.1",
+      "version": "0.7.1",
       "connections": [
         "bitbucket"
       ]
@@ -10,7 +10,7 @@
   ],
   "failed": [
     {
-      "name": "hub.steampipe.io/plugins/turbot/hackernews@0.6.0",
+      "name": "hub.steampipe.io/plugins/turbot/hackernews@0.8.0",
       "reason": "Not installed",
       "connections": [
         "hackernews"

tests/acceptance/test_data/templates/expected_plugin_list_table.txt

@@ -1,6 +1,6 @@
 +--------------------------------------------------+---------+-------------+
 | Installed                                        | Version | Connections |
 +--------------------------------------------------+---------+-------------+
-| hub.steampipe.io/plugins/turbot/bitbucket@0.3.1  | 0.3.1   | bitbucket   |
+| hub.steampipe.io/plugins/turbot/bitbucket@0.7.1  | 0.7.1   | bitbucket   |
-| hub.steampipe.io/plugins/turbot/hackernews@0.6.0 | 0.6.0   | hackernews  |
+| hub.steampipe.io/plugins/turbot/hackernews@0.8.0 | 0.8.0   | hackernews  |
 +--------------------------------------------------+---------+-------------+

tests/acceptance/test_data/templates/expected_plugin_list_table_with_failed_plugins.txt

@@ -1,12 +1,12 @@
 +-------------------------------------------------+---------+-------------+
 | Installed                                       | Version | Connections |
 +-------------------------------------------------+---------+-------------+
-| hub.steampipe.io/plugins/turbot/bitbucket@0.3.1 | 0.3.1   | bitbucket   |
+| hub.steampipe.io/plugins/turbot/bitbucket@0.7.1 | 0.7.1   | bitbucket   |
 +-------------------------------------------------+---------+-------------+
 
 +--------------------------------------------------+-------------+------------------------+
 | Failed                                           | Connections | Reason                 |
 +--------------------------------------------------+-------------+------------------------+
-| hub.steampipe.io/plugins/turbot/hackernews@0.6.0 | hackernews  | plugin failed to start |
+| hub.steampipe.io/plugins/turbot/hackernews@0.8.0 | hackernews  | plugin failed to start |
 +--------------------------------------------------+-------------+------------------------+
 

tests/acceptance/test_data/templates/expected_plugin_list_table_with_missing_plugins.txt

@@ -1,11 +1,11 @@
 +-------------------------------------------------+---------+-------------+
 | Installed                                       | Version | Connections |
 +-------------------------------------------------+---------+-------------+
-| hub.steampipe.io/plugins/turbot/bitbucket@0.3.1 | 0.3.1   | bitbucket   |
+| hub.steampipe.io/plugins/turbot/bitbucket@0.7.1 | 0.7.1   | bitbucket   |
 +-------------------------------------------------+---------+-------------+
 
 +--------------------------------------------------+-------------+---------------+
 | Failed                                           | Connections | Reason        |
 +--------------------------------------------------+-------------+---------------+
-| hub.steampipe.io/plugins/turbot/hackernews@0.6.0 | hackernews  | Not installed |
+| hub.steampipe.io/plugins/turbot/hackernews@0.8.0 | hackernews  | Not installed |
 +--------------------------------------------------+-------------+---------------+

tests/acceptance/test_files/plugin.bats

@@ -2,21 +2,45 @@ load "$LIB_BATS_ASSERT/load.bash"
 load "$LIB_BATS_SUPPORT/load.bash"
 
 @test "plugin install" {
-  run steampipe plugin install net
+  run steampipe plugin install chaos
   assert_success
-  steampipe plugin uninstall net
+  steampipe plugin uninstall chaos
 }
 
 @test "plugin install from stream" {
-  run steampipe plugin install net@0.2
+  run steampipe plugin install chaos@0.4
   assert_success
-  steampipe plugin uninstall net@0.2
+  steampipe plugin uninstall chaos@0.4
 }
 
 @test "plugin install from stream (prefixed with v)" {
-  run steampipe plugin install net@v0.2
+  run steampipe plugin install chaos@v0.4
   assert_success
-  steampipe plugin uninstall net@0.2
+  steampipe plugin uninstall chaos@0.4
+}
+
+@test "plugin install from caret constraint" {
+  run steampipe plugin install chaos@^0.4
+  assert_success
+  steampipe plugin uninstall chaos@^0.4
+}
+
+@test "plugin install from tilde constraint" {
+  run steampipe plugin install chaos@~0.4.0
+  assert_success
+  steampipe plugin uninstall chaos@~0.4.0
+}
+
+@test "plugin install from wildcard constraint" {
+  run steampipe plugin install chaos@0.4.*
+  assert_success
+  steampipe plugin uninstall chaos@0.4.*
+}
+
+@test "plugin install gte constraint" {
+  run steampipe plugin install "chaos@>=0.4"
+  assert_success
+  steampipe plugin uninstall "chaos@>=0.4"
 }
 
 @test "create a local plugin, add connection and query" {
@@ -81,7 +105,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
   # Create a copy of the install directory
   copy_install_directory
 
-  steampipe plugin install hackernews@0.6.0 bitbucket@0.3.1 --progress=false --install-dir $MY_TEST_COPY
+  steampipe plugin install hackernews@0.8.0 bitbucket@0.7.1 --progress=false --install-dir $MY_TEST_COPY
 
   # check table output
   run steampipe plugin list --install-dir $MY_TEST_COPY
@@ -102,9 +126,9 @@ load "$LIB_BATS_SUPPORT/load.bash"
   # Create a copy of the install directory
   copy_install_directory
 
-  steampipe plugin install hackernews@0.6.0 bitbucket@0.3.1 --progress=false --install-dir $MY_TEST_COPY
+  steampipe plugin install hackernews@0.8.0 bitbucket@0.7.1 --progress=false --install-dir $MY_TEST_COPY
   # uninstall a plugin but dont remove the config - to simulate the missing plugin scenario
-  steampipe plugin uninstall hackernews@0.6.0 --install-dir $MY_TEST_COPY
+  steampipe plugin uninstall hackernews@0.8.0 --install-dir $MY_TEST_COPY
 
   # check table output
   run steampipe plugin list --install-dir $MY_TEST_COPY
@@ -128,9 +152,9 @@ load "$LIB_BATS_SUPPORT/load.bash"
   # Create a copy of the install directory
   copy_install_directory
 
-  steampipe plugin install hackernews@0.6.0 bitbucket@0.3.1 --progress=false --install-dir $MY_TEST_COPY
+  steampipe plugin install hackernews@0.8.0 bitbucket@0.7.1 --progress=false --install-dir $MY_TEST_COPY
   # remove the contents of a plugin execuatable to simulate the failed plugin scenario
-  cat /dev/null > $MY_TEST_COPY/plugins/hub.steampipe.io/plugins/turbot/hackernews@0.6.0/steampipe-plugin-hackernews.plugin
+  cat /dev/null > $MY_TEST_COPY/plugins/hub.steampipe.io/plugins/turbot/hackernews@0.8.0/steampipe-plugin-hackernews.plugin
 
   # check table output
   run steampipe plugin list --install-dir $MY_TEST_COPY