jprdonnelly/steampipe
1
0
Fork 0
mirror of https://github.com/turbot/steampipe.git synced 2025-12-19 18:12:43 -05:00
Code Issues Packages Projects Releases Wiki Activity

working on it

Browse Source
This commit is contained in:
kai
2024-08-30 17:44:34 +01:00
committed by Puskar Basu
parent cd07bf20f1
commit fd94b2e2ec
33 changed files with 102 additions and 929 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
cmd/plugin.go
View File

@@ -26,10 +26,8 @@ import (
"github.com/turbot/steampipe/pkg/display"
"github.com/turbot/steampipe/pkg/error_helpers"
"github.com/turbot/steampipe/pkg/installationstate"
"github.com/turbot/steampipe/pkg/ociinstaller"
"github.com/turbot/steampipe/pkg/statushooks"
"github.com/turbot/steampipe/pkg/steampipeconfig"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
)
type installedPlugin struct {
@@ -288,10 +286,10 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) {
bar := createProgressBar(pluginName, progressBars)
ref := putils.NewImageRef(pluginName)
org, name, constraint := ref.GetOrgNameAndConstraint(constants.SteampipeHubOCIBase)
org, name, constraint := ref.GetOrgNameAndStream()
orgAndName := fmt.Sprintf("%s/%s", org, name)
var resolved plugin.ResolvedPluginVersion
if ref.IsFromTurbotHub(constants.SteampipeHubOCIBase) {
if ref.IsFromTurbotHub() {
rpv, err := plugin.GetLatestPluginVersionByConstraint(ctx, state.InstallationID, org, name, constraint)
if err != nil || rpv == nil {
report := &display.PluginInstallReport{
@@ -445,7 +443,7 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
if cmdconfig.Viper().GetBool(constants.ArgAll) {
for k, v := range pluginVersions {
ref := putils.NewImageRef(k)
org, name, constraint := ref.GetOrgNameAndConstraint(constants.SteampipeHubOCIBase)
org, name, constraint := ref.GetOrgNameAndStream()
key := fmt.Sprintf("%s/%s@%s", org, name, constraint)
plugins = append(plugins, key)
@@ -594,11 +592,11 @@ func installPlugin(ctx context.Context, resolvedPlugin plugin.ResolvedPluginVers
}
}()
image, err := plugin.Install(ctx, resolvedPlugin, progress, ociinstaller.WithSkipConfig(viper.GetBool(constants.ArgSkipConfig)))
image, err := plugin.Install(ctx, resolvedPlugin, progress, putils.WithSkipConfig(viper.GetBool(constants.ArgSkipConfig)))
if err != nil {
msg := ""
// used to build data for the plugin install report to be used for display purposes
_, name, constraint := putils.NewImageRef(resolvedPlugin.GetVersionTag()).GetOrgNameAndConstraint(constants.SteampipeHubOCIBase)
_, name, constraint := putils.NewImageRef(resolvedPlugin.GetVersionTag()).GetOrgNameAndStream()
if isPluginNotFoundErr(err) {
exitCode = constants.ExitCodePluginNotFound
msg = constants.InstallMessagePluginNotFound
@@ -614,13 +612,13 @@ func installPlugin(ctx context.Context, resolvedPlugin plugin.ResolvedPluginVers
}
// used to build data for the plugin install report to be used for display purposes
org, name, _ := image.ImageRef.GetOrgNameAndConstraint(constants.SteampipeHubOCIBase)
org, name, _ := image.ImageRef.GetOrgNameAndStream()
versionString := ""
if image.Config.Plugin.Version != "" {
versionString = " v" + image.Config.Plugin.Version
}
docURL := fmt.Sprintf("https://hub.steampipe.io/plugins/%s/%s", org, name)
if !image.ImageRef.IsFromTurbotHub(constants.SteampipeHubOCIBase) {
if !image.ImageRef.IsFromTurbotHub() {
docURL = fmt.Sprintf("https://%s/%s", org, name)
}
return &display.PluginInstallReport{
@@ -856,7 +854,7 @@ func getPluginList(ctx context.Context) (pluginList []plugin.PluginListItem, fai
// TODO do we really need to look at installed plugins - can't we just use the plugin connection map
// get a list of the installed plugins by inspecting the install location
// pass pluginConnectionMap so we can populate the connections for each plugin
pluginList, err := plugin.List(ctx, pluginConnectionMap)
pluginList, err := plugin.List(ctx, pluginConnectionMap, nil)
if err != nil {
res.Error = err
return nil, nil, nil, res

8
pkg/cmdconfig/app_specific.go
View File

@@ -3,17 +3,18 @@ package cmdconfig
import (
"github.com/Masterminds/semver/v3"
"github.com/spf13/viper"
"github.com/turbot/steampipe/pkg/constants"
"os"
"path/filepath"
"strings"
"github.com/turbot/go-kit/files"
"github.com/turbot/pipe-fittings/app_specific"
"github.com/turbot/pipe-fittings/cmdconfig"
"github.com/turbot/pipe-fittings/error_helpers"
)
// TODO kai FIX ME!!!!!
// SetAppSpecificConstants sets app specific constants defined in pipe-fittings
func SetAppSpecificConstants() {
app_specific.AppName = "steampipe"
@@ -26,9 +27,10 @@ func SetAppSpecificConstants() {
app_specific.ConfigExtension = ".tpc"
app_specific.PluginHub = constants.SteampipeHubOCIBase
// set the command pre and post hooks
cmdconfig.CustomPreRunHook = preRunHook
cmdconfig.CustomPostRunHook = postRunHook
//cmdconfig.CustomPreRunHook = preRunHook
//cmdconfig.CustomPostRunHook = postRunHook
// Version check
app_specific.VersionCheckHost = "hub.steampipe.io"

2
pkg/connection/interface.go
View File

@@ -4,9 +4,9 @@ import (
"context"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/turbot/pipe-fittings/error_helpers"
"github.com/turbot/pipe-fittings/plugin"
"github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto"
"github.com/turbot/steampipe/pkg/pluginmanager_service/grpc/shared"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
)
type pluginManager interface {

2
pkg/connection/limiter_map.go
View File

@@ -1,7 +1,7 @@
package connection
import (
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
"github.com/turbot/pipe-fittings/plugin"
"golang.org/x/exp/maps"
)

2
pkg/connection/plugin_limiter_map.go
View File

@@ -1,7 +1,7 @@
package connection
import (
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
"github.com/turbot/pipe-fittings/plugin"
"golang.org/x/exp/maps"
)

2
pkg/display/plugin_install.go
View File

@@ -28,7 +28,7 @@ type PluginInstallReport struct {
func (i *PluginInstallReport) skipString() string {
ref := ociinstaller.NewImageRef(i.Plugin)
_, name, constraint := ref.GetOrgNameAndConstraint(constants.SteampipeHubOCIBase)
_, name, constraint := ref.GetOrgNameAndStream()
return fmt.Sprintf("Plugin: %s\nReason: %s", fmt.Sprintf("%s@%s", name, constraint), i.SkipReason)
}

3
pkg/initialisation/init_data.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
error_helpers2 "github.com/turbot/pipe-fittings/error_helpers"
"github.com/turbot/pipe-fittings/plugin"
"github.com/turbot/steampipe/pkg/steampipeconfig"
"log"
"github.com/jackc/pgx/v5"
@@ -116,7 +117,7 @@ func (i *InitData) Init(ctx context.Context, invoker constants.Invoker, opts ...
statushooks.SetStatus(ctx, "Checking for required plugins")
log.Printf("[INFO] Checking for required plugins")
pluginsInstalled, err := plugin.GetInstalledPlugins(ctx)
pluginsInstalled, err := plugin.GetInstalledPlugins(ctx, steampipeconfig.GlobalConfig.PluginVersions)
if err != nil {
i.Result.Error = err
return

2
pkg/introspection/plugin_table_sql.go
View File

@@ -2,10 +2,10 @@ package introspection
import (
"fmt"
"github.com/turbot/pipe-fittings/plugin"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/db/db_common"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
)
func GetPluginTableCreateSql() db_common.QueryWithArgs {

2
pkg/introspection/rate_limiters_table_sql.go
View File

@@ -2,10 +2,10 @@ package introspection
import (
"fmt"
"github.com/turbot/pipe-fittings/plugin"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/db/db_common"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
)
func GetRateLimiterTableCreateSql() db_common.QueryWithArgs {

3
pkg/modinstaller/mod_installer.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/turbot/pipe-fittings/plugin"
"github.com/turbot/steampipe/pkg/steampipeconfig"
"log"
"os"
"path"
@@ -74,7 +75,7 @@ func NewModInstaller(ctx context.Context, opts *InstallOpts) (*ModInstaller, err
return nil, err
}
installedPlugins, err := plugin.GetInstalledPlugins(ctx)
installedPlugins, err := plugin.GetInstalledPlugins(ctx, steampipeconfig.GlobalConfig.PluginVersions)
if err != nil {
return nil, err
}

4
pkg/ociinstaller/assets.go
View File

@@ -21,7 +21,7 @@ func InstallAssets(ctx context.Context, assetsLocation string) error {
}()
// download the blobs
imageDownloader := NewOciDownloader()
imageDownloader := ociinstaller.NewOciDownloader()
image, err := imageDownloader.Download(ctx, ociinstaller.NewImageRef(constants.DashboardAssetsImageRef), ImageTypeAssets, tempDir.Path)
if err != nil {
return err
@@ -35,7 +35,7 @@ func InstallAssets(ctx context.Context, assetsLocation string) error {
return nil
}
func installAssetsFiles(image *SteampipeImage, tempdir string, dest string) error {
func installAssetsFiles(image *OciImage, tempdir string, dest string) error {
fileName := image.Assets.ReportUI
sourcePath := filepath.Join(tempdir, fileName)
if err := ociinstaller.MoveFolderWithinPartition(sourcePath, filepaths.EnsureDashboardAssetsDir()); err != nil {

13
pkg/ociinstaller/config.go
View File

@@ -1,13 +0,0 @@
package ociinstaller
type pluginInstallConfig struct {
skipConfigFile bool
}
type PluginInstallOption = func(config *pluginInstallConfig)
func WithSkipConfig(skipConfigFile bool) PluginInstallOption {
return func(o *pluginInstallConfig) {
o.skipConfigFile = skipConfigFile
}
}

6
pkg/ociinstaller/db.go
View File

@@ -21,7 +21,7 @@ func InstallDB(ctx context.Context, dblocation string) (string, error) {
}
}()
imageDownloader := NewOciDownloader()
imageDownloader := ociinstaller.NewOciDownloader()
// Download the blobs
image, err := imageDownloader.Download(ctx, ociinstaller.NewImageRef(constants.PostgresImageRef), ImageTypeDatabase, tempDir.Path)
@@ -40,7 +40,7 @@ func InstallDB(ctx context.Context, dblocation string) (string, error) {
return string(image.OCIDescriptor.Digest), nil
}
func updateVersionFileDB(image *SteampipeImage) error {
func updateVersionFileDB(image *OciImage) error {
timeNow := utils.FormatTime(time.Now())
v, err := versionfile.LoadDatabaseVersionFile()
if err != nil {
@@ -55,7 +55,7 @@ func updateVersionFileDB(image *SteampipeImage) error {
return v.Save()
}
func installDbFiles(image *SteampipeImage, tempDir string, dest string) error {
func installDbFiles(image *OciImage, tempDir string, dest string) error {
source := filepath.Join(tempDir, image.Database.ArchiveDir)
return ociinstaller.MoveFolderWithinPartition(source, dest)
}

6
pkg/ociinstaller/fdw.go
View File

@@ -24,7 +24,7 @@ func InstallFdw(ctx context.Context, dbLocation string) (string, error) {
}
}()
imageDownloader := NewOciDownloader()
imageDownloader := ociinstaller.NewOciDownloader()
// download the blobs.
image, err := imageDownloader.Download(ctx, ociinstaller.NewImageRef(constants.FdwImageRef), ImageTypeFdw, tempDir.Path)
@@ -44,7 +44,7 @@ func InstallFdw(ctx context.Context, dbLocation string) (string, error) {
return string(image.OCIDescriptor.Digest), nil
}
func updateVersionFileFdw(image *SteampipeImage) error {
func updateVersionFileFdw(image *OciImage) error {
timeNow := putils.FormatTime(time.Now())
v, err := versionfile.LoadDatabaseVersionFile()
if err != nil {
@@ -59,7 +59,7 @@ func updateVersionFileFdw(image *SteampipeImage) error {
return v.Save()
}
func installFdwFiles(image *SteampipeImage, tempdir string) error {
func installFdwFiles(image *OciImage, tempdir string) error {
fdwBinDir := filepaths.GetFDWBinaryDir()
fdwBinFileSourcePath := filepath.Join(tempdir, image.Fdw.BinaryFile)
fdwBinFileDestPath := filepath.Join(fdwBinDir, constants.FdwBinaryFileName)

4
pkg/ociinstaller/imageconfig.go
View File

@@ -6,6 +6,10 @@ import (
const DefaultConfigSchema string = "2020-11-18"
type OciImageConfig interface {
Name()
}
type config struct {
SchemaVersion string `json:"schemaVersion"`
Plugin *configPlugin `json:"plugin,omitempty"`

121
pkg/ociinstaller/ocidownloader.go
View File

@@ -1,121 +0,0 @@
package ociinstaller
import (
"context"
"encoding/json"
"log"
"strings"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
"github.com/turbot/steampipe/pkg/constants"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/content/file"
"oras.land/oras-go/v2/content/memory"
"oras.land/oras-go/v2/registry/remote"
"oras.land/oras-go/v2/registry/remote/auth"
"oras.land/oras-go/v2/registry/remote/credentials"
"oras.land/oras-go/v2/registry/remote/retry"
)
type ociDownloader struct {
resolver remotes.Resolver
Images []*SteampipeImage
}
// NewOciDownloader creates and returns a ociDownloader instance
func NewOciDownloader() *ociDownloader {
// oras uses containerd, which uses logrus and is set up to log
// warning and above. Set to ErrrLevel to get rid of unwanted error message
logrus.SetLevel(logrus.ErrorLevel)
return &ociDownloader{
resolver: docker.NewResolver(docker.ResolverOptions{}),
}
}
/*
Pull downloads the image from the given `ref` to the supplied `destDir`
Returns
imageDescription, configDescription, config, imageLayers, error
*/
func (o *ociDownloader) Pull(ctx context.Context, ref string, mediaTypes []string, destDir string) (*ocispec.Descriptor, *ocispec.Descriptor, []byte, []ocispec.Descriptor, error) {
split := strings.Split(ref, ":")
tag := split[len(split)-1]
log.Println("[TRACE] ociDownloader.Pull:", "preparing to pull ref", ref, "tag", tag, "destDir", destDir)
// Create the target file store
memoryStore := memory.New()
fileStore, err := file.NewWithFallbackStorage(destDir, memoryStore)
if err != nil {
return nil, nil, nil, nil, err
}
defer fileStore.Close()
// Connect to the remote repository
repo, err := remote.NewRepository(ref)
if err != nil {
return nil, nil, nil, nil, err
}
// Get credentials from the docker credentials store
storeOpts := credentials.StoreOptions{}
var credStore *credentials.DynamicStore
if strings.HasPrefix(ref, constants.BaseImageRef) {
credStore, err = credentials.NewStore("", storeOpts)
} else {
credStore, err = credentials.NewStoreFromDocker(storeOpts)
}
if err != nil {
return nil, nil, nil, nil, err
}
// Prepare the auth client for the registry and credential store
repo.Client = &auth.Client{
Client: retry.DefaultClient,
Cache: auth.DefaultCache,
Credential: credentials.Credential(credStore), // Use the credential store
}
// Copy from the remote repository to the file store
log.Println("[TRACE] ociDownloader.Pull:", "pulling...")
copyOpt := oras.DefaultCopyOptions
manifestDescriptor, err := oras.Copy(ctx, repo, tag, fileStore, tag, copyOpt)
if err != nil {
log.Println("[TRACE] ociDownloader.Pull:", "failed to pull", ref, err)
return nil, nil, nil, nil, err
}
log.Println("[TRACE] ociDownloader.Pull:", "manifest", manifestDescriptor.Digest, manifestDescriptor.MediaType)
// FIXME: this seems redundant as oras.Copy() already downloads all artifacts, but that's the only I found
// to access the manifest config. Also, it shouldn't be an issue as files are not re-downloaded.
manifestJson, err := content.FetchAll(ctx, fileStore, manifestDescriptor)
if err != nil {
log.Println("[TRACE] ociDownloader.Pull:", "failed to fetch manifest", manifestDescriptor)
return nil, nil, nil, nil, err
}
log.Println("[TRACE] ociDownloader.Pull:", "manifest content", string(manifestJson))
// Parse the fetched manifest
var manifest ocispec.Manifest
err = json.Unmarshal(manifestJson, &manifest)
if err != nil {
log.Println("[TRACE] ociDownloader.Pull:", "failed to unmarshall manifest", manifestJson)
return nil, nil, nil, nil, err
}
// Fetch the config from the file store
configData, err := content.FetchAll(ctx, fileStore, manifest.Config)
if err != nil {
log.Println("[TRACE] ociDownloader.Pull:", "failed to fetch config", manifest.Config.MediaType, err)
return nil, nil, nil, nil, err
}
log.Println("[TRACE] ociDownloader.Pull:", "config", string(configData))
return &manifestDescriptor, &manifest.Config, configData, manifest.Layers, err
}

234
pkg/ociinstaller/plugin.go
View File

@@ -1,234 +0,0 @@
package ociinstaller
import (
"bufio"
"bytes"
"context"
"fmt"
"github.com/turbot/pipe-fittings/ociinstaller"
versionfile2 "github.com/turbot/pipe-fittings/ociinstaller/versionfile"
"log"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
"github.com/turbot/pipe-fittings/utils"
putils "github.com/turbot/pipe-fittings/utils"
"github.com/turbot/steampipe/pkg/filepaths"
)
var versionFileUpdateLock = &sync.Mutex{}
// InstallPlugin installs a plugin from an OCI Image
func InstallPlugin(ctx context.Context, imageRef string, constraint string, sub chan struct{}, opts ...PluginInstallOption) (*SteampipeImage, error) {
config := &pluginInstallConfig{}
for _, opt := range opts {
opt(config)
}
tempDir := ociinstaller.NewTempDir(filepaths.EnsurePluginDir())
defer func() {
// send a last beacon to signal completion
sub <- struct{}{}
if err := tempDir.Delete(); err != nil {
log.Printf("[TRACE] Failed to delete temp dir '%s' after installing plugin: %s", tempDir, err)
}
}()
ref := ociinstaller.NewImageRef(imageRef)
imageDownloader := NewOciDownloader()
sub <- struct{}{}
image, err := imageDownloader.Download(ctx, ref, ImageTypePlugin, tempDir.Path)
if err != nil {
return nil, err
}
// update the image ref to include the constraint and use to get the plugin install path
constraintRef := image.ImageRef.DisplayImageRefConstraintOverride(constraint)
pluginPath := filepaths.EnsurePluginInstallDir(constraintRef)
sub <- struct{}{}
if err = installPluginBinary(image, tempDir.Path, pluginPath); err != nil {
return nil, fmt.Errorf("plugin installation failed: %s", err)
}
sub <- struct{}{}
if err = installPluginDocs(image, tempDir.Path, pluginPath); err != nil {
return nil, fmt.Errorf("plugin installation failed: %s", err)
}
if !config.skipConfigFile {
if err = installPluginConfigFiles(image, tempDir.Path, constraint); err != nil {
return nil, fmt.Errorf("plugin installation failed: %s", err)
}
}
sub <- struct{}{}
if err := updatePluginVersionFiles(ctx, image, constraint); err != nil {
return nil, err
}
return image, nil
}
// updatePluginVersionFiles updates the global versions.json to add installation of the plugin
// also adds a version file in the plugin installation directory with the information
func updatePluginVersionFiles(ctx context.Context, image *SteampipeImage, constraint string) error {
versionFileUpdateLock.Lock()
defer versionFileUpdateLock.Unlock()
timeNow := putils.FormatTime(time.Now())
v, err := versionfile2.LoadPluginVersionFile(ctx)
if err != nil {
return err
}
// For the full name we want the constraint (^0.4) used, not the resolved version (0.4.1)
// we override the DisplayImageRef with the constraint here.
pluginFullName := image.ImageRef.DisplayImageRefConstraintOverride(constraint)
installedVersion, ok := v.Plugins[pluginFullName]
if !ok {
installedVersion = versionfile2.EmptyInstalledVersion()
}
installedVersion.Name = pluginFullName
installedVersion.Version = image.Config.Plugin.Version
installedVersion.ImageDigest = string(image.OCIDescriptor.Digest)
installedVersion.BinaryDigest = image.Plugin.BinaryDigest
installedVersion.BinaryArchitecture = image.Plugin.BinaryArchitecture
installedVersion.InstalledFrom = image.ImageRef.ActualImageRef()
installedVersion.LastCheckedDate = timeNow
installedVersion.InstallDate = timeNow
v.Plugins[pluginFullName] = installedVersion
// Ensure that the version file is written to the plugin installation folder
// Having this file is important, since this can be used
// to compose the global version file if it is unavailable or unparseable
// This makes sure that in the event of corruption (global/individual) we don't end up
// losing all the plugin install data
if err := v.EnsurePluginVersionFile(installedVersion); err != nil {
return err
}
return v.Save()
}
func installPluginBinary(image *SteampipeImage, tempDir string, destDir string) error {
sourcePath := filepath.Join(tempDir, image.Plugin.BinaryFile)
// check if system is M1 - if so we need some special handling
isM1, err := utils.IsMacM1()
if err != nil {
return fmt.Errorf("failed to detect system architecture")
}
if isM1 {
// NOTE: for Mac M1 machines, if the binary is updated in place without deleting the existing file,
// the updated plugin binary may crash on execution - for an undetermined reason
// to avoid this, remove the existing plugin folder and re-create it
if err := os.RemoveAll(destDir); err != nil {
return fmt.Errorf("could not remove plugin folder")
}
if err := os.MkdirAll(destDir, 0755); err != nil {
return fmt.Errorf("could not create plugin folder")
}
}
// unzip the file into the plugin folder
if _, err := ociinstaller.Ungzip(sourcePath, destDir); err != nil {
return fmt.Errorf("could not unzip %s to %s", sourcePath, destDir)
}
return nil
}
func installPluginDocs(image *SteampipeImage, tempDir string, destDir string) error {
// if DocsDir is not set, then there are no docs.
if image.Plugin.DocsDir == "" {
return nil
}
// install the docs
sourcePath := filepath.Join(tempDir, image.Plugin.DocsDir)
destPath := filepath.Join(destDir, "docs")
if ociinstaller.FileExists(destPath) {
os.RemoveAll(destPath)
}
if err := ociinstaller.MoveFolderWithinPartition(sourcePath, destPath); err != nil {
return fmt.Errorf("could not copy %s to %s", sourcePath, destPath)
}
return nil
}
func installPluginConfigFiles(image *SteampipeImage, tempdir string, constraint string) error {
installTo := filepaths.EnsureConfigDir()
// if ConfigFileDir is not set, then there are no config files.
if image.Plugin.ConfigFileDir == "" {
return nil
}
// install config files (if they dont already exist)
sourcePath := filepath.Join(tempdir, image.Plugin.ConfigFileDir)
objects, err := os.ReadDir(sourcePath)
if err != nil {
return fmt.Errorf("couldn't read source dir: %s", err)
}
for _, obj := range objects {
sourceFile := filepath.Join(sourcePath, obj.Name())
destFile := filepath.Join(installTo, obj.Name())
if err := copyConfigFileUnlessExists(sourceFile, destFile, constraint); err != nil {
return fmt.Errorf("could not copy config file from %s to %s", sourceFile, destFile)
}
}
return nil
}
func copyConfigFileUnlessExists(sourceFile string, destFile string, constraint string) error {
if ociinstaller.FileExists(destFile) {
return nil
}
inputData, err := os.ReadFile(sourceFile)
if err != nil {
return fmt.Errorf("couldn't open source file: %s", err)
}
inputStat, err := os.Stat(sourceFile)
if err != nil {
return fmt.Errorf("couldn't read source file permissions: %s", err)
}
// update the connection config with the correct plugin version
inputData = addPluginConstraintToConfig(inputData, constraint)
if err = os.WriteFile(destFile, inputData, inputStat.Mode()); err != nil {
return fmt.Errorf("writing to output file failed: %s", err)
}
return nil
}
// The default config files have the plugin set to the 'latest' stream (as this is what is installed by default)
// When installing non-latest plugins, that property needs to be adjusted to the stream actually getting installed.
// Otherwise, during plugin resolution, it will resolve to an incorrect plugin instance
// (or none at all, if 'latest' versions isn't installed)
func addPluginConstraintToConfig(src []byte, constraint string) []byte {
if constraint == "latest" {
return src
}
regex := regexp.MustCompile(`^(\s*)plugin\s*=\s*"(.*)"\s*$`)
substitution := fmt.Sprintf(`$1 plugin = "$2@%s"`, constraint)
srcScanner := bufio.NewScanner(strings.NewReader(string(src)))
srcScanner.Split(bufio.ScanLines)
destBuffer := bytes.NewBufferString("")
for srcScanner.Scan() {
line := srcScanner.Text()
if regex.MatchString(line) {
line = regex.ReplaceAllString(line, substitution)
// remove the extra space we had to add to the substitution token
line = line[1:]
}
destBuffer.WriteString(fmt.Sprintf("%s\n", line))
}
return destBuffer.Bytes()
}

145
pkg/ociinstaller/plugin_test.go
View File

@@ -1,145 +0,0 @@
package ociinstaller
import (
"bytes"
"fmt"
"github.com/turbot/pipe-fittings/ociinstaller"
"github.com/turbot/steampipe/pkg/filepaths"
"os"
"path/filepath"
"testing"
)
type transformTest struct {
ref *ociinstaller.ImageRef
pluginLineContent []byte
expectedTransformedPluginLineContent []byte
}
var transformTests = map[string]transformTest{
"empty": {
ref: NewSteampipeImageRef("chaos"),
pluginLineContent: []byte(`plugin = "chaos"`),
expectedTransformedPluginLineContent: []byte(`plugin = "chaos"`),
},
"latest": {
ref: NewSteampipeImageRef("chaos@latest"),
pluginLineContent: []byte(`plugin = "chaos"`),
expectedTransformedPluginLineContent: []byte(`plugin = "chaos"`),
},
"0": {
ref: NewSteampipeImageRef("chaos@0"),
pluginLineContent: []byte(`plugin = "chaos"`),
expectedTransformedPluginLineContent: []byte(`plugin = "chaos@0"`),
},
"0.2": {
ref: NewSteampipeImageRef("chaos@0.2"),
pluginLineContent: []byte(`plugin = "chaos"`),
expectedTransformedPluginLineContent: []byte(`plugin = "chaos@0.2"`),
},
"0.2.0": {
ref: NewSteampipeImageRef("chaos@0.2.0"),
pluginLineContent: []byte(`plugin = "chaos"`),
expectedTransformedPluginLineContent: []byte(`plugin = "chaos@0.2.0"`),
},
"^0.2": {
ref: NewSteampipeImageRef("chaos@^0.2"),
pluginLineContent: []byte(`plugin = "chaos"`),
expectedTransformedPluginLineContent: []byte(`plugin = "chaos@^0.2"`),
},
">=0.2": {
ref: NewSteampipeImageRef("chaos@>=0.2"),
pluginLineContent: []byte(`plugin = "chaos"`),
expectedTransformedPluginLineContent: []byte(`plugin = "chaos@>=0.2"`),
},
}
func TestAddPluginName(t *testing.T) {
for name, test := range transformTests {
sourcebytes := test.pluginLineContent
expectedBytes := test.expectedTransformedPluginLineContent
_, _, constraint := test.ref.GetOrgNameAndConstraint(constants.SteampipeHubOCIBase)
transformed := bytes.TrimSpace(addPluginConstraintToConfig(sourcebytes, constraint))
if !bytes.Equal(transformed, expectedBytes) {
t.Fatalf("%s failed - expected(%s) - got(%s)", name, test.expectedTransformedPluginLineContent, transformed)
}
}
}
func TestConstraintBasedFilePathsReadWrite(t *testing.T) {
tmpDir, err := os.MkdirTemp(os.TempDir(), "test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
filepaths.SteampipeDir = tmpDir
cases := make(map[string][]string)
fileContent := "test string"
cases["basic_checks"] = []string{
"latest",
"1.2.3",
"1.0",
"1",
}
cases["operators"] = []string{
"!=1.2.3",
">1.2.0",
">1.2",
">1",
">=1.2.0",
"<1.2.0",
"<=1.2.0",
}
cases["hyphen_range"] = []string{
"1.1-1.2.3",
"1.2.1-1.2.3",
}
cases["wild_cards"] = []string{
"*",
"1.x",
"1.*",
"1.1.x",
"1.1.*",
">=1.2.x",
"<=1.1.x",
}
cases["tilde_range"] = []string{
"~1",
"~1.1",
"~1.x",
"~1.1.1",
"~1.1.x",
}
cases["caret_range"] = []string{
"^1",
"^1.1",
"^1.x",
"^1.1.1",
"^1.1.*",
}
for category, testCases := range cases {
for _, testCase := range testCases {
constraintedDir := filepaths.EnsurePluginInstallDir(fmt.Sprintf("constraint-test:%s", testCase))
filePath := filepath.Join(constraintedDir, "test.txt")
// Write Test
err := os.WriteFile(filePath, []byte(fileContent), 0644)
if err != nil {
t.Fatalf("Write failed for constraint %s %s", category, testCase)
}
// Read Test
b, err := os.ReadFile(filePath)
if err != nil || string(b) != fileContent {
t.Fatalf("Read failed for constraint %s %s", category, testCase)
}
// tidy up
if err := os.RemoveAll(constraintedDir); err != nil {
t.Logf("Failed to remove test folder and contents: %s", constraintedDir)
}
}
}
}

48
pkg/ociinstaller/steampipeimage.go
View File

@@ -13,15 +13,19 @@ import (
"github.com/turbot/steampipe/pkg/constants"
)
type SteampipeImage struct {
type OciImageData interface {
Type() ImageType
}
type OciImage struct {
OCIDescriptor *ocispec.Descriptor
ImageRef *ociinstaller.ImageRef
Config *config
Plugin *PluginImage
Database *DbImage
Fdw *HubImage
Assets *AssetsImage
resolver *remotes.Resolver
Config *config
Plugin *PluginImage
Database *DbImage
Fdw *FdwImage
Assets *AssetsImage
resolver *remotes.Resolver
}
type PluginImage struct {
@@ -33,24 +37,42 @@ type PluginImage struct {
LicenseFile string
}
func (s *PluginImage) Type() ImageType {
return ImageTypePlugin
}
type DbImage struct {
ArchiveDir string
ReadmeFile string
LicenseFile string
}
type HubImage struct {
func (s *DbImage) Type() ImageType {
return ImageTypeDatabase
}
type FdwImage struct {
BinaryFile string
ReadmeFile string
LicenseFile string
ControlFile string
SqlFile string
}
func (s *FdwImage) Type() ImageType {
return ImageTypeFdw
}
type AssetsImage struct {
ReportUI string
}
func (o *ociDownloader) newSteampipeImage() *SteampipeImage {
SteampipeImage := &SteampipeImage{
func (s *AssetsImage) Type() ImageType {
return ImageTypeAssets
}
func (o *ociinstaller.ociDownloader) newSteampipeImage() *OciImage {
SteampipeImage := &OciImage{
resolver: &o.resolver,
}
o.Images = append(o.Images, SteampipeImage)
@@ -66,7 +88,7 @@ const (
ImageTypePlugin ImageType = "plugin"
)
func (o *ociDownloader) Download(ctx context.Context, ref *ociinstaller.ImageRef, imageType ImageType, destDir string) (*SteampipeImage, error) {
func (o *ociinstaller.ociDownloader) Download(ctx context.Context, ref *ociinstaller.ImageRef, imageType ImageType, destDir string) (*OciImage, error) {
var mediaTypes []string
Image := o.newSteampipeImage()
Image.ImageRef = ref
@@ -154,8 +176,8 @@ func getDBImageData(layers []ocispec.Descriptor) (*DbImage, error) {
return res, nil
}
func getFdwImageData(layers []ocispec.Descriptor) (*HubImage, error) {
res := &HubImage{}
func getFdwImageData(layers []ocispec.Descriptor) (*FdwImage, error) {
res := &FdwImage{}
// get the binary (steampipe-postgres-fdw.so) info
mediaType, err := MediaTypeForPlatform("fdw")

2
pkg/pluginmanager_service/plugin_manager.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/turbot/pipe-fittings/filepaths"
"github.com/turbot/pipe-fittings/plugin"
"log"
"os"
"os/exec"
@@ -32,7 +33,6 @@ import (
pb "github.com/turbot/steampipe/pkg/pluginmanager_service/grpc/proto"
pluginshared "github.com/turbot/steampipe/pkg/pluginmanager_service/grpc/shared"
"github.com/turbot/steampipe/pkg/steampipeconfig"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
)
// PluginManager is the implementation of grpc.PluginManager

2
pkg/pluginmanager_service/plugin_manager_plugin_instance.go
View File

@@ -2,9 +2,9 @@ package pluginmanager_service
import (
"context"
"github.com/turbot/pipe-fittings/plugin"
"github.com/turbot/steampipe/pkg/connection"
"github.com/turbot/steampipe/pkg/db/db_local"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
"golang.org/x/exp/maps"
)

2
pkg/pluginmanager_service/plugin_manager_rate_limiters.go
View File

@@ -3,6 +3,7 @@ package pluginmanager_service
import (
"context"
"fmt"
"github.com/turbot/pipe-fittings/plugin"
"log"
"github.com/jackc/pgx/v5"
@@ -16,7 +17,6 @@ import (
"github.com/turbot/steampipe/pkg/db/db_local"
"github.com/turbot/steampipe/pkg/introspection"
pb "github.com/turbot/steampipe/pkg/pluginmanager_service/grpc/proto"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
"golang.org/x/exp/maps"
)

6
pkg/steampipeconfig/modconfig/mod.go
View File

@@ -2,6 +2,7 @@ package modconfig
import (
"fmt"
modconfig2 "github.com/turbot/pipe-fittings/plugin"
"os"
"path/filepath"
@@ -13,7 +14,6 @@ import (
"github.com/turbot/pipe-fittings/hclhelpers"
"github.com/turbot/steampipe/pkg/filepaths"
"github.com/zclconf/go-cty/cty"
)
// mod name used if a default mod is created for a workspace which does not define one explicitly
@@ -312,7 +312,7 @@ func (m *Mod) SetFilePath(modFilePath string) {
}
// ValidateRequirements validates that the current steampipe CLI and the installed plugins is compatible with the mod
func (m *Mod) ValidateRequirements(pluginVersionMap map[string]*PluginVersionString) []error {
func (m *Mod) ValidateRequirements(pluginVersionMap map[string]*modconfig2.PluginVersionString) []error {
validationErrors := []error{}
if err := m.validateSteampipeVersion(); err != nil {
validationErrors = append(validationErrors, err)
@@ -329,7 +329,7 @@ func (m *Mod) validateSteampipeVersion() error {
return m.Require.validateSteampipeVersion(m.Name())
}
func (m *Mod) validatePluginVersions(availablePlugins map[string]*PluginVersionString) []error {
func (m *Mod) validatePluginVersions(availablePlugins map[string]*modconfig2.PluginVersionString) []error {
if m.Require == nil {
return nil
}

104
pkg/steampipeconfig/modconfig/plugin.go
View File

@@ -1,104 +0,0 @@
package modconfig
import (
"strings"
"github.com/hashicorp/hcl/v2"
"github.com/turbot/pipe-fittings/hclhelpers"
"github.com/turbot/pipe-fittings/ociinstaller"
"golang.org/x/exp/maps"
)
type Plugin struct {
Instance string `hcl:"name,label" db:"plugin_instance"`
Alias string `hcl:"source,optional"`
MemoryMaxMb *int `hcl:"memory_max_mb,optional" db:"memory_max_mb"`
StartTimeout *int `hcl:"start_timeout,optional"`
Limiters []*RateLimiter `hcl:"limiter,block" db:"limiters"`
FileName *string `db:"file_name"`
StartLineNumber *int `db:"start_line_number"`
EndLineNumber *int `db:"end_line_number"`
// the image ref as a string
Plugin string `db:"plugin"`
// the actual plugin version, as a string
Version string `db:"version"`
}
// NewImplicitPlugin creates a default plugin config struct for a connection
// this is called when there is no explicit plugin config defined
// for a plugin which is used by a connection
func NewImplicitPlugin(connection *Connection, imageRef string) *Plugin {
return &Plugin{
// NOTE: set instance to image ref
Instance: imageRef,
Alias: connection.PluginAlias,
Plugin: imageRef,
}
}
func (l *Plugin) OnDecoded(block *hcl.Block) {
pluginRange := hclhelpers.BlockRange(block)
l.FileName = &pluginRange.Filename
l.StartLineNumber = &pluginRange.Start.Line
l.EndLineNumber = &pluginRange.End.Line
l.Plugin = ResolvePluginImageRef(l.Alias)
}
// IsDefault returns whether this config was created as a default
// i.e. a connection reference this plugin but there was no plugin config
// in this case the Instance will be the imageRef
func (l *Plugin) IsDefault() bool {
return l.Instance == l.Plugin
}
func (l *Plugin) FriendlyName() string {
return ociinstaller.NewImageRef(l.Plugin).GetFriendlyName()
}
func (l *Plugin) GetMaxMemoryBytes() int64 {
memoryMaxMb := 0
if l.MemoryMaxMb != nil {
memoryMaxMb = *l.MemoryMaxMb
}
return int64(1024 * 1024 * memoryMaxMb)
}
func (l *Plugin) GetStartTimeout() int64 {
startTimout := 0
if l.StartTimeout != nil {
startTimout = *l.StartTimeout
}
return int64(startTimout)
}
func (l *Plugin) GetLimiterMap() map[string]*RateLimiter {
res := make(map[string]*RateLimiter, len(l.Limiters))
for _, l := range l.Limiters {
res[l.Name] = l
}
return res
}
func (l *Plugin) Equals(other *Plugin) bool {
return l.Instance == other.Instance &&
l.Alias == other.Alias &&
l.GetMaxMemoryBytes() == other.GetMaxMemoryBytes() &&
l.GetStartTimeout() == other.GetStartTimeout() &&
l.Plugin == other.Plugin &&
// compare limiters ignoring order
maps.EqualFunc(l.GetLimiterMap(), other.GetLimiterMap(), func(l, r *RateLimiter) bool { return l.Equals(r) })
}
// ResolvePluginImageRef resolves the plugin image ref from the plugin alias
// (this handles the special case of locally developed plugins in the plugins/local folder)
func ResolvePluginImageRef(pluginAlias string) string {
if strings.HasPrefix(pluginAlias, `local/`) {
// if a local plugin is specified, return the plugin alias as the image ref.
// this will be used as the path to the plugin in the local folder
return pluginAlias
}
// ok so there is no plugin block reference - build the plugin image ref from the PluginAlias field
return ociinstaller.NewImageRef(pluginAlias).DisplayImageRef()
}

62
pkg/steampipeconfig/modconfig/plugin_version.go
View File

@@ -1,62 +0,0 @@
package modconfig
import (
"fmt"
"strings"
"github.com/Masterminds/semver/v3"
"github.com/hashicorp/hcl/v2"
"github.com/turbot/pipe-fittings/hclhelpers"
"github.com/turbot/pipe-fittings/ociinstaller"
"github.com/turbot/steampipe/pkg/constants"
)
type PluginVersion struct {
// the plugin name, as specified in the mod requires block. , e.g. turbot/mod1, aws
RawName string `cty:"name" hcl:"name,label"`
// the minumum version which satisfies the requirement
MinVersionString string `cty:"min_version" hcl:"min_version,optional"`
Constraint *semver.Constraints
// the org and name which are parsed from the raw name
Org string
Name string
DeclRange hcl.Range
}
func (p *PluginVersion) FullName() string {
if p.MinVersionString == "" {
return p.ShortName()
}
return fmt.Sprintf("%s@%s", p.ShortName(), p.MinVersionString)
}
func (p *PluginVersion) ShortName() string {
return fmt.Sprintf("%s/%s", p.Org, p.Name)
}
func (p *PluginVersion) String() string {
return fmt.Sprintf("plugin %s", p.FullName())
}
// Initialise parses the version and name properties
func (p *PluginVersion) Initialise(block *hcl.Block) hcl.Diagnostics {
var diags hcl.Diagnostics
p.DeclRange = hclhelpers.BlockRange(block)
// convert min version into constraint (including prereleases)
minVersion, err := semver.NewVersion(strings.TrimPrefix(p.MinVersionString, "v"))
if err == nil {
p.Constraint, err = semver.NewConstraint(fmt.Sprintf(">=%s-0", minVersion))
}
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("Invalid plugin version %s", p.MinVersionString),
Subject: &p.DeclRange,
})
}
// parse plugin name
p.Org, p.Name, _ = ociinstaller.NewImageRef(p.RawName).GetOrgNameAndConstraint(constants.SteampipeHubOCIBase)
return diags
}

47
pkg/steampipeconfig/modconfig/plugin_version_string.go
View File

@@ -1,47 +0,0 @@
package modconfig
import (
"github.com/Masterminds/semver/v3"
"github.com/turbot/steampipe-plugin-sdk/v5/sperr"
)
type PluginVersionString struct {
version string
semver *semver.Version
}
func NewPluginVersionString(version string) (*PluginVersionString, error) {
if smv, err := semver.NewVersion(version); err == nil {
pluginVersion := &PluginVersionString{
version: version,
semver: smv,
}
return pluginVersion, nil
}
if version == "local" {
return LocalPluginVersionString(), nil
}
return nil, sperr.New("version must be a valid semver or 'local'; got: %s", version)
}
func LocalPluginVersionString() *PluginVersionString {
return &PluginVersionString{
version: "local",
}
}
func (p *PluginVersionString) IsLocal() bool {
return p.semver == nil
}
func (p *PluginVersionString) IsSemver() bool {
return p.semver != nil
}
func (p *PluginVersionString) Semver() *semver.Version {
return p.semver
}
func (p *PluginVersionString) String() string {
return p.version
}

132
pkg/steampipeconfig/modconfig/rate_limiter.go
View File

@@ -1,132 +0,0 @@
package modconfig
import (
"sort"
"strings"
"github.com/hashicorp/hcl/v2"
"github.com/turbot/pipe-fittings/hclhelpers"
"github.com/turbot/pipe-fittings/ociinstaller"
"github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto"
)
const (
LimiterSourceConfig = "config"
LimiterSourcePlugin = "plugin"
LimiterStatusActive = "active"
LimiterStatusOverridden = "overridden"
)
type RateLimiter struct {
Name string `hcl:"name,label" db:"name"`
BucketSize *int64 `hcl:"bucket_size,optional" db:"bucket_size"`
FillRate *float32 `hcl:"fill_rate,optional" db:"fill_rate"`
MaxConcurrency *int64 `hcl:"max_concurrency,optional" db:"max_concurrency"`
Scope []string `hcl:"scope,optional" db:"scope"`
Where *string `hcl:"where,optional" db:"where"`
Plugin string `db:"plugin"`
PluginInstance string `db:"plugin_instance"`
FileName *string `db:"file_name" json:"-"`
StartLineNumber *int `db:"start_line_number" json:"-"`
EndLineNumber *int `db:"end_line_number" json:"-"`
Status string `db:"status"`
Source string `db:"source_type"`
ImageRef *ociinstaller.ImageRef `db:"-" json:"-"`
}
// RateLimiterFromProto converts the proto format RateLimiterDefinition into a Defintion
func RateLimiterFromProto(p *proto.RateLimiterDefinition, pluginImageRef, pluginInstance string) (*RateLimiter, error) {
var res = &RateLimiter{
Name: p.Name,
Scope: p.Scope,
}
if p.FillRate != 0 {
res.FillRate = &p.FillRate
res.BucketSize = &p.BucketSize
}
if p.MaxConcurrency != 0 {
res.MaxConcurrency = &p.MaxConcurrency
}
if p.Where != "" {
res.Where = &p.Where
}
if res.Scope == nil {
res.Scope = []string{}
}
// set ImageRef and Plugin fields
res.setPluginImageRef(pluginImageRef)
res.PluginInstance = pluginInstance
return res, nil
}
func (l *RateLimiter) AsProto() *proto.RateLimiterDefinition {
res := &proto.RateLimiterDefinition{
Name: l.Name,
Scope: l.Scope,
}
if l.MaxConcurrency != nil {
res.MaxConcurrency = *l.MaxConcurrency
}
if l.BucketSize != nil {
res.BucketSize = *l.BucketSize
}
if l.FillRate != nil {
res.FillRate = *l.FillRate
}
if l.Where != nil {
res.Where = *l.Where
}
return res
}
func (l *RateLimiter) OnDecoded(block *hcl.Block) {
limiterRange := hclhelpers.BlockRange(block)
l.FileName = &limiterRange.Filename
l.StartLineNumber = &limiterRange.Start.Line
l.EndLineNumber = &limiterRange.End.Line
if l.Scope == nil {
l.Scope = []string{}
}
l.Status = LimiterStatusActive
l.Source = LimiterSourceConfig
}
func (l *RateLimiter) scopeString() string {
scope := l.Scope
sort.Strings(scope)
return strings.Join(scope, "'")
}
func (l *RateLimiter) Equals(other *RateLimiter) bool {
return l.Name == other.Name &&
pointersHaveSameValue(l.BucketSize, other.BucketSize) &&
pointersHaveSameValue(l.FillRate, other.FillRate) &&
pointersHaveSameValue(l.MaxConcurrency, other.MaxConcurrency) &&
pointersHaveSameValue(l.Where, other.Where) &&
l.scopeString() == other.scopeString() &&
l.Plugin == other.Plugin &&
l.PluginInstance == other.PluginInstance &&
l.Source == other.Source
}
func (l *RateLimiter) SetPlugin(plugin *Plugin) {
l.PluginInstance = plugin.Instance
l.setPluginImageRef(plugin.Alias)
}
func (l *RateLimiter) setPluginImageRef(alias string) {
l.ImageRef = ociinstaller.NewImageRef(alias)
l.Plugin = l.ImageRef.DisplayImageRef()
}
func pointersHaveSameValue[T comparable](l, r *T) bool {
if l == nil {
return r == nil
}
if r == nil {
return false
}
return *l == *r
}

16
pkg/steampipeconfig/modconfig/require.go
View File

@@ -2,6 +2,7 @@ package modconfig
import (
"fmt"
modconfig2 "github.com/turbot/pipe-fittings/plugin"
"sort"
"github.com/Masterminds/semver/v3"
@@ -9,16 +10,15 @@ import (
"github.com/turbot/pipe-fittings/hclhelpers"
"github.com/turbot/pipe-fittings/ociinstaller"
"github.com/turbot/steampipe-plugin-sdk/v5/sperr"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/version"
)
// Require is a struct representing mod dependencies
type Require struct {
Plugins []*PluginVersion `hcl:"plugin,block"`
DeprecatedSteampipeVersionString string `hcl:"steampipe,optional"`
Steampipe *SteampipeRequire `hcl:"steampipe,block"`
Mods []*ModVersionConstraint `hcl:"mod,block"`
Plugins []*modconfig2.PluginVersion `hcl:"plugin,block"`
DeprecatedSteampipeVersionString string `hcl:"steampipe,optional"`
Steampipe *SteampipeRequire `hcl:"steampipe,block"`
Mods []*ModVersionConstraint `hcl:"mod,block"`
// map keyed by name [and alias]
modMap map[string]*ModVersionConstraint
// range of the definition of the require block
@@ -136,7 +136,7 @@ func (r *Require) validateSteampipeVersion(modName string) error {
}
// validatePluginVersions validates that for every plugin requirement there's at least one plugin installed
func (r *Require) validatePluginVersions(modName string, plugins map[string]*PluginVersionString) []error {
func (r *Require) validatePluginVersions(modName string, plugins map[string]*modconfig2.plugin) []error {
if len(r.Plugins) == 0 {
return nil
}
@@ -151,9 +151,9 @@ func (r *Require) validatePluginVersions(modName string, plugins map[string]*Plu
// searchInstalledPluginForRequirement returns plugin validation errors if no plugin is found which satisfies
// the mod requirement. If plugin is found nil error is returned.
func (r *Require) searchInstalledPluginForRequirement(modName string, requirement *PluginVersion, plugins map[string]*PluginVersionString) error {
func (r *Require) searchInstalledPluginForRequirement(modName string, requirement *modconfig2.PluginVersion, plugins map[string]*modconfig2.plugin) error {
for installedName, installed := range plugins {
org, name, _ := ociinstaller.NewImageRef(installedName).GetOrgNameAndConstraint(constants.SteampipeHubOCIBase)
org, name, _ := ociinstaller.NewImageRef(installedName).GetOrgNameAndStream()
if org != requirement.Org || name != requirement.Name {
// no point checking - different plugin
continue

2
pkg/steampipeconfig/parse/limiter.go
View File

@@ -3,7 +3,7 @@ package parse
import (
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
"github.com/turbot/pipe-fittings/plugin"
)
func DecodeLimiter(block *hcl.Block) (*modconfig.RateLimiter, hcl.Diagnostics) {

5
pkg/steampipeconfig/parse/plugin.go
View File

@@ -4,11 +4,12 @@ import (
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/hashicorp/hcl/v2/hclsyntax"
modconfig2 "github.com/turbot/pipe-fittings/plugin"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
)
func DecodePlugin(block *hcl.Block) (*modconfig.Plugin, hcl.Diagnostics) {
func DecodePlugin(block *hcl.Block) (*modconfig2.Plugin, hcl.Diagnostics) {
// manually decode child limiter blocks
content, rest, diags := block.Body.PartialContent(PluginBlockSchema)
if diags.HasErrors() {
@@ -17,7 +18,7 @@ func DecodePlugin(block *hcl.Block) (*modconfig.Plugin, hcl.Diagnostics) {
body := rest.(*hclsyntax.Body)
// decode attributes using 'rest' (these are automativally parsed so are not in schema)
var plugin = &modconfig.Plugin{
var plugin = &modconfig2.Plugin{
// default source and name to label
Instance: block.Labels[0],
Alias: block.Labels[0],

2
pkg/steampipeconfig/plugin_remove.go
View File

@@ -25,7 +25,7 @@ func (r PluginRemoveReports) Print() {
if length > 0 {
fmt.Printf("\nUninstalled %s:\n", utils.Pluralize("plugin", length))
for _, report := range r {
org, name, _ := report.Image.GetOrgNameAndConstraint(constants.SteampipeHubOCIBase)
org, name, _ := report.Image.GetOrgNameAndStream()
fmt.Printf("* %s/%s\n", org, name)
staleConnections = append(staleConnections, report.Connections...)

21
pkg/steampipeconfig/steampipeconfig.go
View File

@@ -2,6 +2,7 @@ package steampipeconfig
import (
"fmt"
modconfig2 "github.com/turbot/pipe-fittings/plugin"
"log"
"os"
"strings"
@@ -25,9 +26,9 @@ import (
type SteampipeConfig struct {
// map of plugin configs, keyed by plugin image ref
// (for each image ref we store an array of configs)
Plugins map[string][]*modconfig.Plugin
Plugins map[string][]*modconfig2.Plugin
// map of plugin configs, keyed by plugin instance
PluginsInstances map[string]*modconfig.Plugin
PluginsInstances map[string]*modconfig2.Plugin
// map of connection name to partially parsed connection config
Connections map[string]*modconfig.Connection
@@ -44,8 +45,8 @@ type SteampipeConfig struct {
func NewSteampipeConfig(commandName string) *SteampipeConfig {
return &SteampipeConfig{
Connections: make(map[string]*modconfig.Connection),
Plugins: make(map[string][]*modconfig.Plugin),
PluginsInstances: make(map[string]*modconfig.Plugin),
Plugins: make(map[string][]*modconfig2.Plugin),
PluginsInstances: make(map[string]*modconfig2.Plugin),
}
}
@@ -239,7 +240,7 @@ func (c *SteampipeConfig) ConnectionsForPlugin(pluginLongName string, pluginVers
for _, con := range c.Connections {
// extract constraint from plugin
ref := ociinstaller.NewImageRef(con.Plugin)
org, plugin, constraint := ref.GetOrgNameAndConstraint(constants.SteampipeHubOCIBase)
org, plugin, constraint := ref.GetOrgNameAndStream()
longName := fmt.Sprintf("%s/%s", org, plugin)
if longName == pluginLongName {
if constraint == "latest" {
@@ -278,7 +279,7 @@ func (c *SteampipeConfig) ConnectionList() []*modconfig.Connection {
// add a plugin config to PluginsInstances and Plugins
// NOTE: this returns an error if we already have a config with the same label
func (c *SteampipeConfig) addPlugin(plugin *modconfig.Plugin) error {
func (c *SteampipeConfig) addPlugin(plugin *modconfig2.Plugin) error {
if existingPlugin, exists := c.PluginsInstances[plugin.Instance]; exists {
return duplicatePluginError(existingPlugin, plugin)
}
@@ -302,7 +303,7 @@ func (c *SteampipeConfig) addPlugin(plugin *modconfig.Plugin) error {
return nil
}
func duplicatePluginError(existingPlugin, newPlugin *modconfig.Plugin) error {
func duplicatePluginError(existingPlugin, newPlugin *modconfig2.Plugin) error {
return sperr.New("duplicate plugin instance: '%s'\n\t(%s:%d)\n\t(%s:%d)",
existingPlugin.Instance, *existingPlugin.FileName, *existingPlugin.StartLineNumber,
*newPlugin.FileName, *newPlugin.StartLineNumber)
@@ -360,7 +361,7 @@ func (c *SteampipeConfig) initializePlugins() {
NOTE: if there is more than one config for the plugin this is an error
5) create a default config for the plugin (with the label set to the image ref)
*/
func (c *SteampipeConfig) resolvePluginInstanceForConnection(connection *modconfig.Connection) (*modconfig.Plugin, error) {
func (c *SteampipeConfig) resolvePluginInstanceForConnection(connection *modconfig.Connection) (*modconfig2.Plugin, error) {
// NOTE: at this point, c.Plugin is NOT populated, only either c.PluginAlias or c.PluginInstance
// we populate c.Plugin AFTER resolving the plugin
@@ -381,7 +382,7 @@ func (c *SteampipeConfig) resolvePluginInstanceForConnection(connection *modconf
}
// resolve the image ref (this handles the special case of locally developed plugins in the plugins/local folder)
imageRef := modconfig.ResolvePluginImageRef(connection.PluginAlias)
imageRef := modconfig2.ResolvePluginImageRef(connection.PluginAlias)
// verify the plugin is installed - if not return nil
if _, ok := c.PluginVersions[imageRef]; !ok {
@@ -405,7 +406,7 @@ func (c *SteampipeConfig) resolvePluginInstanceForConnection(connection *modconf
switch len(pluginsForImageRef) {
case 0:
// there is no plugin instance for this connection - add an implicit plugin instance
p := modconfig.NewImplicitPlugin(connection, imageRef)
p := modconfig2.NewImplicitPlugin(connection.PluginAlias, imageRef)
// now add to our map
if err := c.addPlugin(p); err != nil {

3
pkg/workspace/workspace_require.go
View File

@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"github.com/turbot/pipe-fittings/plugin"
"github.com/turbot/steampipe/pkg/steampipeconfig"
"sort"
"strings"
@@ -15,7 +16,7 @@ import (
func (w *Workspace) CheckRequiredPluginsInstalled(ctx context.Context) error {
// get the list of all installed plugins
installedPlugins, err := plugin.GetInstalledPlugins(ctx)
installedPlugins, err := plugin.GetInstalledPlugins(ctx, steampipeconfig.GlobalConfig.PluginVersions)
if err != nil {
return err
}