compiles and seems to work

connection state not populating
This commit is contained in:
kai
2024-09-02 17:19:51 +01:00
committed by Puskar Basu
parent 80ad514e1d
commit c5cc5a3e7a
14 changed files with 82 additions and 95 deletions

View File

@@ -9,10 +9,10 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/viper"
filehelpers "github.com/turbot/go-kit/files"
"github.com/turbot/pipe-fittings/app_specific"
"github.com/turbot/pipe-fittings/utils"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/error_helpers"
"github.com/turbot/steampipe/pkg/filepaths"
"github.com/turbot/steampipe/pkg/statushooks"
"github.com/turbot/steampipe/pkg/version"
)
@@ -52,7 +52,7 @@ func InitCmd() {
utils.LogTime("cmd.root.InitCmd start")
defer utils.LogTime("cmd.root.InitCmd end")
defaultInstallDir, err := filehelpers.Tildefy(filepaths.DefaultInstallDir)
defaultInstallDir, err := filehelpers.Tildefy(app_specific.DefaultInstallDir)
error_helpers.FailOnError(err)
rootCmd.SetVersionTemplate(fmt.Sprintf("Steampipe v%s\n", version.SteampipeVersion.String()))

View File

@@ -14,6 +14,7 @@ import (
"github.com/turbot/go-kit/helpers"
"github.com/turbot/pipe-fittings/utils"
"github.com/turbot/steampipe/cmd"
"github.com/turbot/steampipe/pkg/cmdconfig"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/error_helpers"
)
@@ -46,6 +47,8 @@ func main() {
cmd.InitCmd()
cmdconfig.SetAppSpecificConstants()
// execute the command
exitCode = cmd.Execute()
}

View File

@@ -1,36 +1,24 @@
package cmdconfig
import (
"github.com/Masterminds/semver/v3"
"github.com/spf13/viper"
"github.com/turbot/steampipe/pkg/constants"
"os"
"path/filepath"
"strings"
"github.com/turbot/go-kit/files"
"github.com/turbot/pipe-fittings/app_specific"
"github.com/turbot/pipe-fittings/error_helpers"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/version"
)
// TODO kai FIX ME!!!!!
// SetAppSpecificConstants sets app specific constants defined in pipe-fittings
func SetAppSpecificConstants() {
app_specific.AppName = "steampipe"
versionString := viper.GetString("main.version")
app_specific.AppVersion = semver.MustParse(versionString)
app_specific.AppVersion = version.SteampipeVersion
// set all app specific env var keys
app_specific.SetAppSpecificEnvVarKeys("STEAMPIPE_")
app_specific.ConfigExtension = ".tpc"
app_specific.ConfigExtension = ".spc"
app_specific.PluginHub = constants.SteampipeHubOCIBase
// set the command pre and post hooks
//cmdconfig.CustomPreRunHook = preRunHook
//cmdconfig.CustomPostRunHook = postRunHook
// Version check
app_specific.VersionCheckHost = "hub.steampipe.io"
@@ -41,20 +29,12 @@ func SetAppSpecificConstants() {
error_helpers.FailOnError(err)
app_specific.DefaultInstallDir = defaultInstallDir
// set the default config path
globalConfigPath := filepath.Join(defaultInstallDir, "config")
// check whether install-dir env has been set - if so, respect it
if envInstallDir, ok := os.LookupEnv(app_specific.EnvInstallDir); ok {
globalConfigPath = filepath.Join(envInstallDir, "config")
app_specific.InstallDir = envInstallDir
} else {
/*
NOTE:
If InstallDir is settable outside of default & env var, need to add
the following code to end of initGlobalConfig in init.go
app_specific.InstallDir = viper.GetString(constants.ArgInstallDir) at end of
*/
// NOTE: install dir will be set to configured value at the end of InitGlobalConfig
app_specific.InstallDir = defaultInstallDir
}
app_specific.DefaultConfigPath = strings.Join([]string{".", globalConfigPath}, ":")
}

View File

@@ -4,8 +4,6 @@ import (
"bytes"
"context"
"fmt"
error_helpers2 "github.com/turbot/pipe-fittings/error_helpers"
"github.com/turbot/pipe-fittings/ociinstaller/versionfile"
"io"
"log"
"os"
@@ -21,6 +19,9 @@ import (
filehelpers "github.com/turbot/go-kit/files"
"github.com/turbot/go-kit/helpers"
"github.com/turbot/go-kit/logging"
"github.com/turbot/pipe-fittings/app_specific"
perror_helpers "github.com/turbot/pipe-fittings/error_helpers"
"github.com/turbot/pipe-fittings/ociinstaller/versionfile"
"github.com/turbot/pipe-fittings/utils"
sdklogging "github.com/turbot/steampipe-plugin-sdk/v5/logging"
"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
@@ -131,7 +132,7 @@ func setMemoryLimit() {
// task run is complete
//
// runScheduledTasks skips running tasks if this instance is the plugin manager
func runScheduledTasks(ctx context.Context, cmd *cobra.Command, args []string, ew error_helpers2.ErrorAndWarnings) chan struct{} {
func runScheduledTasks(ctx context.Context, cmd *cobra.Command, args []string, ew perror_helpers.ErrorAndWarnings) chan struct{} {
// skip running the task runner if this is the plugin manager
// since it's supposed to be a daemon
if task.IsPluginManagerCmd(cmd) {
@@ -190,7 +191,7 @@ func envLogLevelSet() bool {
}
// initGlobalConfig reads in config file and ENV variables if set.
func initGlobalConfig() error_helpers2.ErrorAndWarnings {
func initGlobalConfig() perror_helpers.ErrorAndWarnings {
utils.LogTime("cmdconfig.initGlobalConfig start")
defer utils.LogTime("cmdconfig.initGlobalConfig end")
@@ -200,7 +201,7 @@ func initGlobalConfig() error_helpers2.ErrorAndWarnings {
// load workspace profile from the configured install dir
loader, err := getWorkspaceProfileLoader(ctx)
if err != nil {
return error_helpers2.NewErrorsAndWarning(err)
return perror_helpers.NewErrorsAndWarning(err)
}
// set global workspace profile
@@ -209,7 +210,7 @@ func initGlobalConfig() error_helpers2.ErrorAndWarnings {
// set-up viper with defaults from the env and default workspace profile
err = bootstrapViper(loader, cmd)
if err != nil {
return error_helpers2.NewErrorsAndWarning(err)
return perror_helpers.NewErrorsAndWarning(err)
}
// set global containing the configured install dir (create directory if needed)
@@ -264,8 +265,8 @@ func initGlobalConfig() error_helpers2.ErrorAndWarnings {
return loadConfigErrorsAndWarnings
}
func handleDeprecations() error_helpers2.ErrorAndWarnings {
var ew = error_helpers2.ErrorAndWarnings{}
func handleDeprecations() perror_helpers.ErrorAndWarnings {
var ew = perror_helpers.ErrorAndWarnings{}
// if deprecated cloud-token or cloud-host is set, show a warning and copy the value to the new arg
if viper.IsSet(constants.ArgCloudToken) {
if viper.IsSet(constants.ArgPipesToken) {
@@ -371,8 +372,8 @@ func getWorkspaceProfileLoader(ctx context.Context) (*steampipeconfig.WorkspaceP
// now validate config values have appropriate values
// (currently validates telemetry)
func validateConfig() error_helpers2.ErrorAndWarnings {
var res = error_helpers2.ErrorAndWarnings{}
func validateConfig() perror_helpers.ErrorAndWarnings {
var res = perror_helpers.ErrorAndWarnings{}
telemetry := viper.GetString(constants.ArgTelemetry)
if !helpers.StringSliceContains(constants.TelemetryLevels, telemetry) {
res.Error = sperr.New(`invalid value of 'telemetry' (%s), must be one of: %s`, telemetry, strings.Join(constants.TelemetryLevels, ", "))
@@ -396,7 +397,7 @@ func createLogger(logBuffer *bytes.Buffer, cmd *cobra.Command) {
level := sdklogging.LogLevel()
var logDestination io.Writer
if len(filepaths.SteampipeDir) == 0 {
if len(app_specific.InstallDir) == 0 {
// write to the buffer - this is to make sure that we don't lose logs
// till the time we get the log directory
logDestination = logBuffer
@@ -454,13 +455,16 @@ func ensureInstallDir() {
error_helpers.FailOnErrorWithMessage(err, fmt.Sprintf("could not create pipes installation directory: %s", pipesInstallDir))
}
// store as SteampipeDir and PipesInstallDir
filepaths.SteampipeDir = installDir
// store as app_specific.InstallDir and PipesInstallDir
app_specific.InstallDir = installDir
filepaths.PipesInstallDir = pipesInstallDir
// now we have loaded all config, set the install dir in the app specific config
app_specific.InstallDir = viper.GetString(constants.ArgInstallDir)
}
// displayDeprecationWarnings shows the deprecated warnings in a formatted way
func displayDeprecationWarnings(errorsAndWarnings error_helpers2.ErrorAndWarnings) {
func displayDeprecationWarnings(errorsAndWarnings perror_helpers.ErrorAndWarnings) {
if len(errorsAndWarnings.Warnings) > 0 {
fmt.Println(color.YellowString(fmt.Sprintf("\nDeprecation %s:", utils.Pluralize("warning", len(errorsAndWarnings.Warnings)))))
for _, warning := range errorsAndWarnings.Warnings {

View File

@@ -13,9 +13,9 @@ import (
"strings"
"time"
"github.com/turbot/go-kit/files"
"github.com/shirou/gopsutil/process"
"github.com/turbot/go-kit/files"
"github.com/turbot/pipe-fittings/app_specific"
putils "github.com/turbot/pipe-fittings/utils"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/error_helpers"
@@ -113,7 +113,7 @@ func killRunningDbInstance(ctx context.Context) error {
// check if the name of the process is prefixed with the $STEAMPIPE_INSTALL_DIR
// that means this is a steampipe service from this installation directory
if strings.HasPrefix(cmdLine, filepaths.SteampipeDir) {
if strings.HasPrefix(cmdLine, app_specific.InstallDir) {
log.Println("[TRACE] Terminating running postgres process")
if err := p.Kill(); err != nil {
error_helpers.ShowWarning(fmt.Sprintf("Failed to kill orphan postgres process PID %d", p.Pid))

View File

@@ -8,13 +8,13 @@ import (
"time"
filehelpers "github.com/turbot/go-kit/files"
"github.com/turbot/pipe-fittings/app_specific"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/filepaths"
)
func TestTrimBackups(t *testing.T) {
filepaths.SteampipeDir, _ = filehelpers.Tildefy("~/.steampipe")
app_specific.InstallDir, _ = filehelpers.Tildefy("~/.steampipe")
// create backups more than MaxBackups
backupDir := filepaths.EnsureBackupsDir()
filesCreated := []string{}

View File

@@ -6,14 +6,14 @@ import (
"sync"
"testing"
"github.com/turbot/pipe-fittings/app_specific"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/filepaths"
)
// test used for debug purposes to replicate `tuple concurrently updated` DB error
func TestConcurrentPerms(t *testing.T) {
t.Skip()
filepaths.SteampipeDir = "/users/kai/.steampipe"
app_specific.InstallDir = "/users/kai/.steampipe"
ctx := context.Background()
res := StartServices(ctx, []string{"localhost"}, constants.DatabaseDefaultPort, "query")

View File

@@ -15,6 +15,7 @@ import (
psutils "github.com/shirou/gopsutil/process"
"github.com/spf13/viper"
"github.com/turbot/go-kit/helpers"
"github.com/turbot/pipe-fittings/app_specific"
perror_helpers "github.com/turbot/pipe-fittings/error_helpers"
putils "github.com/turbot/pipe-fittings/utils"
"github.com/turbot/steampipe-plugin-sdk/v5/sperr"
@@ -469,7 +470,7 @@ func createCmd(ctx context.Context, port int, listenAddresses []string) *exec.Cm
)
}
postgresCmd.Env = append(os.Environ(), fmt.Sprintf("STEAMPIPE_INSTALL_DIR=%s", filepaths.SteampipeDir))
postgresCmd.Env = append(os.Environ(), fmt.Sprintf("STEAMPIPE_INSTALL_DIR=%s", app_specific.InstallDir))
// Check if the /etc/ssl directory exist in os
dirExist, _ := os.Stat(constants.SslConfDir)

View File

@@ -2,18 +2,18 @@ package filepaths
import (
"fmt"
filepaths2 "github.com/turbot/pipe-fittings/filepaths"
"os"
"path/filepath"
filehelpers "github.com/turbot/go-kit/files"
"github.com/turbot/pipe-fittings/app_specific"
"github.com/turbot/pipe-fittings/filepaths"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/error_helpers"
)
// Constants for Config
const (
DefaultInstallDir = "~/.steampipe"
DefaultPipesInstallDir = "~/.pipes"
connectionsStateFileName = "connection.json"
@@ -28,8 +28,6 @@ const (
localPluginFolder = "local"
)
var SteampipeDir string
func ensureSteampipeSubDir(dirName string) string {
subDir := steampipeSubDir(dirName)
@@ -42,10 +40,10 @@ func ensureSteampipeSubDir(dirName string) string {
}
func steampipeSubDir(dirName string) string {
if SteampipeDir == "" {
panic(fmt.Errorf("cannot call any Steampipe directory functions before SteampipeDir is set"))
if app_specific.InstallDir == "" {
panic(fmt.Errorf("cannot call any Steampipe directory functions before app_specific.InstallDir is set"))
}
return filepath.Join(SteampipeDir, dirName)
return filepath.Join(app_specific.InstallDir, dirName)
}
// EnsureTemplateDir returns the path to the templates directory (creates if missing)
@@ -77,7 +75,7 @@ func PluginInstallDir(pluginImageDisplayRef string) string {
}
func PluginBinaryPath(pluginImageDisplayRef, pluginAlias string) string {
return filepath.Join(PluginInstallDir(pluginImageDisplayRef), filepaths2.PluginAliasToLongName(pluginAlias)+".plugin")
return filepath.Join(PluginInstallDir(pluginImageDisplayRef), filepaths.PluginAliasToLongName(pluginAlias)+".plugin")
}
// EnsureConfigDir returns the path to the config directory (creates if missing)

View File

@@ -9,10 +9,10 @@ import (
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-plugin"
"github.com/turbot/pipe-fittings/app_specific"
"github.com/turbot/steampipe-plugin-sdk/v5/logging"
"github.com/turbot/steampipe-plugin-sdk/v5/sperr"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/filepaths"
"github.com/turbot/steampipe/pkg/pluginmanager_service/grpc"
pb "github.com/turbot/steampipe/pkg/pluginmanager_service/grpc/proto"
pluginshared "github.com/turbot/steampipe/pkg/pluginmanager_service/grpc/shared"
@@ -43,11 +43,11 @@ func StartNewInstance(steampipeExecutablePath string) (*State, error) {
// when the plugin mananager is first started by steampipe, we derive the exe path from the running process and
// store it in the plugin manager state file - then if the fdw needs to start the plugin manager it knows how to
func start(steampipeExecutablePath string) (*State, error) {
// note: we assume the install dir has been assigned to file_paths.SteampipeDir
// note: we assume the install dir has been assigned to file_paths.app_specific.InstallDir
// - this is done both by the FDW and Steampipe
pluginManagerCmd := exec.Command(steampipeExecutablePath,
"plugin-manager",
"--"+constants.ArgInstallDir, filepaths.SteampipeDir)
"--"+constants.ArgInstallDir, app_specific.InstallDir)
// set attributes on the command to ensure the process is not shutdown when its parent terminates
pluginManagerCmd.SysProcAttr = &syscall.SysProcAttr{
Setpgid: true,

View File

@@ -377,7 +377,7 @@ connection "b" {
// // set steampipe dir
// os.Chdir("./test_data/connections_to_update")
// wd, _ := os.Getwd()
// filepaths.SteampipeDir = wd
// app_specific.InstallDir = wd
// for name, test := range testCasesGetConnectionsToUpdate {
// // setup connection config

View File

@@ -8,12 +8,12 @@ import (
"strings"
"testing"
utils "github.com/turbot/pipe-fittings/utils"
"golang.org/x/exp/maps"
"github.com/turbot/steampipe/pkg/filepaths"
"github.com/turbot/pipe-fittings/app_specific"
"github.com/turbot/pipe-fittings/plugin"
"github.com/turbot/pipe-fittings/utils"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
"github.com/turbot/steampipe/pkg/steampipeconfig/options"
"golang.org/x/exp/maps"
)
// TODO KAI add plugin block tests
@@ -60,14 +60,14 @@ var testCasesLoadConfig = map[string]loadConfigTest{
Type: "",
ImportSchema: "enabled",
Config: "access_key = \"aws_dmi_001_access_key\"\nregions = \"- us-east-1\\n-us-west-\"\nsecret_key = \"aws_dmi_001_secret_key\"\n",
DeclRange: modconfig.Range{
DeclRange: plugin.Range{
Filename: "$$test_pwd$$/testdata/connection_config/multiple_connections/config/connection1.spc",
Start: modconfig.Pos{
Start: plugin.Pos{
Line: 1,
Column: 1,
Byte: 0,
},
End: modconfig.Pos{
End: plugin.Pos{
Line: 1,
Column: 11,
Byte: 10,
@@ -82,14 +82,14 @@ var testCasesLoadConfig = map[string]loadConfigTest{
Type: "",
ImportSchema: "enabled",
Config: "access_key = \"aws_dmi_002_access_key\"\nregions = \"- us-east-1\\n-us-west-\"\nsecret_key = \"aws_dmi_002_secret_key\"\n",
DeclRange: modconfig.Range{
DeclRange: plugin.Range{
Filename: "$$test_pwd$$/testdata/connection_config/multiple_connections/config/connection2.spc",
Start: modconfig.Pos{
Start: plugin.Pos{
Line: 1,
Column: 1,
Byte: 0,
},
End: modconfig.Pos{
End: plugin.Pos{
Line: 1,
Column: 11,
Byte: 10,
@@ -115,14 +115,14 @@ var testCasesLoadConfig = map[string]loadConfigTest{
// Type: "",
// ImportSchema: "enabled",
// Config: "",
// DeclRange: modconfig.Range{
// DeclRange: plugin.Range{
// Filename: "$$test_pwd$$/testdata/connection_config/single_connection/config/connection1.spc",
// Start: modconfig.Pos{
// Start: plugin.Pos{
// Line: 1,
// Column: 1,
// Byte: 0,
// },
// End: modconfig.Pos{
// End: plugin.Pos{
// Line: 1,
// Column: 11,
// Byte: 10,
@@ -148,14 +148,14 @@ var testCasesLoadConfig = map[string]loadConfigTest{
// Type: "",
// ImportSchema: "enabled",
// Config: "",
// DeclRange: modconfig.Range{
// DeclRange: plugin.Range{
// Filename: "$$test_pwd$$/testdata/connection_config/single_connection_with_default_options/config/connection1.spc",
// Start: modconfig.Pos{
// Start: plugin.Pos{
// Line: 1,
// Column: 1,
// Byte: 0,
// },
// End: modconfig.Pos{
// End: plugin.Pos{
// Line: 1,
// Column: 11,
// Byte: 10,
@@ -195,14 +195,14 @@ var testCasesLoadConfig = map[string]loadConfigTest{
// Type: "",
// ImportSchema: "enabled",
// Config: "",
// DeclRange: modconfig.Range{
// DeclRange: plugin.Range{
// Filename: "$$test_pwd$$/testdata/connection_config/single_connection_with_default_options/config/connection1.spc",
// Start: modconfig.Pos{
// Start: plugin.Pos{
// Line: 1,
// Column: 1,
// Byte: 0,
// },
// End: modconfig.Pos{
// End: plugin.Pos{
// Line: 1,
// Column: 11,
// Byte: 10,
@@ -237,14 +237,14 @@ var testCasesLoadConfig = map[string]loadConfigTest{
// Type: "",
// ImportSchema: "enabled",
// Config: "",
// DeclRange: modconfig.Range{
// DeclRange: plugin.Range{
// Filename: "$$test_pwd$$/testdata/connection_config/single_connection_with_default_options/config/connection1.spc",
// Start: modconfig.Pos{
// Start: plugin.Pos{
// Line: 1,
// Column: 1,
// Byte: 0,
// },
// End: modconfig.Pos{
// End: plugin.Pos{
// Line: 1,
// Column: 11,
// Byte: 10,
@@ -281,14 +281,14 @@ var testCasesLoadConfig = map[string]loadConfigTest{
// Cache: &trueVal,
// CacheTTL: &ttlVal,
// },
// DeclRange: modconfig.Range{
// DeclRange: plugin.Range{
// Filename: "$$test_pwd$$/testdata/connection_config/single_connection_with_default_and_connection_options/config/connection1.spc",
// Start: modconfig.Pos{
// Start: plugin.Pos{
// Line: 1,
// Column: 1,
// Byte: 0,
// },
// End: modconfig.Pos{
// End: plugin.Pos{
// Line: 1,
// Column: 11,
// Byte: 10,
@@ -359,8 +359,8 @@ func TestLoadConfig(t *testing.T) {
t.Errorf("failed to build absolute config filepath from %s", workspaceDir)
}
// set SteampipeDir
filepaths.SteampipeDir = steampipeDir
// set app_specific.InstallDir
app_specific.InstallDir = steampipeDir
// now load config
config, errorsAndWarnings := loadSteampipeConfig(context.TODO(), workspaceDir, "")

View File

@@ -1,12 +1,13 @@
package steampipeconfig
import (
filepaths2 "github.com/turbot/pipe-fittings/filepaths"
"os"
"path/filepath"
"testing"
filehelpers "github.com/turbot/go-kit/files"
"github.com/turbot/pipe-fittings/app_specific"
pfilepaths "github.com/turbot/pipe-fittings/filepaths"
"github.com/turbot/steampipe/pkg/filepaths"
)
@@ -36,8 +37,7 @@ func setupTestData() {
}
func TestFindPluginFolderTest(t *testing.T) {
filepaths.SteampipeDir, _ = filehelpers.Tildefy("~/.steampipe")
app_specific.InstallDir, _ = filehelpers.Tildefy("~/.steampipe")
setupTestData()
directories := []string{
@@ -49,7 +49,7 @@ func TestFindPluginFolderTest(t *testing.T) {
setupFindPluginFolderTest(directories)
for name, test := range testCasesFindPluginFolderTest {
path, err := filepaths2.FindPluginFolder(test.schema)
path, err := pfilepaths.FindPluginFolder(test.schema)
if err != nil {
if test.expected != "ERROR" {
t.Errorf(`Test: '%s'' FAILED : unexpected error %v`, name, err)

View File

@@ -9,6 +9,7 @@ import (
"time"
"github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/turbot/pipe-fittings/app_specific"
"github.com/turbot/pipe-fittings/utils"
"github.com/turbot/steampipe/pkg/version"
)
@@ -87,7 +88,7 @@ func (c *versionChecker) versionCheckURL() url.URL {
var u url.URL
//https://hub.steampipe.io/api/cli/version/latest
u.Scheme = "https"
u.Host = "hub.steampipe.io"
u.Path = "api/cli/version/latest"
u.Host = app_specific.VersionCheckHost
u.Path = app_specific.VersionCheckPath
return u
}