diff --git a/pkg/cloud/cloud_metadata.go b/pkg/cloud/cloud_metadata.go deleted file mode 100644 index 255f582db..000000000 --- a/pkg/cloud/cloud_metadata.go +++ /dev/null @@ -1,78 +0,0 @@ -package cloud - -import ( - "context" - "fmt" - "strings" - - steampipecloud "github.com/turbot/steampipe-cloud-sdk-go" - "github.com/turbot/steampipe-plugin-sdk/v5/sperr" - "github.com/turbot/steampipe/pkg/error_helpers" - "github.com/turbot/steampipe/pkg/steampipeconfig" -) - -func GetCloudMetadata(ctx context.Context, workspaceDatabaseString, token string) (*steampipeconfig.CloudMetadata, error) { - client := newSteampipeCloudClient(token) - - parts := strings.Split(workspaceDatabaseString, "/") - if len(parts) != 2 { - return nil, sperr.New("invalid 'workspace-database' argument '%s' - must be either a connection string or in format /", workspaceDatabaseString) - } - identityHandle := parts[0] - workspaceHandle := parts[1] - - // get the identity - identity, _, err := client.Identities.Get(ctx, identityHandle).Execute() - if err != nil { - return nil, sperr.New("Invalid 'workspace-database' argument '%s'.\nPlease check the identity and workspace names and try again.", workspaceDatabaseString) - } - - // get the workspace - var cloudWorkspace steampipecloud.Workspace - if identity.Type == "user" { - cloudWorkspace, _, err = client.UserWorkspaces.Get(ctx, identityHandle, workspaceHandle).Execute() - } else { - cloudWorkspace, _, err = client.OrgWorkspaces.Get(ctx, identityHandle, workspaceHandle).Execute() - } - - if error_helpers.IsInvalidWorkspaceDatabaseArg(err) { - return nil, sperr.New("Invalid 'workspace-database' argument '%s'.\nPlease check the workspace name and try again.", workspaceDatabaseString) - } else if error_helpers.IsInvalidCloudToken(err) { - return nil, error_helpers.InvalidCloudTokenError - } - - workspaceHost := cloudWorkspace.GetHost() - databaseName := cloudWorkspace.GetDatabaseName() - - actor, _, err := client.Actors.Get(ctx).Execute() - if err != nil { - return nil, error_helpers.InvalidCloudTokenError - } - - password, _, err := client.Users.GetDBPassword(ctx, actor.GetHandle()).Execute() - if err != nil { - return nil, sperr.Wrap(err) - } - - connectionString := fmt.Sprintf("postgresql://%s:%s@%s-%s.%s:9193/%s", actor.Handle, password.Password, identityHandle, workspaceHandle, workspaceHost, databaseName) - - cloudMetadata := &steampipeconfig.CloudMetadata{ - Actor: &steampipeconfig.ActorMetadata{ - Id: actor.Id, - Handle: actor.Handle, - }, - Identity: &steampipeconfig.IdentityMetadata{ - Id: cloudWorkspace.IdentityId, - Type: identity.Type, - Handle: identityHandle, - }, - WorkspaceDatabase: &steampipeconfig.WorkspaceMetadata{ - Id: cloudWorkspace.Id, - Handle: cloudWorkspace.Handle, - }, - - ConnectionString: connectionString, - } - - return cloudMetadata, nil -} diff --git a/pkg/cloud/login.go b/pkg/cloud/login.go index ea582da5e..dff23d9f7 100644 --- a/pkg/cloud/login.go +++ b/pkg/cloud/login.go @@ -19,8 +19,6 @@ import ( "github.com/turbot/steampipe/pkg/filepaths" ) -var UnconfirmedError = "Not confirmed" - // WebLogin POSTs to ${envBaseUrl}/api/latest/login/token to retrieve a login is // it then opens the login webpage and returns th eid func WebLogin(ctx context.Context) (string, error) { diff --git a/pkg/cloud/workspace.go b/pkg/cloud/workspace.go deleted file mode 100644 index 88d417002..000000000 --- a/pkg/cloud/workspace.go +++ /dev/null @@ -1,38 +0,0 @@ -package cloud - -import ( - "context" - "fmt" - - "github.com/turbot/steampipe-plugin-sdk/v5/sperr" - "github.com/turbot/steampipe/pkg/error_helpers" -) - -// GetUserWorkspaceHandle returns the handle of the user workspace -// -// in format actorHandle/workspaceHandle -// -// if there are 0 or > 1 workspaces this is an error -func GetUserWorkspaceHandle(ctx context.Context, token string) (string, error) { - client := newSteampipeCloudClient(token) - actor, _, err := client.Actors.Get(ctx).Execute() - if err != nil { - return "", error_helpers.InvalidCloudTokenError - } - userHandler := actor.Handle - workspacesResponse, _, err := client.UserWorkspaces.List(ctx, userHandler).Execute() - if err != nil { - return "", sperr.Wrap(err) - } - workspaces := workspacesResponse.GetItems() - - if len(workspaces) == 0 { - return "", sperr.New("snapshot-location is not specified and no workspaces exist for user %s", getActorName(actor)) - } - if len(workspaces) > 1 { - return "", sperr.New("more than one workspace found for user %s", getActorName(actor)) - } - - workspaceHandle := fmt.Sprintf("%s/%s", actor.GetHandle(), workspaces[0].GetHandle()) - return workspaceHandle, nil -} diff --git a/pkg/cmdconfig/validate.go b/pkg/cmdconfig/validate.go index f821aba97..ec24e1140 100644 --- a/pkg/cmdconfig/validate.go +++ b/pkg/cmdconfig/validate.go @@ -7,10 +7,10 @@ import ( "github.com/spf13/viper" filehelpers "github.com/turbot/go-kit/files" + "github.com/turbot/pipe-fittings/cloud" pconstants "github.com/turbot/pipe-fittings/constants" - "github.com/turbot/steampipe/pkg/cloud" + "github.com/turbot/pipe-fittings/steampipeconfig" "github.com/turbot/steampipe/pkg/error_helpers" - "github.com/turbot/steampipe/pkg/steampipeconfig" ) func ValidateSnapshotArgs(ctx context.Context) error { diff --git a/pkg/connection/config_map.go b/pkg/connection/config_map.go index 2d4deb2de..c0deba635 100644 --- a/pkg/connection/config_map.go +++ b/pkg/connection/config_map.go @@ -2,15 +2,15 @@ package connection import ( typehelpers "github.com/turbot/go-kit/types" + "github.com/turbot/pipe-fittings/modconfig" sdkproto "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" ) type ConnectionConfigMap map[string]*sdkproto.ConnectionConfig // NewConnectionConfigMap creates a map of sdkproto.ConnectionConfig keyed by connection name // NOTE: connections in error are EXCLUDED -func NewConnectionConfigMap(connectionMap map[string]*modconfig.Connection) ConnectionConfigMap { +func NewConnectionConfigMap(connectionMap map[string]*modconfig.SteampipeConnection) ConnectionConfigMap { configMap := make(ConnectionConfigMap) for k, v := range connectionMap { if v.Error != nil { diff --git a/pkg/db/db_local/search_path.go b/pkg/db/db_local/search_path.go index a46d11849..19ea4e320 100644 --- a/pkg/db/db_local/search_path.go +++ b/pkg/db/db_local/search_path.go @@ -9,10 +9,10 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/spf13/viper" + "github.com/turbot/pipe-fittings/modconfig" "github.com/turbot/steampipe/pkg/constants" "github.com/turbot/steampipe/pkg/db/db_common" "github.com/turbot/steampipe/pkg/steampipeconfig" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" ) func SetUserSearchPath(ctx context.Context, pool *pgxpool.Pool) ([]string, error) { diff --git a/pkg/initialisation/cloud_metadata.go b/pkg/initialisation/cloud_metadata.go index c26ccad08..cebd7e505 100644 --- a/pkg/initialisation/cloud_metadata.go +++ b/pkg/initialisation/cloud_metadata.go @@ -5,10 +5,10 @@ import ( "strings" "github.com/spf13/viper" + "github.com/turbot/pipe-fittings/cloud" "github.com/turbot/pipe-fittings/constants" - "github.com/turbot/steampipe/pkg/cloud" + "github.com/turbot/pipe-fittings/steampipeconfig" "github.com/turbot/steampipe/pkg/error_helpers" - "github.com/turbot/steampipe/pkg/steampipeconfig" ) func getCloudMetadata(ctx context.Context) (*steampipeconfig.CloudMetadata, error) { diff --git a/pkg/interactive/interactive_client.go b/pkg/interactive/interactive_client.go index 9457327bb..cada54ad7 100644 --- a/pkg/interactive/interactive_client.go +++ b/pkg/interactive/interactive_client.go @@ -20,6 +20,7 @@ import ( "github.com/spf13/viper" "github.com/turbot/go-kit/helpers" pconstants "github.com/turbot/pipe-fittings/constants" + "github.com/turbot/pipe-fittings/modconfig" "github.com/turbot/pipe-fittings/querydisplay" "github.com/turbot/pipe-fittings/utils" "github.com/turbot/steampipe/pkg/cmdconfig" @@ -32,7 +33,6 @@ import ( "github.com/turbot/steampipe/pkg/query/queryhistory" "github.com/turbot/steampipe/pkg/statushooks" "github.com/turbot/steampipe/pkg/steampipeconfig" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" "github.com/turbot/steampipe/pkg/version" ) diff --git a/pkg/introspection/connection_table_sql.go b/pkg/introspection/connection_table_sql.go index 09ca41291..91aa05e98 100644 --- a/pkg/introspection/connection_table_sql.go +++ b/pkg/introspection/connection_table_sql.go @@ -3,10 +3,10 @@ package introspection import ( "fmt" + "github.com/turbot/pipe-fittings/modconfig" "github.com/turbot/steampipe/pkg/constants" "github.com/turbot/steampipe/pkg/db/db_common" "github.com/turbot/steampipe/pkg/steampipeconfig" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" "golang.org/x/exp/maps" ) @@ -136,7 +136,7 @@ DO return getConnectionStateQueries(queryFormat, args) } -func GetNewConnectionStateFromConnectionInsertSql(c *modconfig.Connection) []db_common.QueryWithArgs { +func GetNewConnectionStateFromConnectionInsertSql(c *modconfig.SteampipeConnection) []db_common.QueryWithArgs { queryFormat := `INSERT INTO %s.%s (name, state, type, diff --git a/pkg/parse/connection.go b/pkg/parse/connection.go deleted file mode 100644 index 4f9363761..000000000 --- a/pkg/parse/connection.go +++ /dev/null @@ -1,135 +0,0 @@ -package parse - -import ( - "fmt" - "log" - "strings" - - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/gohcl" - "github.com/hashicorp/hcl/v2/hclsyntax" - "github.com/turbot/pipe-fittings/hclhelpers" - pmodconfig "github.com/turbot/pipe-fittings/modconfig" - "github.com/turbot/pipe-fittings/parse" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" - "github.com/zclconf/go-cty/cty" - "golang.org/x/exp/maps" -) - -func DecodeConnection(block *hcl.Block) (*modconfig.Connection, hcl.Diagnostics) { - connectionContent, rest, diags := block.Body.PartialContent(ConnectionBlockSchema) - if diags.HasErrors() { - return nil, diags - } - - connection := modconfig.NewConnection(block) - - // decode the plugin property - // NOTE: this mutates connection to set PluginAlias and possible PluginInstance - diags = decodeConnectionPluginProperty(connectionContent, connection) - if diags.HasErrors() { - return nil, diags - } - - if connectionContent.Attributes["type"] != nil { - var connectionType string - diags = gohcl.DecodeExpression(connectionContent.Attributes["type"].Expr, nil, &connectionType) - if diags.HasErrors() { - return nil, diags - } - connection.Type = connectionType - } - if connectionContent.Attributes["import_schema"] != nil { - var importSchema string - diags = gohcl.DecodeExpression(connectionContent.Attributes["import_schema"].Expr, nil, &importSchema) - if diags.HasErrors() { - return nil, diags - } - connection.ImportSchema = importSchema - } - if connectionContent.Attributes["connections"] != nil { - var connections []string - diags = gohcl.DecodeExpression(connectionContent.Attributes["connections"].Expr, nil, &connections) - if diags.HasErrors() { - return nil, diags - } - connection.ConnectionNames = connections - } - - // blocks are not (currently) supported in connections - for _, connectionBlock := range connectionContent.Blocks { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: fmt.Sprintf("connections do not support '%s' blocks", block.Type), - Subject: hclhelpers.BlockRangePointer(connectionBlock), - }) - } - if hclBody, ok := rest.(*hclsyntax.Body); ok { - for _, b := range hclBody.Blocks { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: fmt.Sprintf("connections do not support '%s' blocks", b.Type), - Subject: hclhelpers.HclSyntaxBlockRangePointer(b), - }) - } - } - - // convert the remaining config to a hcl string to pass to the plugin - config, moreDiags := hclhelpers.HclBodyToHclString(rest, connectionContent) - if moreDiags.HasErrors() { - diags = append(diags, moreDiags...) - } else { - connection.Config = config - } - - return connection, diags -} - -func decodeConnectionPluginProperty(connectionContent *hcl.BodyContent, connection *modconfig.Connection) hcl.Diagnostics { - var pluginName string - evalCtx := &hcl.EvalContext{Variables: make(map[string]cty.Value)} - - diags := gohcl.DecodeExpression(connectionContent.Attributes["plugin"].Expr, evalCtx, &pluginName) - res := parse.NewDecodeResult() - res.HandleDecodeDiags(diags) - if res.Diags.HasErrors() { - return res.Diags - } - if len(res.Depends) > 0 { - log.Printf("[INFO] decodeConnectionPluginProperty plugin property is HCL reference") - // if this is a plugin reference, extract the plugin instance - pluginInstance, ok := getPluginInstanceFromDependency(maps.Values(res.Depends)) - if !ok { - log.Printf("[INFO] failed to resolve plugin property") - // return the original diagnostics - return diags - } - - // so we have resolved a reference to a plugin config - // we will validate that this block exists later in initializePlugins - // set PluginInstance ONLY - // (the PluginInstance property being set means that we will raise the correct error if we fail to resolve the plugin block) - connection.PluginInstance = &pluginInstance - return nil - } - - // NOTE: plugin property is set in initializePlugins - connection.PluginAlias = pluginName - - return nil -} - -func getPluginInstanceFromDependency(dependencies []*pmodconfig.ResourceDependency) (string, bool) { - if len(dependencies) != 1 { - return "", false - } - if len(dependencies[0].Traversals) != 1 { - return "", false - } - traversalString := hclhelpers.TraversalAsString(dependencies[0].Traversals[0]) - split := strings.Split(traversalString, ".") - if len(split) != 2 || split[0] != "plugin" { - return "", false - } - return split[1], true -} diff --git a/pkg/parse/schema.go b/pkg/parse/schema.go deleted file mode 100644 index 65965eb11..000000000 --- a/pkg/parse/schema.go +++ /dev/null @@ -1,27 +0,0 @@ -package parse - -import "github.com/hashicorp/hcl/v2" - -var ConnectionBlockSchema = &hcl.BodySchema{ - Attributes: []hcl.AttributeSchema{ - { - Name: "plugin", - Required: true, - }, - { - Name: "type", - }, - { - Name: "connections", - }, - { - Name: "import_schema", - }, - }, - Blocks: []hcl.BlockHeaderSchema{ - { - Type: "options", - LabelNames: []string{"type"}, - }, - }, -} diff --git a/pkg/query/init_data.go b/pkg/query/init_data.go index 3da843895..c971c74bc 100644 --- a/pkg/query/init_data.go +++ b/pkg/query/init_data.go @@ -10,13 +10,13 @@ import ( "github.com/spf13/viper" pconstants "github.com/turbot/pipe-fittings/constants" + "github.com/turbot/pipe-fittings/modconfig" "github.com/turbot/steampipe/pkg/constants" "github.com/turbot/steampipe/pkg/db/db_client" "github.com/turbot/steampipe/pkg/error_helpers" "github.com/turbot/steampipe/pkg/export" "github.com/turbot/steampipe/pkg/initialisation" "github.com/turbot/steampipe/pkg/statushooks" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" ) type InitData struct { diff --git a/pkg/query/queryexecute/execute.go b/pkg/query/queryexecute/execute.go index 5b33ccb27..b88af8b2a 100644 --- a/pkg/query/queryexecute/execute.go +++ b/pkg/query/queryexecute/execute.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/viper" pconstants "github.com/turbot/pipe-fittings/constants" "github.com/turbot/pipe-fittings/contexthelpers" + "github.com/turbot/pipe-fittings/modconfig" "github.com/turbot/pipe-fittings/querydisplay" "github.com/turbot/pipe-fittings/utils" "github.com/turbot/steampipe/pkg/cmdconfig" @@ -17,7 +18,6 @@ import ( "github.com/turbot/steampipe/pkg/error_helpers" "github.com/turbot/steampipe/pkg/interactive" "github.com/turbot/steampipe/pkg/query" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" ) func RunInteractiveSession(ctx context.Context, initData *query.InitData) error { diff --git a/pkg/steampipeconfig/cloud_metadata.go b/pkg/steampipeconfig/cloud_metadata.go deleted file mode 100644 index 6d7934da8..000000000 --- a/pkg/steampipeconfig/cloud_metadata.go +++ /dev/null @@ -1,24 +0,0 @@ -package steampipeconfig - -type CloudMetadata struct { - Actor *ActorMetadata `json:"actor,omitempty"` - Identity *IdentityMetadata `json:"identity,omitempty"` - WorkspaceDatabase *WorkspaceMetadata `json:"workspace,omitempty"` - ConnectionString string `json:"-"` -} - -type ActorMetadata struct { - Id string `json:"id,omitempty"` - Handle string `json:"handle,omitempty"` -} - -type IdentityMetadata struct { - Id string `json:"id,omitempty"` - Handle string `json:"handle,omitempty"` - Type string `json:"type,omitempty"` -} - -type WorkspaceMetadata struct { - Id string `json:"id,omitempty"` - Handle string `json:"handle,omitempty"` -} diff --git a/pkg/steampipeconfig/cloud_workspace.go b/pkg/steampipeconfig/cloud_workspace.go deleted file mode 100644 index 6b2da9f9e..000000000 --- a/pkg/steampipeconfig/cloud_workspace.go +++ /dev/null @@ -1,9 +0,0 @@ -package steampipeconfig - -import "strings" - -// IsCloudWorkspaceIdentifier returns whether name is a cloud workspace identifier -// of the form: {identity_handle}/{workspace_handle}, -func IsCloudWorkspaceIdentifier(name string) bool { - return len(strings.Split(name, "/")) == 2 -} diff --git a/pkg/steampipeconfig/connection_plugin.go b/pkg/steampipeconfig/connection_plugin.go index 6a01e1636..be593b123 100644 --- a/pkg/steampipeconfig/connection_plugin.go +++ b/pkg/steampipeconfig/connection_plugin.go @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/go-plugin" typehelpers "github.com/turbot/go-kit/types" pconstants "github.com/turbot/pipe-fittings/constants" + "github.com/turbot/pipe-fittings/modconfig" "github.com/turbot/pipe-fittings/utils" sdkgrpc "github.com/turbot/steampipe-plugin-sdk/v5/grpc" sdkproto "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" @@ -15,7 +16,6 @@ import ( "github.com/turbot/steampipe/pkg/error_helpers" "github.com/turbot/steampipe/pkg/pluginmanager_service/grpc/proto" pluginshared "github.com/turbot/steampipe/pkg/pluginmanager_service/grpc/shared" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" "golang.org/x/exp/maps" ) @@ -97,7 +97,7 @@ func CreateConnectionPlugins(pluginManager pluginshared.PluginManager, connectio } log.Printf("[TRACE] CreateConnectionPlugin creating %d %s", len(connectionNamesToCreate), utils.Pluralize("connection", len(connectionNamesToCreate))) - var connectionsToCreate = make([]*modconfig.Connection, len(connectionNamesToCreate)) + var connectionsToCreate = make([]*modconfig.SteampipeConnection, len(connectionNamesToCreate)) for i, name := range connectionNamesToCreate { connectionsToCreate[i] = GlobalConfig.Connections[name] } @@ -170,7 +170,7 @@ func CreateConnectionPlugins(pluginManager pluginshared.PluginManager, connectio return requestedConnectionPluginMap, res } -func handleGetFailures(getResponse *proto.GetResponse, res *RefreshConnectionResult, connectionsToCreate []*modconfig.Connection) { +func handleGetFailures(getResponse *proto.GetResponse, res *RefreshConnectionResult, connectionsToCreate []*modconfig.SteampipeConnection) { // handle PluginSdkCompatibilityError separately var pluginsWithCompatibilityError = make(map[string]struct{}) var compatibilityErrorConnectionCount int @@ -282,7 +282,7 @@ func fullConnectionPluginMap(sparseConnectionPluginMap map[string]*ConnectionPlu } // createConnectionPlugin attaches to the plugin process -func createConnectionPlugin(connection *modconfig.Connection, reattach *proto.ReattachConfig) (*ConnectionPlugin, error) { +func createConnectionPlugin(connection *modconfig.SteampipeConnection, reattach *proto.ReattachConfig) (*ConnectionPlugin, error) { // we must have a plugin instance if connection.PluginInstance == nil { // unexpected diff --git a/pkg/steampipeconfig/connection_state.go b/pkg/steampipeconfig/connection_state.go index 126dc4d20..4ca958798 100644 --- a/pkg/steampipeconfig/connection_state.go +++ b/pkg/steampipeconfig/connection_state.go @@ -6,9 +6,9 @@ import ( "time" typehelpers "github.com/turbot/go-kit/types" + "github.com/turbot/pipe-fittings/modconfig" "github.com/turbot/steampipe-plugin-sdk/v5/plugin" "github.com/turbot/steampipe/pkg/constants" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" ) // ConnectionState is a struct containing all details for a connection @@ -46,7 +46,7 @@ type ConnectionState struct { EndLineNumber int `json:"end_line_number" db:"end_line_number"` } -func NewConnectionState(connection *modconfig.Connection, creationTime time.Time) *ConnectionState { +func NewConnectionState(connection *modconfig.SteampipeConnection, creationTime time.Time) *ConnectionState { state := &ConnectionState{ Plugin: connection.Plugin, PluginInstance: connection.PluginInstance, @@ -64,7 +64,7 @@ func NewConnectionState(connection *modconfig.Connection, creationTime time.Time return state } -func (d *ConnectionState) setFilename(connection *modconfig.Connection) { +func (d *ConnectionState) setFilename(connection *modconfig.SteampipeConnection) { d.FileName = connection.DeclRange.Filename d.StartLineNumber = connection.DeclRange.Start.Line d.EndLineNumber = connection.DeclRange.End.Line diff --git a/pkg/steampipeconfig/connection_state_map.go b/pkg/steampipeconfig/connection_state_map.go index f65a497e6..e9ce2e460 100644 --- a/pkg/steampipeconfig/connection_state_map.go +++ b/pkg/steampipeconfig/connection_state_map.go @@ -8,11 +8,11 @@ import ( pconstants "github.com/turbot/pipe-fittings/constants" "github.com/turbot/pipe-fittings/error_helpers" + "github.com/turbot/pipe-fittings/modconfig" "github.com/turbot/pipe-fittings/utils" sdkplugin "github.com/turbot/steampipe-plugin-sdk/v5/plugin" "github.com/turbot/steampipe/pkg/constants" "github.com/turbot/steampipe/pkg/filepaths" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" "golang.org/x/exp/maps" ) @@ -21,7 +21,7 @@ type ConnectionStateSummary map[string]int type ConnectionStateMap map[string]*ConnectionState // GetRequiredConnectionStateMap populates a map of connection data for all connections in connectionMap -func GetRequiredConnectionStateMap(connectionMap map[string]*modconfig.Connection, currentConnectionState ConnectionStateMap) (ConnectionStateMap, map[string][]modconfig.Connection, error_helpers.ErrorAndWarnings) { +func GetRequiredConnectionStateMap(connectionMap map[string]*modconfig.SteampipeConnection, currentConnectionState ConnectionStateMap) (ConnectionStateMap, map[string][]modconfig.SteampipeConnection, error_helpers.ErrorAndWarnings) { utils.LogTime("steampipeconfig.GetRequiredConnectionStateMap start") defer utils.LogTime("steampipeconfig.GetRequiredConnectionStateMap end") @@ -32,7 +32,7 @@ func GetRequiredConnectionStateMap(connectionMap map[string]*modconfig.Connectio pluginModTimeMap := make(map[string]time.Time) // map of missing plugins, keyed by plugin alias, value is list of connections using missing plugin - missingPluginMap := make(map[string][]modconfig.Connection) + missingPluginMap := make(map[string][]modconfig.SteampipeConnection) utils.LogTime("steampipeconfig.getRequiredConnections config - iteration start") // populate file mod time for each referenced plugin @@ -85,7 +85,7 @@ func GetRequiredConnectionStateMap(connectionMap map[string]*modconfig.Connectio return requiredState, missingPluginMap, res } -func newErrorConnectionState(connection *modconfig.Connection) *ConnectionState { +func newErrorConnectionState(connection *modconfig.SteampipeConnection) *ConnectionState { res := NewConnectionState(connection, time.Now()) res.SetError(connection.Error.Error()) return res diff --git a/pkg/steampipeconfig/connection_updates.go b/pkg/steampipeconfig/connection_updates.go index 505da0b70..b8c9242bf 100644 --- a/pkg/steampipeconfig/connection_updates.go +++ b/pkg/steampipeconfig/connection_updates.go @@ -10,13 +10,13 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/turbot/go-kit/helpers" + "github.com/turbot/pipe-fittings/modconfig" "github.com/turbot/pipe-fittings/utils" "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" "github.com/turbot/steampipe-plugin-sdk/v5/plugin" "github.com/turbot/steampipe/pkg/constants" "github.com/turbot/steampipe/pkg/db/db_common" pluginshared "github.com/turbot/steampipe/pkg/pluginmanager_service/grpc/shared" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" "golang.org/x/exp/maps" ) @@ -28,7 +28,7 @@ type ConnectionUpdates struct { MissingComments ConnectionStateMap // map of missing plugins, keyed by plugin ALIAS // NOTE: we key by alias so the error message refers to the string which was used to specify the plugin - MissingPlugins map[string][]modconfig.Connection + MissingPlugins map[string][]modconfig.SteampipeConnection // the connections which will exist after the update FinalConnectionState ConnectionStateMap // connection plugins required to perform the updates, keyed by connection name @@ -353,7 +353,7 @@ func (u *ConnectionUpdates) getConnectionsToCreate(alreadyCreatedConnectionPlugi // ensure we instantiate all plugins required for schema AND comment updates connections := append(maps.Keys(u.Update), maps.Keys(u.MissingComments)...) // put connections into a map to avoid dupes - var connectionMap = make(map[string]*modconfig.Connection, len(connections)) + var connectionMap = make(map[string]*modconfig.SteampipeConnection, len(connections)) for _, connectionName := range connections { connection := GlobalConfig.Connections[connectionName] connectionMap[connectionName] = connection diff --git a/pkg/steampipeconfig/inputvars/collect_variables.go b/pkg/steampipeconfig/inputvars/collect_variables.go deleted file mode 100644 index cdd5b908d..000000000 --- a/pkg/steampipeconfig/inputvars/collect_variables.go +++ /dev/null @@ -1,343 +0,0 @@ -package inputvars - -import ( - "fmt" - "github.com/turbot/steampipe/pkg/error_helpers" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" - "log" - "os" - "regexp" - "strings" - - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclsyntax" - "github.com/spf13/viper" - "github.com/turbot/steampipe/pkg/constants" - "github.com/turbot/steampipe/pkg/filepaths" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig/var_config" - "github.com/turbot/terraform-components/tfdiags" -) - -// CollectVariableValues inspects the various places that configuration input variable -// values can come from and constructs a map ready to be passed to the -// backend as part of a Operation. -// -// This method returns diagnostics relating to the collection of the values, -// but the values themselves may produce additional diagnostics when finally -// parsed. -func CollectVariableValues(workspacePath string, variableFileArgs []string, variablesArgs []string, workspaceMod *modconfig.Mod) (map[string]UnparsedVariableValue, error) { - workspaceModName := workspaceMod.ShortName - var modNames = make(map[string]struct{}) - for _, m := range workspaceMod.ResourceMaps.Mods { - modNames[m.ShortName] = struct{}{} - } - - ret := map[string]UnparsedVariableValue{} - - // First we'll deal with environment variables - // since they have the lowest precedence. - // (apart from values in the mod Require proeprty, which are handled separately later) - { - env := os.Environ() - for _, raw := range env { - if !strings.HasPrefix(raw, constants.EnvInputVarPrefix) { - continue - } - raw = raw[len(constants.EnvInputVarPrefix):] // trim the prefix - - eq := strings.Index(raw, "=") - if eq == -1 { - // Seems invalid, so we'll ignore it. - continue - } - - name := raw[:eq] - rawVal := raw[eq+1:] - - ret[name] = unparsedVariableValueString{ - str: rawVal, - name: name, - sourceType: ValueFromEnvVar, - } - log.Printf("[INFO] adding value for variable '%s' from environment", name) - } - } - - // Next up we have some implicit files that are loaded automatically - // if they are present. There's the original terraform.tfvars - // (constants.DefaultVarsFilename) along with the later-added search for all files - // ending in .auto.spvars. - defaultVarsPath := filepaths.DefaultVarsFilePath(workspacePath) - if _, err := os.Stat(defaultVarsPath); err == nil { - log.Printf("[INFO] adding values from %s", defaultVarsPath) - diags := addVarsFromFile(defaultVarsPath, ValueFromAutoFile, ret) - if diags.HasErrors() { - return nil, error_helpers.DiagsToError(fmt.Sprintf("failed to load variables from '%s'", defaultVarsPath), diags) - } - - } - - if infos, err := os.ReadDir("."); err == nil { - // "infos" is already sorted by name, so we just need to filter it here. - for _, info := range infos { - name := info.Name() - if !isAutoVarFile(name) { - continue - } - log.Printf("[INFO] adding values from %s", name) - diags := addVarsFromFile(name, ValueFromAutoFile, ret) - if diags.HasErrors() { - return nil, error_helpers.DiagsToError(fmt.Sprintf("failed to load variables from '%s'", name), diags) - } - - } - } - - // Finally we process values given explicitly on the command line, either - // as individual literal settings or as additional files to read. - for _, fileArg := range variableFileArgs { - log.Printf("[INFO] adding values from %s", fileArg) - diags := addVarsFromFile(fileArg, ValueFromNamedFile, ret) - if diags.HasErrors() { - return nil, error_helpers.DiagsToError(fmt.Sprintf("failed to load variables from '%s'", fileArg), diags) - } - } - - var diags tfdiags.Diagnostics - for _, variableArg := range variablesArgs { - // Value should be in the form "name=value", where value is a - // raw string whose interpretation will depend on the variable's - // parsing mode. - raw := variableArg - eq := strings.Index(raw, "=") - if eq == -1 { - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Error, - fmt.Sprintf("The given --var option %q is not correctly specified. It must be a variable name and value separated an equals sign: --var key=value", raw), - "", - )) - continue - } - - name := raw[:eq] - rawVal := raw[eq+1:] - ret[name] = unparsedVariableValueString{ - str: rawVal, - name: name, - sourceType: ValueFromCLIArg, - } - log.Printf("[INFO] adding value for variable '%s' from command line arg", name) - } - - if diags.HasErrors() { - return nil, error_helpers.DiagsToError("failed to evaluate var args:", diags) - } - - // check viper for any interactively added variables - if varMap := viper.GetStringMap(constants.ConfigInteractiveVariables); varMap != nil { - for name, rawVal := range varMap { - // Value should be in the form "name=value", where value is a - // raw string whose interpretation will depend on the variable's - // parsing mode. - ret[name] = UnparsedInteractiveVariableValue{ - Name: name, - RawValue: rawVal.(string), - } - log.Printf("[INFO] adding value for variable '%s' specified on interactive prompt", name) - } - } - - // now map any variable names of form . to .var. - // - if any var value is qualified with the workspace mod, remove the qualification - // - remove any variables which are not in the root mod or first level dependencies - ret = transformVarNames(ret, workspaceModName, modNames) - - return ret, nil -} - -// map any variable names of form . to .var. -func transformVarNames(rawValues map[string]UnparsedVariableValue, workspaceModName string, modNames map[string]struct{}) map[string]UnparsedVariableValue { - ret := make(map[string]UnparsedVariableValue, len(rawValues)) - for k, v := range rawValues { - - parts := strings.Split(k, ".") - if len(parts) > 1 { - if _, ok := modNames[parts[0]]; !ok { - // NOTE: skip any variables which are not in the root mod or first level dependencies - continue - } - if parts[0] == workspaceModName { - k = parts[1] - } else { - k = fmt.Sprintf("%s.var.%s", parts[0], parts[1]) - } - } - - ret[k] = v - } - return ret -} - -func addVarsFromFile(filename string, sourceType ValueSourceType, to map[string]UnparsedVariableValue) tfdiags.Diagnostics { - var diags tfdiags.Diagnostics - - src, err := os.ReadFile(filename) - if err != nil { - if os.IsNotExist(err) { - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Error, - "Failed to read variables file", - fmt.Sprintf("Given variables file %s does not exist.", filename), - )) - } else { - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Error, - "Failed to read variables file", - fmt.Sprintf("Error while reading %s: %s.", filename, err), - )) - } - return diags - } - - // replace syntax `.=` with `___steampipe__= - sanitisedSrc, depVarAliases := sanitiseVariableNames(src) - - var f *hcl.File - var hclDiags hcl.Diagnostics - - // attempt to parse the config - f, hclDiags = hclsyntax.ParseConfig(sanitisedSrc, filename, hcl.Pos{Line: 1, Column: 1}) - diags = diags.Append(hclDiags) - if f == nil || f.Body == nil { - return diags - } - - // Before we do our real decode, we'll probe to see if there are any blocks - // of type "variable" in this body, since it's a common mistake for new - // users to put variable declarations in tfvars rather than variable value - // definitions, and otherwise our error message for that case is not so - // helpful. - { - content, _, _ := f.Body.PartialContent(&hcl.BodySchema{ - Blocks: []hcl.BlockHeaderSchema{ - { - Type: "variable", - LabelNames: []string{"name"}, - }, - }, - }) - for _, block := range content.Blocks { - name := block.Labels[0] - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Variable declaration in .tfvars file", - Detail: fmt.Sprintf("A .tfvars file is used to assign values to variables that have already been declared in .tf files, not to declare new variables. To declare variable %q, place this block in one of your .tf files, such as variables.tf.\n\nTo set a value for this variable in %s, use the definition syntax instead:\n %s = ", name, block.TypeRange.Filename, name), - Subject: &block.TypeRange, - }) - } - if diags.HasErrors() { - // If we already found problems then JustAttributes below will find - // the same problems with less-helpful messages, so we'll bail for - // now to let the user focus on the immediate problem. - return diags - } - } - - attrs, hclDiags := f.Body.JustAttributes() - diags = diags.Append(hclDiags) - - for name, attr := range attrs { - // check for aliases - if alias, ok := depVarAliases[name]; ok { - name = alias - } - to[name] = unparsedVariableValueExpression{ - expr: attr.Expr, - sourceType: sourceType, - } - log.Printf("[INFO] adding value for variable '%s' from var file", name) - } - return diags -} - -func sanitiseVariableNames(src []byte) ([]byte, map[string]string) { - // replace syntax `.=` with `____steampipe_mod______= - - lines := strings.Split(string(src), "\n") - // make map of varname aliases - var depVarAliases = make(map[string]string) - - for i, line := range lines { - - r := regexp.MustCompile(`^ ?(([a-z0-9\-_]+)\.([a-z0-9\-_]+)) ?=`) - captureGroups := r.FindStringSubmatch(line) - if captureGroups != nil && len(captureGroups) == 4 { - fullVarName := captureGroups[1] - mod := captureGroups[2] - varName := captureGroups[3] - - aliasedName := fmt.Sprintf("____steampipe_mod_%s_variable_%s____", mod, varName) - depVarAliases[aliasedName] = fullVarName - lines[i] = strings.Replace(line, fullVarName, aliasedName, 1) - - } - } - - // now try again - src = []byte(strings.Join(lines, "\n")) - return src, depVarAliases -} - -// unparsedVariableValueLiteral is a UnparsedVariableValue -// implementation that was actually already parsed (!). This is -// intended to deal with expressions inside "tfvars" files. -type unparsedVariableValueExpression struct { - expr hcl.Expression - sourceType ValueSourceType -} - -func (v unparsedVariableValueExpression) ParseVariableValue(mode var_config.VariableParsingMode) (*InputValue, tfdiags.Diagnostics) { - var diags tfdiags.Diagnostics - val, hclDiags := v.expr.Value(nil) // nil because no function calls or variable references are allowed here - diags = diags.Append(hclDiags) - - rng := tfdiags.SourceRangeFromHCL(v.expr.Range()) - - return &InputValue{ - Value: val, - SourceType: v.sourceType, - SourceRange: rng, - }, diags -} - -// unparsedVariableValueString is a UnparsedVariableValue -// implementation that parses its value from a string. This can be used -// to deal with values given directly on the command line and via environment -// variables. -type unparsedVariableValueString struct { - str string - name string - sourceType ValueSourceType -} - -func (v unparsedVariableValueString) ParseVariableValue(mode var_config.VariableParsingMode) (*InputValue, tfdiags.Diagnostics) { - var diags tfdiags.Diagnostics - - val, hclDiags := mode.Parse(v.name, v.str) - diags = diags.Append(hclDiags) - - return &InputValue{ - Value: val, - SourceType: v.sourceType, - }, diags -} - -// isAutoVarFile determines if the file ends with .auto.spvars or .auto.spvars.json -func isAutoVarFile(path string) bool { - for _, ext := range constants.AutoVariablesExtensions { - if strings.HasSuffix(path, ext) { - return true - } - } - return false -} diff --git a/pkg/steampipeconfig/inputvars/doc.go b/pkg/steampipeconfig/inputvars/doc.go deleted file mode 100644 index 2fdb863b9..000000000 --- a/pkg/steampipeconfig/inputvars/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package input_vars contains code from https://github.com/hashicorp/terraform, with minor modifications - -package inputvars diff --git a/pkg/steampipeconfig/inputvars/input_values.go b/pkg/steampipeconfig/inputvars/input_values.go deleted file mode 100644 index 2c574dcdc..000000000 --- a/pkg/steampipeconfig/inputvars/input_values.go +++ /dev/null @@ -1,309 +0,0 @@ -package inputvars - -import ( - "fmt" - - "github.com/hashicorp/hcl/v2" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" - "github.com/turbot/terraform-components/tfdiags" - "github.com/zclconf/go-cty/cty" - "github.com/zclconf/go-cty/cty/convert" -) - -// InputValue represents a value for a variable in the configuration, provided -// as part of the definition of an operation. -type InputValue struct { - Value cty.Value - SourceType ValueSourceType - - // SourceRange provides source location information for values whose - // SourceType is either ValueFromConfig or ValueFromFile. It is not - // populated for other source types, and so should not be used. - SourceRange tfdiags.SourceRange -} - -// ValueSourceType describes what broad category of source location provided -// a particular value. -type ValueSourceType rune - -const ( - // ValueFromUnknown is the zero value of ValueSourceType and is not valid. - ValueFromUnknown ValueSourceType = 0 - - // ValueFromConfig indicates that a value came from a .tf or .tf.json file, - // e.g. the default value defined for a variable. - ValueFromConfig ValueSourceType = 'C' - - // ValueFromAutoFile indicates that a value came from a "values file", like - // a .tfvars file, that was implicitly loaded by naming convention. - ValueFromAutoFile ValueSourceType = 'F' - - // ValueFromNamedFile indicates that a value came from a named "values file", - // like a .tfvars file, that was passed explicitly on the command line (e.g. - // -var-file=foo.tfvars). - ValueFromNamedFile ValueSourceType = 'N' - - // ValueFromCLIArg indicates that the value was provided directly in - // a CLI argument. The name of this argument is not recorded and so it must - // be inferred from context. - ValueFromCLIArg ValueSourceType = 'A' - - // ValueFromEnvVar indicates that the value was provided via an environment - // variable. The name of the variable is not recorded and so it must be - // inferred from context. - ValueFromEnvVar ValueSourceType = 'E' - - // ValueFromInput indicates that the value was provided at an interactive - // input prompt. - ValueFromInput ValueSourceType = 'I' - - // ValueFromModFile indicates that the value was provided in the 'Require' section of a mod file - ValueFromModFile ValueSourceType = 'M' -) - -func (v *InputValue) GoString() string { - if (v.SourceRange != tfdiags.SourceRange{}) { - return fmt.Sprintf("&InputValue{Value: %#v, SourceType: %#v, SourceRange: %#v}", v.Value, v.SourceType, v.SourceRange) - } else { - return fmt.Sprintf("&InputValue{Value: %#v, SourceType: %#v}", v.Value, v.SourceType) - } -} - -func (v *InputValue) SourceTypeString() string { - switch v.SourceType { - case ValueFromConfig: - return "config" - case ValueFromAutoFile: - return "auto file" - case ValueFromNamedFile: - return "name file" - case ValueFromCLIArg: - return "CLI arg" - case ValueFromEnvVar: - return "env var" - case ValueFromInput: - return "user input" - default: - return "unknown" - } -} - -//go:generate go run golang.org/x/tools/cmd/stringer -type ValueSourceType - -// InputValues is a map of InputValue instances. -type InputValues map[string]*InputValue - -// Override merges the given value maps with the receiver, overriding any -// conflicting keys so that the latest definition wins. -func (vv InputValues) Override(others ...InputValues) InputValues { - ret := make(InputValues) - for k, v := range vv { - ret[k] = v - } - for _, other := range others { - for k, v := range other { - ret[k] = v - } - } - return ret -} - -// JustValues returns a map that just includes the values, discarding the -// source information. -func (vv InputValues) JustValues() map[string]cty.Value { - ret := make(map[string]cty.Value, len(vv)) - for k, v := range vv { - ret[k] = v.Value - } - return ret -} - -// SameValues returns true if the given InputValues has the same values as -// the receiver, disregarding the source types and source ranges. -// -// Values are compared using the cty "RawEquals" method, which means that -// unknown values can be considered equal to one another if they are of the -// same type. -func (vv InputValues) SameValues(other InputValues) bool { - if len(vv) != len(other) { - return false - } - - for k, v := range vv { - ov, exists := other[k] - if !exists { - return false - } - if !v.Value.RawEquals(ov.Value) { - return false - } - } - - return true -} - -// HasValues returns true if the reciever has the same values as in the given -// map, disregarding the source types and source ranges. -// -// Values are compared using the cty "RawEquals" method, which means that -// unknown values can be considered equal to one another if they are of the -// same type. -func (vv InputValues) HasValues(vals map[string]cty.Value) bool { - if len(vv) != len(vals) { - return false - } - - for k, v := range vv { - oVal, exists := vals[k] - if !exists { - return false - } - if !v.Value.RawEquals(oVal) { - return false - } - } - - return true -} - -// Identical returns true if the given InputValues has the same values, -// source types, and source ranges as the receiver. -// -// Values are compared using the cty "RawEquals" method, which means that -// unknown values can be considered equal to one another if they are of the -// same type. -// -// This method is primarily for testing. For most practical purposes, it's -// better to use SameValues or HasValues. -func (vv InputValues) Identical(other InputValues) bool { - if len(vv) != len(other) { - return false - } - - for k, v := range vv { - ov, exists := other[k] - if !exists { - return false - } - if !v.Value.RawEquals(ov.Value) { - return false - } - if v.SourceType != ov.SourceType { - return false - } - if v.SourceRange != ov.SourceRange { - return false - } - } - - return true -} - -func (vv InputValues) DefaultTo(other InputValues) { - for k, otherVal := range other { - if val, ok := vv[k]; !ok || !val.Value.IsKnown() { - vv[k] = otherVal - } - } -} - -// CheckInputVariables ensures that variable values supplied at the UI conform -// to their corresponding declarations in configuration. -// -// The set of values is considered valid only if the returned diagnostics -// does not contain errors. A valid set of values may still produce warnings, -// which should be returned to the user. -func CheckInputVariables(vcs map[string]*modconfig.Variable, vs InputValues) tfdiags.Diagnostics { - var diags tfdiags.Diagnostics - - for name, vc := range vcs { - val, isSet := vs[name] - if !isSet { - // Always an error, since the caller should already have included - // default values from the configuration in the values map. - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Error, - "Unassigned variable", - fmt.Sprintf("The input variable %q has not been assigned a value. This is a bug in Steampipe; please report it in a GitHub issue.", name), - )) - continue - } - - wantType := vc.Type - - // A given value is valid if it can convert to the desired type. - _, err := convert.Convert(val.Value, wantType) - if err != nil { - switch val.SourceType { - case ValueFromConfig, ValueFromAutoFile, ValueFromNamedFile: - // We have source location information for these. - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid value for input variable", - Detail: fmt.Sprintf("The given value is not valid for variable %q: %s.", name, err), - Subject: val.SourceRange.ToHCL().Ptr(), - }) - case ValueFromEnvVar: - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Error, - "Invalid value for input variable", - fmt.Sprintf("The environment variable SP_VAR_%s does not contain a valid value for variable %q: %s.", name, name, err), - )) - case ValueFromCLIArg: - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Error, - "Invalid value for input variable", - fmt.Sprintf("The argument --var=\"%s=...\" does not contain a valid value for variable %q: %s.", name, name, err), - )) - case ValueFromInput: - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Error, - "Invalid value for input variable", - fmt.Sprintf("The value entered for variable %q is not valid: %s.", name, err), - )) - default: - // The above gets us good coverage for the situations users - // are likely to encounter with their own inputs. The other - // cases are generally implementation bugs, so we'll just - // use a generic error for these. - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Error, - "Invalid value for input variable", - fmt.Sprintf("The value provided for variable %q is not valid: %s.", name, err), - )) - } - } - } - - // Check for any variables that are assigned without being configured. - // This is always an implementation error in the caller, because we - // expect undefined variables to be caught during context construction - // where there is better context to report it well. - for name := range vs { - if _, defined := vcs[name]; !defined { - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Error, - "Value assigned to undeclared variable", - fmt.Sprintf("A value was assigned to an undeclared input variable %q.", name), - )) - } - } - - return diags -} - -// SetVariableValues determines whether the given variable is a public variable and if so sets its value -func (vv InputValues) SetVariableValues(m *modconfig.ModVariableMap) { - for name, inputValue := range vv { - variable, ok := m.PublicVariables[name] - // if this variable does not exist in public variables, skip - if !ok { - // we should have already caught this - continue - } - variable.SetInputValue( - inputValue.Value, - inputValue.SourceTypeString(), - inputValue.SourceRange) - } -} diff --git a/pkg/steampipeconfig/inputvars/typeexpr/doc.go b/pkg/steampipeconfig/inputvars/typeexpr/doc.go deleted file mode 100644 index 5f976335e..000000000 --- a/pkg/steampipeconfig/inputvars/typeexpr/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Package typeexpr contains code from https://github.com/hashicorp/terraform/tree/main/internal/typeexpr, -// with minor modifications - -package typeexpr diff --git a/pkg/steampipeconfig/inputvars/typeexpr/get_type.go b/pkg/steampipeconfig/inputvars/typeexpr/get_type.go deleted file mode 100644 index de5465b99..000000000 --- a/pkg/steampipeconfig/inputvars/typeexpr/get_type.go +++ /dev/null @@ -1,250 +0,0 @@ -package typeexpr - -import ( - "fmt" - - "github.com/hashicorp/hcl/v2" - "github.com/zclconf/go-cty/cty" -) - -const invalidTypeSummary = "Invalid type specification" - -// getType is the internal implementation of both Type and TypeConstraint, -// using the passed flag to distinguish. When constraint is false, the "any" -// keyword will produce an error. -func getType(expr hcl.Expression, constraint bool) (cty.Type, hcl.Diagnostics) { - // First we'll try for one of our keywords - kw := hcl.ExprAsKeyword(expr) - switch kw { - case "bool": - return cty.Bool, nil - case "string": - return cty.String, nil - case "number": - return cty.Number, nil - case "any": - if constraint { - return cty.DynamicPseudoType, nil - } - return cty.DynamicPseudoType, hcl.Diagnostics{{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: fmt.Sprintf("The keyword %q cannot be used in this type specification: an exact type is required.", kw), - Subject: expr.Range().Ptr(), - }} - case "list", "map", "set": - return cty.DynamicPseudoType, hcl.Diagnostics{{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: fmt.Sprintf("The %s type constructor requires one argument specifying the element type.", kw), - Subject: expr.Range().Ptr(), - }} - case "object": - return cty.DynamicPseudoType, hcl.Diagnostics{{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: "The object type constructor requires one argument specifying the attribute types and values as a map.", - Subject: expr.Range().Ptr(), - }} - case "tuple": - return cty.DynamicPseudoType, hcl.Diagnostics{{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: "The tuple type constructor requires one argument specifying the element types as a list.", - Subject: expr.Range().Ptr(), - }} - case "": - // okay! we'll fall through and try processing as a call, then. - default: - return cty.DynamicPseudoType, hcl.Diagnostics{{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: fmt.Sprintf("The keyword %q is not a valid type specification.", kw), - Subject: expr.Range().Ptr(), - }} - } - - // If we get down here then our expression isn't just a keyword, so we'll - // try to process it as a call instead. - call, diags := hcl.ExprCall(expr) - if diags.HasErrors() { - return cty.DynamicPseudoType, hcl.Diagnostics{{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: "A type specification is either a primitive type keyword (bool, number, string) or a complex type constructor call, like list(string).", - Subject: expr.Range().Ptr(), - }} - } - - switch call.Name { - case "bool", "string", "number", "any": - return cty.DynamicPseudoType, hcl.Diagnostics{{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: fmt.Sprintf("Primitive type keyword %q does not expect arguments.", call.Name), - Subject: &call.ArgsRange, - }} - } - - if len(call.Arguments) != 1 { - contextRange := call.ArgsRange - subjectRange := call.ArgsRange - if len(call.Arguments) > 1 { - // If we have too many arguments (as opposed to too _few_) then - // we'll highlight the extraneous arguments as the diagnostic - // subject. - subjectRange = hcl.RangeBetween(call.Arguments[1].Range(), call.Arguments[len(call.Arguments)-1].Range()) - } - - switch call.Name { - case "list", "set", "map": - return cty.DynamicPseudoType, hcl.Diagnostics{{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: fmt.Sprintf("The %s type constructor requires one argument specifying the element type.", call.Name), - Subject: &subjectRange, - Context: &contextRange, - }} - case "object": - return cty.DynamicPseudoType, hcl.Diagnostics{{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: "The object type constructor requires one argument specifying the attribute types and values as a map.", - Subject: &subjectRange, - Context: &contextRange, - }} - case "tuple": - return cty.DynamicPseudoType, hcl.Diagnostics{{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: "The tuple type constructor requires one argument specifying the element types as a list.", - Subject: &subjectRange, - Context: &contextRange, - }} - } - } - - switch call.Name { - - case "list": - ety, diags := getType(call.Arguments[0], constraint) - return cty.List(ety), diags - case "set": - ety, diags := getType(call.Arguments[0], constraint) - return cty.Set(ety), diags - case "map": - ety, diags := getType(call.Arguments[0], constraint) - return cty.Map(ety), diags - case "object": - attrDefs, diags := hcl.ExprMap(call.Arguments[0]) - if diags.HasErrors() { - return cty.DynamicPseudoType, hcl.Diagnostics{{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: "Object type constructor requires a map whose keys are attribute names and whose values are the corresponding attribute types.", - Subject: call.Arguments[0].Range().Ptr(), - Context: expr.Range().Ptr(), - }} - } - - atys := make(map[string]cty.Type) - var optAttrs []string - for _, attrDef := range attrDefs { - attrName := hcl.ExprAsKeyword(attrDef.Key) - if attrName == "" { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: "Object constructor map keys must be attribute names.", - Subject: attrDef.Key.Range().Ptr(), - Context: expr.Range().Ptr(), - }) - continue - } - atyExpr := attrDef.Value - - // the attribute type expression might be wrapped in the special - // modifier optional(...) to indicate an optional attribute. If - // so, we'll unwrap that first and make a note about it being - // optional for when we construct the type below. - if call, callDiags := hcl.ExprCall(atyExpr); !callDiags.HasErrors() { - if call.Name == "optional" { - if len(call.Arguments) < 1 { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: "Optional attribute modifier requires the attribute type as its argument.", - Subject: call.ArgsRange.Ptr(), - Context: atyExpr.Range().Ptr(), - }) - continue - } - if constraint { - if len(call.Arguments) > 1 { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: "Optional attribute modifier expects only one argument: the attribute type.", - Subject: call.ArgsRange.Ptr(), - Context: atyExpr.Range().Ptr(), - }) - } - optAttrs = append(optAttrs, attrName) - } else { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: "Optional attribute modifier is only for type constraints, not for exact types.", - Subject: call.NameRange.Ptr(), - Context: atyExpr.Range().Ptr(), - }) - } - atyExpr = call.Arguments[0] - } - } - - aty, attrDiags := getType(atyExpr, constraint) - diags = append(diags, attrDiags...) - atys[attrName] = aty - } - // NOTE: ObjectWithOptionalAttrs is experimental in cty at the - // time of writing, so this interface might change even in future - // minor versions of cty. We're accepting that because Terraform - // itself is considering optional attributes as experimental right now. - return cty.ObjectWithOptionalAttrs(atys, optAttrs), diags - case "tuple": - elemDefs, diags := hcl.ExprList(call.Arguments[0]) - if diags.HasErrors() { - return cty.DynamicPseudoType, hcl.Diagnostics{{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: "Tuple type constructor requires a list of element types.", - Subject: call.Arguments[0].Range().Ptr(), - Context: expr.Range().Ptr(), - }} - } - etys := make([]cty.Type, len(elemDefs)) - for i, defExpr := range elemDefs { - ety, elemDiags := getType(defExpr, constraint) - diags = append(diags, elemDiags...) - etys[i] = ety - } - return cty.Tuple(etys), diags - case "optional": - return cty.DynamicPseudoType, hcl.Diagnostics{{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: fmt.Sprintf("Keyword %q is valid only as a modifier for object type attributes.", call.Name), - Subject: call.NameRange.Ptr(), - }} - default: - // Can't access call.Arguments in this path because we've not validated - // that it contains exactly one expression here. - return cty.DynamicPseudoType, hcl.Diagnostics{{ - Severity: hcl.DiagError, - Summary: invalidTypeSummary, - Detail: fmt.Sprintf("Keyword %q is not a valid type constructor.", call.Name), - Subject: expr.Range().Ptr(), - }} - } -} diff --git a/pkg/steampipeconfig/inputvars/typeexpr/public.go b/pkg/steampipeconfig/inputvars/typeexpr/public.go deleted file mode 100644 index 3b8f618fb..000000000 --- a/pkg/steampipeconfig/inputvars/typeexpr/public.go +++ /dev/null @@ -1,129 +0,0 @@ -package typeexpr - -import ( - "bytes" - "fmt" - "sort" - - "github.com/hashicorp/hcl/v2/hclsyntax" - - "github.com/hashicorp/hcl/v2" - "github.com/zclconf/go-cty/cty" -) - -// Type attempts to process the given expression as a type expression and, if -// successful, returns the resulting type. If unsuccessful, error diagnostics -// are returned. -func Type(expr hcl.Expression) (cty.Type, hcl.Diagnostics) { - return getType(expr, false) -} - -// TypeConstraint attempts to parse the given expression as a type constraint -// and, if successful, returns the resulting type. If unsuccessful, error -// diagnostics are returned. -// -// A type constraint has the same structure as a type, but it additionally -// allows the keyword "any" to represent cty.DynamicPseudoType, which is often -// used as a wildcard in type checking and type conversion operations. -func TypeConstraint(expr hcl.Expression) (cty.Type, hcl.Diagnostics) { - return getType(expr, true) -} - -// TypeString returns a string rendering of the given type as it would be -// expected to appear in the HCL native syntax. -// -// This is primarily intended for showing types to the user in an application -// that uses typexpr, where the user can be assumed to be familiar with the -// type expression syntax. In applications that do not use typeexpr these -// results may be confusing to the user and so type.FriendlyName may be -// preferable, even though it's less precise. -// -// TypeString produces reasonable results only for types like what would be -// produced by the Type and TypeConstraint functions. In particular, it cannot -// support capsule types. -func TypeString(ty cty.Type) string { - // Easy cases first - switch ty { - case cty.String: - return "string" - case cty.Bool: - return "bool" - case cty.Number: - return "number" - case cty.DynamicPseudoType: - return "any" - } - - if ty.IsCapsuleType() { - panic("TypeString does not support capsule types") - } - - if ty.IsCollectionType() { - ety := ty.ElementType() - etyString := TypeString(ety) - switch { - case ty.IsListType(): - return fmt.Sprintf("list(%s)", etyString) - case ty.IsSetType(): - return fmt.Sprintf("set(%s)", etyString) - case ty.IsMapType(): - return fmt.Sprintf("map(%s)", etyString) - default: - // Should never happen because the above is exhaustive - panic("unsupported collection type") - } - } - - if ty.IsObjectType() { - var buf bytes.Buffer - buf.WriteString("object({") - atys := ty.AttributeTypes() - names := make([]string, 0, len(atys)) - for name := range atys { - names = append(names, name) - } - sort.Strings(names) - first := true - for _, name := range names { - aty := atys[name] - if !first { - buf.WriteByte(',') - } - if !hclsyntax.ValidIdentifier(name) { - // Should never happen for any type produced by this package, - // but we'll do something reasonable here just so we don't - // produce garbage if someone gives us a hand-assembled object - // type that has weird attribute names. - // Using Go-style quoting here isn't perfect, since it doesn't - // exactly match HCL syntax, but it's fine for an edge-case. - buf.WriteString(fmt.Sprintf("%q", name)) - } else { - buf.WriteString(name) - } - buf.WriteByte('=') - buf.WriteString(TypeString(aty)) - first = false - } - buf.WriteString("})") - return buf.String() - } - - if ty.IsTupleType() { - var buf bytes.Buffer - buf.WriteString("tuple([") - etys := ty.TupleElementTypes() - first := true - for _, ety := range etys { - if !first { - buf.WriteByte(',') - } - buf.WriteString(TypeString(ety)) - first = false - } - buf.WriteString("])") - return buf.String() - } - - // Should never happen because we covered all cases above. - panic(fmt.Errorf("unsupported type %#v", ty)) -} diff --git a/pkg/steampipeconfig/inputvars/typeexpr/type_type.go b/pkg/steampipeconfig/inputvars/typeexpr/type_type.go deleted file mode 100644 index e72bf6bef..000000000 --- a/pkg/steampipeconfig/inputvars/typeexpr/type_type.go +++ /dev/null @@ -1,119 +0,0 @@ -package typeexpr - -import ( - "fmt" - "reflect" - - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/ext/customdecode" - "github.com/zclconf/go-cty/cty" - "github.com/zclconf/go-cty/cty/convert" - "github.com/zclconf/go-cty/cty/function" -) - -// TypeConstraintType is a cty capsule type that allows cty type constraints to -// be used as values. -// -// If TypeConstraintType is used in a context supporting the -// customdecode.CustomExpressionDecoder extension then it will implement -// expression decoding using the TypeConstraint function, thus allowing -// type expressions to be used in contexts where value expressions might -// normally be expected, such as in arguments to function calls. -var TypeConstraintType cty.Type - -// TypeConstraintVal constructs a cty.Value whose type is -// TypeConstraintType. -func TypeConstraintVal(ty cty.Type) cty.Value { - return cty.CapsuleVal(TypeConstraintType, &ty) -} - -// TypeConstraintFromVal extracts the type from a cty.Value of -// TypeConstraintType that was previously constructed using TypeConstraintVal. -// -// If the given value isn't a known, non-null value of TypeConstraintType -// then this function will panic. -func TypeConstraintFromVal(v cty.Value) cty.Type { - if !v.Type().Equals(TypeConstraintType) { - panic("value is not of TypeConstraintType") - } - ptr := v.EncapsulatedValue().(*cty.Type) - return *ptr -} - -// ConvertFunc is a cty function that implements type conversions. -// -// Its signature is as follows: -// -// convert(value, type_constraint) -// -// ...where type_constraint is a type constraint expression as defined by -// typeexpr.TypeConstraint. -// -// It relies on HCL's customdecode extension and so it's not suitable for use -// in non-HCL contexts or if you are using a HCL syntax implementation that -// does not support customdecode for function arguments. However, it _is_ -// supported for function calls in the HCL native expression syntax. -var ConvertFunc function.Function - -func init() { - TypeConstraintType = cty.CapsuleWithOps("type constraint", reflect.TypeOf(cty.Type{}), &cty.CapsuleOps{ - ExtensionData: func(key interface{}) interface{} { - switch key { - case customdecode.CustomExpressionDecoder: - return customdecode.CustomExpressionDecoderFunc( - func(expr hcl.Expression, ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) { - ty, diags := TypeConstraint(expr) - if diags.HasErrors() { - return cty.NilVal, diags - } - return TypeConstraintVal(ty), nil - }, - ) - default: - return nil - } - }, - TypeGoString: func(_ reflect.Type) string { - return "typeexpr.TypeConstraintType" - }, - GoString: func(raw interface{}) string { - tyPtr := raw.(*cty.Type) - return fmt.Sprintf("typeexpr.TypeConstraintVal(%#v)", *tyPtr) - }, - RawEquals: func(a, b interface{}) bool { - aPtr := a.(*cty.Type) - bPtr := b.(*cty.Type) - return (*aPtr).Equals(*bPtr) - }, - }) - - ConvertFunc = function.New(&function.Spec{ - Params: []function.Parameter{ - { - Name: "value", - Type: cty.DynamicPseudoType, - AllowNull: true, - AllowDynamicType: true, - }, - { - Name: "type", - Type: TypeConstraintType, - }, - }, - Type: func(args []cty.Value) (cty.Type, error) { - wantTypePtr := args[1].EncapsulatedValue().(*cty.Type) - got, err := convert.Convert(args[0], *wantTypePtr) - if err != nil { - return cty.NilType, function.NewArgError(0, err) - } - return got.Type(), nil - }, - Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { - v, err := convert.Convert(args[0], retType) - if err != nil { - return cty.NilVal, function.NewArgError(0, err) - } - return v, nil - }, - }) -} diff --git a/pkg/steampipeconfig/inputvars/ui_input.go b/pkg/steampipeconfig/inputvars/ui_input.go deleted file mode 100644 index b037e7ed9..000000000 --- a/pkg/steampipeconfig/inputvars/ui_input.go +++ /dev/null @@ -1,187 +0,0 @@ -package inputvars - -import ( - "bufio" - "bytes" - "context" - "errors" - "fmt" - "io" - "os" - "os/signal" - "strings" - "sync" - "sync/atomic" - "unicode" - - "github.com/bgentry/speakeasy" - "github.com/mattn/go-isatty" - "github.com/mitchellh/colorstring" - "github.com/turbot/terraform-components/terraform" -) - -var defaultInputReader io.Reader -var defaultInputWriter io.Writer -var testInputResponse []string -var testInputResponseMap map[string]string - -// UIInput is an implementation of terraform.UIInput that asks the CLI -// for input stdin. -type UIInput struct { - // Colorize will color the output. - Colorize *colorstring.Colorize - - // Reader and Writer for IO. If these aren't set, they will default to - // Stdin and Stdout respectively. - Reader io.Reader - Writer io.Writer - - listening int32 - result chan string - err chan string - - interrupted bool - l sync.Mutex - once sync.Once -} - -func (i *UIInput) Input(ctx context.Context, opts *terraform.InputOpts) (string, error) { - i.once.Do(i.init) - - r := i.Reader - w := i.Writer - if r == nil { - r = defaultInputReader - } - if w == nil { - w = defaultInputWriter - } - if r == nil { - r = os.Stdin - } - if w == nil { - w = os.Stdout - } - - // Make sure we only ask for input once at a time. Steampipe - // should enforce this, but it doesn't hurt to verify. - i.l.Lock() - defer i.l.Unlock() - - // If we're interrupted, then don't ask for input - if i.interrupted { - return "", errors.New("interrupted") - } - - // If we have test results, return those. testInputResponse is the - // "old" way of doing it and we should remove that. - if testInputResponse != nil { - v := testInputResponse[0] - testInputResponse = testInputResponse[1:] - return v, nil - } - - // testInputResponseMap is the new way for test responses, based on - // the query ID. - if testInputResponseMap != nil { - v, ok := testInputResponseMap[opts.Id] - if !ok { - return "", fmt.Errorf("unexpected input request in test: %s", opts.Id) - } - - return v, nil - } - - // Listen for interrupts so we can cancel the input ask - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, os.Interrupt) - defer signal.Stop(sigCh) - - // Build the output format for asking - var buf bytes.Buffer - buf.WriteString("[reset]") - buf.WriteString(fmt.Sprintf("[bold]%s[reset]\n", opts.Query)) - if opts.Description != "" { - s := bufio.NewScanner(strings.NewReader(opts.Description)) - for s.Scan() { - buf.WriteString(fmt.Sprintf(" %s\n", s.Text())) - } - buf.WriteString("\n") - } - if opts.Default != "" { - buf.WriteString(" [bold]Default:[reset] ") - buf.WriteString(opts.Default) - buf.WriteString("\n") - } - buf.WriteString(" [bold]Enter a value:[reset] ") - - // Ask the user for their input - if _, err := fmt.Fprint(w, i.Colorize.Color(buf.String())); err != nil { //nolint:forbidigo // acceptable - return "", err - } - - // Listen for the input in a goroutine. This will allow us to - // interrupt this if we are interrupted (SIGINT). - go func() { - if !atomic.CompareAndSwapInt32(&i.listening, 0, 1) { - return // We are already listening for input. - } - defer atomic.CompareAndSwapInt32(&i.listening, 1, 0) - - var line string - var err error - if opts.Secret && isatty.IsTerminal(os.Stdin.Fd()) { - line, err = speakeasy.Ask("") - } else { - buf := bufio.NewReader(r) - line, err = buf.ReadString('\n') - } - if err != nil { - i.err <- err.Error() - } else { - i.result <- strings.TrimRightFunc(line, unicode.IsSpace) - } - }() - - select { - case err := <-i.err: - return "", errors.New(err) - - case line := <-i.result: - fmt.Fprint(w, "\n") //nolint:forbidigo // acceptable - - if line == "" { - line = opts.Default - } - - return line, nil - case <-ctx.Done(): - fmt.Printf("ctx.Done()\n") //nolint:forbidigo // acceptable - // Print a newline so that any further output starts properly - // on a new line. - fmt.Fprintln(w) //nolint:forbidigo // acceptable - - return "", ctx.Err() - case <-sigCh: - fmt.Printf("SIG\n") //nolint:forbidigo // acceptable - // Print a newline so that any further output starts properly - // on a new line. - fmt.Fprintln(w) //nolint:forbidigo // acceptable - - // Mark that we were interrupted so future Ask calls fail. - i.interrupted = true - - return "", errors.New("interrupted") - } -} - -func (i *UIInput) init() { - i.result = make(chan string) - i.err = make(chan string) - - if i.Colorize == nil { - i.Colorize = &colorstring.Colorize{ - Colors: colorstring.DefaultColors, - } - } -} diff --git a/pkg/steampipeconfig/inputvars/unparsed_value.go b/pkg/steampipeconfig/inputvars/unparsed_value.go deleted file mode 100644 index 769d1267f..000000000 --- a/pkg/steampipeconfig/inputvars/unparsed_value.go +++ /dev/null @@ -1,210 +0,0 @@ -package inputvars - -import ( - "fmt" - "github.com/zclconf/go-cty/cty" - "log" - "strings" - - "github.com/hashicorp/hcl/v2" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig/var_config" - "github.com/turbot/terraform-components/tfdiags" -) - -// UnparsedVariableValue represents a variable value provided by the caller -// whose parsing must be deferred until configuration is available. -// -// This exists to allow processing of variable-setting arguments (e.g. in the -// command package) to be separated from parsing (in the backend package). -type UnparsedVariableValue interface { - // ParseVariableValue information in the provided variable configuration - // to parse (if necessary) and return the variable value encapsulated in - // the receiver. - // - // If error diagnostics are returned, the resulting value may be invalid - // or incomplete. - ParseVariableValue(mode var_config.VariableParsingMode) (*InputValue, tfdiags.Diagnostics) -} - -// ParseVariableValues processes a map of unparsed variable values by -// correlating each one with the given variable declarations which should -// be from a configuration. -// -// The map of unparsed variable values should include variables from all -// possible configuration declarations sources such that it is as complete as -// it can possibly be for the current operation. If any declared variables -// are not included in the map, ParseVariableValues will either substitute -// a configured default value or produce an error. -// -// If this function returns without any errors in the diagnostics, the -// resulting input values map is guaranteed to be valid and ready to pass -// to terraform.NewContext. If the diagnostics contains errors, the returned -// InputValues may be incomplete but will include the subset of variables -// that were successfully processed, allowing for careful analysis of the -// partial result. -func ParseVariableValues(inputValuesUnparsed map[string]UnparsedVariableValue, variablesMap *modconfig.ModVariableMap, validate bool) (InputValues, tfdiags.Diagnostics) { - - var diags tfdiags.Diagnostics - ret := make(InputValues, len(inputValuesUnparsed)) - - publicVariables := variablesMap.PublicVariables - - // Currently we're generating only warnings for undeclared variables - // defined in files (see below) but we only want to generate a few warnings - // at a time because existing deployments may have lots of these and - // the result can therefore be overwhelming. - seenUndeclaredInFile := 0 - - for name, unparsedVal := range inputValuesUnparsed { - var mode var_config.VariableParsingMode - config, declared := publicVariables[name] - if declared { - mode = config.ParsingMode - } else { - mode = var_config.VariableParseLiteral - } - - val, valDiags := unparsedVal.ParseVariableValue(mode) - diags = diags.Append(valDiags) - if valDiags.HasErrors() { - continue - } - - if !declared { - switch val.SourceType { - case ValueFromConfig, ValueFromAutoFile, ValueFromNamedFile: - // We allow undeclared names for variable values from files and warn in case - // users have forgotten a variable {} declaration or have a typo in their var name. - // Some users will actively ignore this warning because they use a .tfvars file - // across multiple configurations. - if seenUndeclaredInFile < 2 { - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Warning, - "Value for undeclared variable", - getUndeclaredVariableError(name, variablesMap), //, val.SourceRange.Filename), - )) - } - seenUndeclaredInFile++ - - case ValueFromEnvVar: - // We allow and ignore undeclared names for environment - // variables, because users will often set these globally - // when they are used across many (but not necessarily all) - // configurations. - case ValueFromCLIArg: - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Error, - "Value for undeclared variable", - getUndeclaredVariableError(name, variablesMap), - )) - default: - // For all other source types we are more vague, but other situations - // don't generally crop up at this layer in practice. - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Error, - "Value for undeclared variable", - getUndeclaredVariableError(name, variablesMap), - )) - } - continue - } - - ret[name] = val - } - - if seenUndeclaredInFile > 2 { - extras := seenUndeclaredInFile - 2 - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagWarning, - Summary: "Values for undeclared variables", - Detail: fmt.Sprintf("In addition to the other similar warnings shown, %d other variable(s) defined without being declared.", extras), - }) - } - - // By this point we should've gathered all of the required variables - // from one of the many possible sources. - // We'll now populate any we haven't gathered as their defaults and fail if any of the - // missing ones are required. - for name, vc := range publicVariables { - if _, defined := ret[name]; defined { - continue - } - - // are we missing a required variable? - if vc.Required() { - - // We'll include a placeholder value anyway, just so that our - // result is complete for any calling code that wants to cautiously - // analyze it for diagnostic purposes. Since our diagnostics now - // includes an error, normal processing will ignore this result. - ret[name] = &InputValue{ - Value: cty.DynamicVal, - SourceType: ValueFromConfig, - SourceRange: tfdiags.SourceRangeFromHCL(vc.DeclRange), - } - - // if validation flag is set, raise an error - if validate { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "No value for required variable", - Detail: fmt.Sprintf("The input variable %q is not set, and has no default value. Use a --var or --var-file command line argument to provide a value for this variable.", name), - Subject: vc.DeclRange.Ptr(), - }) - } - } else { - // not required - use default - ret[name] = &InputValue{ - Value: vc.Default, - SourceType: ValueFromConfig, - SourceRange: tfdiags.SourceRangeFromHCL(vc.DeclRange), - } - log.Printf("[INFO] adding value for variable '%s' from var default", name) - } - } - - return ret, diags -} - -func getUndeclaredVariableError(name string, variablesMap *modconfig.ModVariableMap) string { - // is this a qualified variable? - if len(strings.Split(name, ".")) == 1 { - // unqualifid - return fmt.Sprintf("\"%s\". If you meant to use this value, add a \"variable\" block to the mod.\n", name) - } - - // parse to extract the mod name - parsedVarName, err := modconfig.ParseResourceName(name) - if err != nil { - return fmt.Sprintf("Invalid variable name: \"%s\". It should be of form \"var_name\" or \"mod_name.var_name\".", name) - } - - for _, m := range variablesMap.Mod.ResourceMaps.Mods { - if m.ShortName == parsedVarName.Mod { - return fmt.Sprintf("\"%s\": Dependency mod \"%s\" has no variable \"%s\"", name, m.DependencyName, parsedVarName.Name) - // so it is a dependency mod - } - } - return fmt.Sprintf("\"%s\": Mod \"%s\" is not a dependency of the current workspace.", name, parsedVarName.Mod) - -} - -type UnparsedInteractiveVariableValue struct { - Name, RawValue string -} - -//var _ UnparsedVariableValue = UnparsedInteractiveVariableValue{} - -func (v UnparsedInteractiveVariableValue) ParseVariableValue(mode var_config.VariableParsingMode) (*InputValue, tfdiags.Diagnostics) { - var diags tfdiags.Diagnostics - val, valDiags := mode.Parse(v.Name, v.RawValue) - diags = diags.Append(valDiags) - if diags.HasErrors() { - return nil, diags - } - return &InputValue{ - Value: val, - SourceType: ValueFromInput, - }, diags -} diff --git a/pkg/steampipeconfig/load_config.go b/pkg/steampipeconfig/load_config.go index ec72e459a..73a9c0e81 100644 --- a/pkg/steampipeconfig/load_config.go +++ b/pkg/steampipeconfig/load_config.go @@ -18,8 +18,10 @@ import ( perror_helpers "github.com/turbot/pipe-fittings/error_helpers" pfilepaths "github.com/turbot/pipe-fittings/filepaths" "github.com/turbot/pipe-fittings/hclhelpers" + "github.com/turbot/pipe-fittings/modconfig" poptions "github.com/turbot/pipe-fittings/options" pparse "github.com/turbot/pipe-fittings/parse" + "github.com/turbot/pipe-fittings/schema" "github.com/turbot/pipe-fittings/utils" "github.com/turbot/pipe-fittings/versionfile" "github.com/turbot/pipe-fittings/workspace_profile" @@ -29,8 +31,6 @@ import ( "github.com/turbot/steampipe/pkg/error_helpers" "github.com/turbot/steampipe/pkg/filepaths" "github.com/turbot/steampipe/pkg/options" - "github.com/turbot/steampipe/pkg/parse" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" ) var GlobalWorkspaceProfile *workspace_profile.SteampipeWorkspaceProfile @@ -271,7 +271,7 @@ func loadConfig(ctx context.Context, configFolder string, steampipeConfig *Steam for _, block := range content.Blocks { switch block.Type { - case modconfig.BlockTypePlugin: + case schema.BlockTypePlugin: plugin, moreDiags := pparse.DecodePlugin(block) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { @@ -283,8 +283,8 @@ func loadConfig(ctx context.Context, configFolder string, steampipeConfig *Steam return perror_helpers.NewErrorsAndWarning(err) } - case modconfig.BlockTypeConnection: - connection, moreDiags := parse.DecodeConnection(block) + case schema.BlockTypeConnection: + connection, moreDiags := pparse.DecodeConnection(block) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { continue @@ -298,7 +298,7 @@ func loadConfig(ctx context.Context, configFolder string, steampipeConfig *Steam } steampipeConfig.Connections[connection.Name] = connection - case modconfig.BlockTypeOptions: + case schema.BlockTypeOptions: // check this options type is permitted based on the options passed in if err := optionsBlockPermitted(block, optionBlockMap, opts); err != nil { return perror_helpers.NewErrorsAndWarning(err) @@ -345,7 +345,7 @@ func loadConfig(ctx context.Context, configFolder string, steampipeConfig *Steam return res } -func getDuplicateConnectionError(existingConnection, newConnection *modconfig.Connection) error { +func getDuplicateConnectionError(existingConnection, newConnection *modconfig.SteampipeConnection) error { return sperr.New("duplicate connection name: '%s'\n\t(%s:%d)\n\t(%s:%d)", existingConnection.Name, existingConnection.DeclRange.Filename, existingConnection.DeclRange.Start.Line, newConnection.DeclRange.Filename, newConnection.DeclRange.Start.Line) diff --git a/pkg/steampipeconfig/load_config_test.go b/pkg/steampipeconfig/load_config_test.go index 931964203..8d751ec47 100644 --- a/pkg/steampipeconfig/load_config_test.go +++ b/pkg/steampipeconfig/load_config_test.go @@ -9,10 +9,9 @@ import ( "testing" "github.com/turbot/pipe-fittings/app_specific" - "github.com/turbot/pipe-fittings/options" + "github.com/turbot/pipe-fittings/modconfig" "github.com/turbot/pipe-fittings/plugin" "github.com/turbot/pipe-fittings/utils" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" "golang.org/x/exp/maps" ) @@ -51,7 +50,7 @@ var testCasesLoadConfig = map[string]loadConfigTest{ "multiple_connections": { steampipeDir: "testdata/connection_config/multiple_connections", expected: &SteampipeConfig{ - Connections: map[string]*modconfig.Connection{ + Connections: map[string]*modconfig.SteampipeConnection{ "aws_dmi_001": { Name: "aws_dmi_001", PluginAlias: "aws", @@ -97,10 +96,6 @@ var testCasesLoadConfig = map[string]loadConfigTest{ }, }, }, - DefaultConnectionOptions: &options.Connection{ - Cache: &trueVal, - CacheTTL: &ttlVal, - }, }, }, //"single_connection": { @@ -393,18 +388,12 @@ func SteampipeConfigEquals(left, right *SteampipeConfig) bool { } if !maps.EqualFunc(left.Connections, right.Connections, - func(c1, c2 *modconfig.Connection) bool { return c1.Equals(c2) }) { - return false - } - if !reflect.DeepEqual(left.DefaultConnectionOptions, right.DefaultConnectionOptions) { + func(c1, c2 *modconfig.SteampipeConnection) bool { return c1.Equals(c2) }) { return false } if !reflect.DeepEqual(left.DatabaseOptions, right.DatabaseOptions) { return false } - if !reflect.DeepEqual(left.TerminalOptions, right.TerminalOptions) { - return false - } if !reflect.DeepEqual(left.GeneralOptions, right.GeneralOptions) { return false } diff --git a/pkg/steampipeconfig/modconfig/attributes.go b/pkg/steampipeconfig/modconfig/attributes.go deleted file mode 100644 index 856250cd2..000000000 --- a/pkg/steampipeconfig/modconfig/attributes.go +++ /dev/null @@ -1,65 +0,0 @@ -package modconfig - -import ( - "log" - "reflect" - - "github.com/hashicorp/hcl/v2/hcldec" - "github.com/turbot/go-kit/helpers" - "github.com/turbot/terraform-components/configs/configschema" - "github.com/zclconf/go-cty/cty" - "github.com/zclconf/go-cty/cty/gocty" -) - -// GetCtyTypes builds a map of cty types for all tagged properties. -// It is used to convert the struct to a cty value -func GetCtyTypes(item interface{}) map[string]cty.Type { - defer func() { - if r := recover(); r != nil { - log.Printf("[WARN] GetCtyTypes failed with panic: %v", r) - } - }() - var res = make(map[string]cty.Type) - - t := reflect.TypeOf(helpers.DereferencePointer(item)) - val := reflect.ValueOf(item) - if val.Type().Kind() == reflect.Ptr { - val = val.Elem() - } - - for i := 0; i < val.NumField(); i++ { - structField := t.Field(i) - attribute, ok := structField.Tag.Lookup("cty") - if ok && attribute != "-" { - valField := val.Field(i) - // get cty type - ctyType, err := gocty.ImpliedType(valField.Interface()) - if err != nil { - panic(err) - } - - res[attribute] = ctyType - } - } - return res -} - -// GetCtyValue converts the item into a cty value -func GetCtyValue(item interface{}) (cty.Value, error) { - // TODO [node_reuse] look at merging base struct schemas - - // build the block schema - var block = configschema.Block{Attributes: make(map[string]*configschema.Attribute)} - - // get the hcl attributes - these include the cty type - for attribute, ctyType := range GetCtyTypes(item) { - // TODO how to determine optional? - block.Attributes[attribute] = &configschema.Attribute{Optional: true, Type: ctyType} - } - - // get cty spec - spec := block.DecoderSpec() - ty := hcldec.ImpliedType(spec) - - return gocty.ToCtyValue(item, ty) -} diff --git a/pkg/steampipeconfig/modconfig/block_type.go b/pkg/steampipeconfig/modconfig/block_type.go deleted file mode 100644 index 789c0577e..000000000 --- a/pkg/steampipeconfig/modconfig/block_type.go +++ /dev/null @@ -1,131 +0,0 @@ -package modconfig - -import "github.com/turbot/go-kit/helpers" - -// NOTE: when adding a block type, be sure to update QueryProviderBlocks/ReferenceBlocks/AllBlockTypes as needed -const ( - // require blocks - BlockTypeSteampipe = "steampipe" - BlockTypeMod = "mod" - // resource blocks - BlockTypeQuery = "query" - BlockTypeControl = "control" - BlockTypeBenchmark = "benchmark" - BlockTypeDashboard = "dashboard" - BlockTypeContainer = "container" - BlockTypeChart = "chart" - BlockTypeCard = "card" - BlockTypeFlow = "flow" - BlockTypeGraph = "graph" - BlockTypeHierarchy = "hierarchy" - BlockTypeImage = "image" - BlockTypeInput = "input" - BlockTypeTable = "table" - BlockTypeText = "text" - BlockTypeLocals = "locals" - BlockTypeVariable = "variable" - BlockTypeParam = "param" - BlockTypeRequire = "require" - BlockTypeNode = "node" - BlockTypeEdge = "edge" - BlockTypeLegacyRequires = "requires" - BlockTypeCategory = "category" - BlockTypeWith = "with" - - // config blocks - BlockTypeRateLimiter = "limiter" - BlockTypePlugin = "plugin" - BlockTypeConnection = "connection" - BlockTypeOptions = "options" - BlockTypeWorkspaceProfile = "workspace" - - ResourceTypeSnapshot = "snapshot" - AttributeArgs = "args" - AttributeQuery = "query" -) - -// QueryProviderBlocks is a list of block types which implement QueryProvider -var QueryProviderBlocks = []string{ - BlockTypeCard, - BlockTypeChart, - BlockTypeControl, - BlockTypeEdge, - BlockTypeFlow, - BlockTypeGraph, - BlockTypeHierarchy, - BlockTypeImage, - BlockTypeInput, - BlockTypeQuery, - BlockTypeNode, - BlockTypeTable, - BlockTypeWith, -} - -// NodeAndEdgeProviderBlocks is a list of block types which implementnodeAndEdgeProvider -var NodeAndEdgeProviderBlocks = []string{ - BlockTypeHierarchy, - BlockTypeFlow, - BlockTypeGraph, -} - -// ReferenceBlocks is a list of block types we store references for -var ReferenceBlocks = []string{ - BlockTypeMod, - BlockTypeQuery, - BlockTypeControl, - BlockTypeBenchmark, - BlockTypeDashboard, - BlockTypeContainer, - BlockTypeCard, - BlockTypeChart, - BlockTypeFlow, - BlockTypeGraph, - BlockTypeHierarchy, - BlockTypeImage, - BlockTypeInput, - BlockTypeTable, - BlockTypeText, - BlockTypeParam, - BlockTypeCategory, - BlockTypeWith, -} - -var ValidResourceItemTypes = []string{ - BlockTypeMod, - BlockTypeQuery, - BlockTypeControl, - BlockTypeBenchmark, - BlockTypeDashboard, - BlockTypeContainer, - BlockTypeChart, - BlockTypeCard, - BlockTypeFlow, - BlockTypeGraph, - BlockTypeHierarchy, - BlockTypeImage, - BlockTypeInput, - BlockTypeTable, - BlockTypeText, - BlockTypeLocals, - BlockTypeVariable, - BlockTypeParam, - BlockTypeRequire, - BlockTypeNode, - BlockTypeEdge, - BlockTypeLegacyRequires, - BlockTypeCategory, - BlockTypeConnection, - BlockTypeOptions, - BlockTypeWorkspaceProfile, - BlockTypeWith, - // local is not an actual block name but is a resource type - "local", - // references - "ref", - // variables - "var", -} - -func IsValidResourceItemType(blockType string) bool { - return helpers.StringSliceContains(ValidResourceItemTypes, blockType) -} diff --git a/pkg/steampipeconfig/modconfig/connection.go b/pkg/steampipeconfig/modconfig/connection.go deleted file mode 100644 index 31422b953..000000000 --- a/pkg/steampipeconfig/modconfig/connection.go +++ /dev/null @@ -1,230 +0,0 @@ -package modconfig - -import ( - "fmt" - "github.com/turbot/pipe-fittings/plugin" - "log" - "path" - "strings" - - "github.com/hashicorp/hcl/v2" - "github.com/turbot/go-kit/helpers" - "github.com/turbot/pipe-fittings/hclhelpers" - "github.com/turbot/pipe-fittings/utils" - "github.com/turbot/steampipe/pkg/constants" - "golang.org/x/exp/maps" -) - -const ( - ConnectionTypePlugin = "plugin" - ConnectionTypeAggregator = "aggregator" - ImportSchemaEnabled = "enabled" - ImportSchemaDisabled = "disabled" -) - -var ValidImportSchemaValues = []string{ImportSchemaEnabled, ImportSchemaDisabled} - -// Connection is a struct representing the partially parsed connection -// -// (Partial as the connection config, which is plugin specific, is stored as raw HCL. -// This will be parsed by the plugin) -// json tags needed as this is stored in the connection state file -type Connection struct { - // connection name - Name string `json:"name"` - // name of plugin as mentioned in config - this may be an alias to a plugin image ref - // OR the label of a plugin config - PluginAlias string `json:"plugin_short_name"` - // image ref plugin. - // we resolve this after loading all plugin configs - Plugin string `json:"plugin"` - // the label of the plugin config we are using - PluginInstance *string `json:"plugin_instance"` - // Path to the installed plugin (if it exists) - PluginPath *string - // connection type - supported values: "aggregator" - Type string `json:"type,omitempty"` - // should a schema be created for this connection - supported values: "enabled", "disabled" - ImportSchema string `json:"import_schema"` - // list of names or wildcards which are resolved to connections - // (only valid for "aggregator" type) - ConnectionNames []string `json:"connections,omitempty"` - // a map of the resolved child connections - // (only valid for "aggregator" type) - Connections map[string]*Connection `json:"-"` - // a list of the names resolved child connections - // (only valid for "aggregator" type) - ResolvedConnectionNames []string `json:"resolved_connections,omitempty"` - // unparsed HCL of plugin specific connection config - Config string `json:"config,omitempty"` - - Error error - - DeclRange plugin.Range `json:"decl_range"` -} - -func (c *Connection) GetDeclRange() plugin.Range { - return c.DeclRange -} - -func (c *Connection) GetName() string { - return c.Name -} - -func (c *Connection) GetDisplayName() string { - if c.ImportDisabled() { - return fmt.Sprintf("%s (disabled)", c.Name) - } - return c.Name -} - -func NewConnection(block *hcl.Block) *Connection { - return &Connection{ - Name: block.Labels[0], - DeclRange: plugin.NewRange(hclhelpers.BlockRange(block)), - ImportSchema: ImportSchemaEnabled, - // default to plugin - Type: ConnectionTypePlugin, - } -} - -func (c *Connection) ImportDisabled() bool { - return c.ImportSchema == constants.ConnectionStateDisabled -} - -func (c *Connection) Equals(other *Connection) bool { - return c.Name == other.Name && - c.Plugin == other.Plugin && - c.Type == other.Type && - strings.Join(c.ConnectionNames, ",") == strings.Join(other.ConnectionNames, ",") && - c.Config == other.Config && - c.ImportSchema == other.ImportSchema - -} - -func (c *Connection) String() string { - return fmt.Sprintf("\n----\nName: %s\nPlugin: %s\nConfig:\n%s\n", c.Name, c.Plugin, c.Config) -} - -// Validate verifies the Type property is valid, -// if this is an aggregator connection, there must be at least one child, and no duplicates -// if this is NOT an aggregator, there must be no children -func (c *Connection) Validate(map[string]*Connection) (warnings []string, errors []string) { - validConnectionTypes := []string{ConnectionTypePlugin, ConnectionTypeAggregator} - if !helpers.StringSliceContains(validConnectionTypes, c.Type) { - return nil, []string{fmt.Sprintf("connection '%s' has invalid connection type '%s'", c.Name, c.Type)} - } - - if c.Type == ConnectionTypeAggregator { - return c.ValidateAggregatorConnection() - } - - // this is NOT an aggregator group - there should be no children - var validationErrors []string - - if len(c.ConnectionNames) != 0 { - validationErrors = append(validationErrors, fmt.Sprintf("connection '%s' has %d children, but is not of type 'aggregator'", c.Name, len(c.ConnectionNames))) - } - validImportSchemaValues := utils.SliceToLookup(ValidImportSchemaValues) - if _, isValid := validImportSchemaValues[c.ImportSchema]; !isValid { - validationErrors = append(validationErrors, fmt.Sprintf("invalid value '%s'for import_schema, must be one of ['%s']", c.ImportSchema, strings.Join(ValidImportSchemaValues, "','"))) - } - - return nil, validationErrors - -} - -func (c *Connection) ValidateAggregatorConnection() (warnings, errors []string) { - if len(c.Connections) == 0 { - /// there should be at least one connection - raise as warning - return []string{c.GetEmptyAggregatorError()}, nil - } - - var validationErrors []string - - // now ensure all child connections are loaded and use the same plugin as the parent connection - for _, childConnection := range c.Connections { - if childConnection.Plugin != c.Plugin { - validationErrors = append(validationErrors, - fmt.Sprintf("aggregator connection '%s' uses plugin %s but child connection '%s' uses plugin '%s'", - c.Name, - c.Plugin, - childConnection.Name, - childConnection.Plugin, - )) - } - - } - return nil, validationErrors -} - -func (c *Connection) GetEmptyAggregatorError() string { - patterns := c.ConnectionNames - if len(patterns) == 0 { - return fmt.Sprintf("aggregator '%s' defines no child connections", c.Name) - } - if len(patterns) == 1 { - return fmt.Sprintf("aggregator '%s' with pattern '%s' matches no connections", - c.Name, - patterns[0]) - } - return fmt.Sprintf("aggregator '%s' with patterns ['%s'] matches no connections", - c.Name, - strings.Join(patterns, "','")) -} - -func (c *Connection) PopulateChildren(connectionMap map[string]*Connection) []string { - log.Printf("[TRACE] Connection.PopulateChildren for aggregator connection %s", c.Name) - c.Connections = make(map[string]*Connection) - var failures []string - for _, childPattern := range c.ConnectionNames { - // if this resolves as an existing connection, populate it - if childConnection, ok := connectionMap[childPattern]; ok { - // verify this child connection has the same plugin instance - if childConnection.PluginInstance != c.PluginInstance { - msg := fmt.Sprintf("aggregator connection %s specifies child connection %s but it has a different plugin instance", - c.Name, childPattern) - log.Println("[WARN]", msg) - failures = append(failures, msg) - } else { - log.Printf("[TRACE] Connection.PopulateChildren found matching connection %s", childPattern) - c.Connections[childPattern] = childConnection - } - continue - } - - log.Printf("[TRACE] Connection.PopulateChildren no connection matches %s - treating as a wildcard", childPattern) - // otherwise treat the connection name as a wildcard and see what matches - for name, connection := range connectionMap { - // if this is an aggregator connection, skip (this will also avoid us adding ourselves) - if connection.Type == ConnectionTypeAggregator { - continue - } - // have we already added this connection - if _, ok := c.Connections[name]; ok { - continue - } - if match, _ := path.Match(childPattern, name); match { - // verify that this connection is the same plugin instance - if connection.PluginInstance == c.PluginInstance { - c.Connections[name] = connection - log.Printf("[TRACE] connection '%s' matches pattern '%s'", name, childPattern) - } - } - } - } - c.ResolvedConnectionNames = maps.Keys(c.Connections) - return failures -} - -// GetResolveConnectionNames return the names of all child connections -// (will only be non-empty for aggregator connections) -func (c *Connection) GetResolveConnectionNames() []string { - res := make([]string, len(c.Connections)) - idx := 0 - for k := range c.Connections { - res[idx] = k - idx++ - } - return res -} diff --git a/pkg/steampipeconfig/modconfig/connection_test.go b/pkg/steampipeconfig/modconfig/connection_test.go deleted file mode 100644 index f8f55ec5e..000000000 --- a/pkg/steampipeconfig/modconfig/connection_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package modconfig - -import "testing" - -type connectionEquality struct { - connection1 *Connection - connection2 *Connection - expectation bool -} - -var conn1 *Connection = &Connection{ - Name: "connection", - Config: "hcl_helpers", -} - -var conn1_duplicate *Connection = &Connection{ - Name: "connection", - Config: "hcl_helpers", -} - -var other_conn *Connection = &Connection{ - Name: "connection2", - Config: "connection_config2", -} - -var equalsCases = map[string]connectionEquality{ - "expected_equal": {connection1: conn1, connection2: conn1_duplicate, expectation: true}, - "not_expected_equal": {connection1: conn1, connection2: other_conn, expectation: false}, -} - -func TestConnectionEquals(t *testing.T) { - for caseName, caseData := range equalsCases { - isEqual := caseData.connection1.Equals(caseData.connection2) - if caseData.expectation != isEqual { - t.Errorf(`Test: '%s' FAILED: expected: %v, actual: %v`, caseName, caseData.expectation, isEqual) - } - } -} diff --git a/pkg/steampipeconfig/modconfig/interfaces.go b/pkg/steampipeconfig/modconfig/interfaces.go deleted file mode 100644 index 376007819..000000000 --- a/pkg/steampipeconfig/modconfig/interfaces.go +++ /dev/null @@ -1,9 +0,0 @@ -package modconfig - -import ( - "github.com/zclconf/go-cty/cty" -) - -type CtyValueProvider interface { - CtyValue() (cty.Value, error) -} diff --git a/pkg/steampipeconfig/modconfig/named_item.go b/pkg/steampipeconfig/modconfig/named_item.go deleted file mode 100644 index 16c639e79..000000000 --- a/pkg/steampipeconfig/modconfig/named_item.go +++ /dev/null @@ -1,20 +0,0 @@ -package modconfig - -// NamedItem is a struct used by benchmark, container and dashboard to specify children of different types -type NamedItem struct { - Name string `cty:"name"` -} - -func (c NamedItem) String() string { - return c.Name -} - -type NamedItemList []NamedItem - -func (l NamedItemList) StringList() []string { - res := make([]string, len(l)) - for i, n := range l { - res[i] = n.Name - } - return res -} diff --git a/pkg/steampipeconfig/modconfig/param_def.go b/pkg/steampipeconfig/modconfig/param_def.go deleted file mode 100644 index 33f4e9603..000000000 --- a/pkg/steampipeconfig/modconfig/param_def.go +++ /dev/null @@ -1,71 +0,0 @@ -package modconfig - -import ( - "encoding/json" - "fmt" - - "github.com/hashicorp/hcl/v2" - typehelpers "github.com/turbot/go-kit/types" - "github.com/turbot/pipe-fittings/hclhelpers" -) - -type ParamDef struct { - ShortName string `cty:"name" json:"name"` - UnqualifiedName string `cty:"full_name" json:"-"` - Description *string `cty:"description" json:"description"` - Default *string `cty:"default" json:"default"` - // tactical - is the raw value a string - IsString bool `cty:"is_string" json:"-"` - - DeclRange hcl.Range `json:"-"` -} - -func NewParamDef(block *hcl.Block) *ParamDef { - return &ParamDef{ - ShortName: block.Labels[0], - UnqualifiedName: fmt.Sprintf("param.%s", block.Labels[0]), - DeclRange: hclhelpers.BlockRange(block), - } -} - -func (p *ParamDef) String() string { - return fmt.Sprintf("Name: %s, Description: %s, Default: %s", p.UnqualifiedName, typehelpers.SafeString(p.Description), typehelpers.SafeString(p.Default)) -} - -func (p *ParamDef) Equals(other *ParamDef) bool { - return p.ShortName == other.ShortName && - typehelpers.SafeString(p.Description) == typehelpers.SafeString(other.Description) && - typehelpers.SafeString(p.Default) == typehelpers.SafeString(other.Default) -} - -// SetDefault sets the default as a atring points, marshalling to json is the underlying value is NOT a string -func (p *ParamDef) SetDefault(value interface{}) error { - strVal, ok := value.(string) - if ok { - p.IsString = true - // no need to convert to string - p.Default = &strVal - return nil - } - // format the arg value as a JSON string - jsonBytes, err := json.Marshal(value) - if err != nil { - return err - } - def := string(jsonBytes) - p.Default = &def - return nil -} - -// GetDefault returns the default as an interface{}, unmarshalling json is the underlying value was NOT a string -func (p *ParamDef) GetDefault() (any, error) { - if p.Default == nil { - return nil, nil - } - if p.IsString { - return *p.Default, nil - } - var val any - err := json.Unmarshal([]byte(*p.Default), &val) - return val, err -} diff --git a/pkg/steampipeconfig/modconfig/parsed_name.go b/pkg/steampipeconfig/modconfig/parsed_name.go deleted file mode 100644 index 1e648bd2c..000000000 --- a/pkg/steampipeconfig/modconfig/parsed_name.go +++ /dev/null @@ -1,88 +0,0 @@ -package modconfig - -import ( - "fmt" - "strings" - - "github.com/turbot/steampipe-plugin-sdk/v5/sperr" -) - -type ParsedResourceName struct { - Mod string - ItemType string - Name string -} - -func ParseResourceName(fullName string) (res *ParsedResourceName, err error) { - if fullName == "" { - return &ParsedResourceName{}, nil - } - res = &ParsedResourceName{} - - parts := strings.Split(fullName, ".") - - switch len(parts) { - case 0: - err = sperr.New("empty name passed to ParseResourceName") - case 1: - res.Name = parts[0] - case 2: - res.ItemType = parts[0] - res.Name = parts[1] - case 3: - res.Mod = parts[0] - res.ItemType = parts[1] - res.Name = parts[2] - default: - err = sperr.New("invalid name '%s' passed to ParseResourceName", fullName) - } - if !IsValidResourceItemType(res.ItemType) { - err = sperr.New("invalid name '%s' passed to ParseResourceName", fullName) - } - return -} - -func (p *ParsedResourceName) ToResourceName() string { - return BuildModResourceName(p.ItemType, p.Name) -} - -func (p *ParsedResourceName) ToFullName() (string, error) { - return BuildFullResourceName(p.Mod, p.ItemType, p.Name) -} - -func (p *ParsedResourceName) ToFullNameWithMod(mod string) (string, error) { - if p.Mod != "" { - return p.ToFullName() - } - return BuildFullResourceName(mod, p.ItemType, p.Name) -} - -// BuildFullResourceName generates a fully qualified name from the given components -// e.g: aws_compliance.benchmark.cis_v150_1 -func BuildFullResourceName(mod, blockType, name string) (string, error) { - if mod == "" { - return "", sperr.New("mod name not provided") - } - if blockType == "" { - return "", sperr.New("block type not provided") - } - if name == "" { - return "", sperr.New("resource name not provided") - } - return fmt.Sprintf("%s.%s.%s", mod, blockType, name), nil -} - -// UnqualifiedResourceName removes the mod prefix from the given name -func UnqualifiedResourceName(fullName string) string { - parts := strings.Split(fullName, ".") - switch len(parts) { - case 3: - return strings.Join(parts[1:], ".") - default: - return fullName - } -} - -func BuildModResourceName(blockType, name string) string { - return fmt.Sprintf("%s.%s", blockType, name) -} diff --git a/pkg/steampipeconfig/modconfig/query.go b/pkg/steampipeconfig/modconfig/query.go deleted file mode 100644 index 2798d3dd0..000000000 --- a/pkg/steampipeconfig/modconfig/query.go +++ /dev/null @@ -1,181 +0,0 @@ -package modconfig - -import ( - "github.com/hashicorp/hcl/v2" -) - -// Query is a struct representing the Query resource -type Query struct { - //ResourceWithMetadataImpl - //QueryProviderImpl - - // required to allow partial decoding - Remain hcl.Body `hcl:",remain" json:"-"` - - // only here as otherwise gocty.ImpliedType panics - Unused string `cty:"unused" json:"-"` -} - -// -//func NewQuery(block *hcl.Block, mod *Mod, shortName string) HclResource { -// fullName := fmt.Sprintf("%s.%s.%s", mod.ShortName, block.Type, shortName) -// // queries cannot be anonymous -// q := &Query{ -// QueryProviderImpl: QueryProviderImpl{ -// RuntimeDependencyProviderImpl: RuntimeDependencyProviderImpl{ -// ModTreeItemImpl: ModTreeItemImpl{ -// HclResourceImpl: HclResourceImpl{ -// ShortName: shortName, -// FullName: fullName, -// UnqualifiedName: fmt.Sprintf("%s.%s", block.Type, shortName), -// DeclRange: hclhelpers.BlockRange(block), -// blockType: block.Type, -// }, -// Mod: mod, -// }, -// }, -// }, -// } -// return q -//} -// -//func QueryFromFile(modPath, filePath string, mod *Mod) (MappableResource, []byte, error) { -// q := &Query{ -// QueryProviderImpl: QueryProviderImpl{ -// RuntimeDependencyProviderImpl: RuntimeDependencyProviderImpl{ -// ModTreeItemImpl: ModTreeItemImpl{ -// Mod: mod, -// }, -// }, -// }, -// } -// return q.InitialiseFromFile(modPath, filePath) -//} -// -//// InitialiseFromFile implements MappableResource -//func (q *Query) InitialiseFromFile(modPath, filePath string) (MappableResource, []byte, error) { -// // only valid for sql files -// if filepath.Ext(filePath) != constants.SqlExtension { -// return nil, nil, fmt.Errorf("Query.InitialiseFromFile must be called with .sql files only - filepath: '%s'", filePath) -// } -// -// sqlBytes, err := os.ReadFile(filePath) -// if err != nil { -// return nil, nil, err -// } -// -// sql := string(sqlBytes) -// if sql == "" { -// log.Printf("[TRACE] SQL file %s contains no query", filePath) -// return nil, nil, nil -// } -// // get a sluggified version of the filename -// name, err := PseudoResourceNameFromPath(modPath, filePath) -// if err != nil { -// return nil, nil, err -// } -// q.ShortName = name -// q.UnqualifiedName = fmt.Sprintf("query.%s", name) -// q.FullName = fmt.Sprintf("%s.query.%s", q.Mod.ShortName, name) -// q.SQL = &sql -// q.DeclRange = hcl.Range{ -// Filename: filePath, -// Start: hcl.Pos{ -// Line: 0, -// Column: 0, -// Byte: 0, -// }, -// End: hcl.Pos{ -// Line: len(sql), -// }, -// } -// -// return q, sqlBytes, nil -//} -// -//func (q *Query) Equals(other *Query) bool { -// res := q.ShortName == other.ShortName && -// q.FullName == other.FullName && -// typehelpers.SafeString(q.Description) == typehelpers.SafeString(other.Description) && -// typehelpers.SafeString(q.Documentation) == typehelpers.SafeString(other.Documentation) && -// typehelpers.SafeString(q.SQL) == typehelpers.SafeString(other.SQL) && -// typehelpers.SafeString(q.Title) == typehelpers.SafeString(other.Title) -// if !res { -// return res -// } -// -// // tags -// if q.Tags == nil { -// if other.Tags != nil { -// return false -// } -// } else { -// // we have tags -// if other.Tags == nil { -// return false -// } -// for k, v := range q.Tags { -// if otherVal, ok := (other.Tags)[k]; !ok && v != otherVal { -// return false -// } -// } -// } -// -// // params -// if len(q.Params) != len(other.Params) { -// return false -// } -// for i, p := range q.Params { -// if !p.Equals(other.Params[i]) { -// return false -// } -// } -// -// return true -//} -// -//func (q *Query) String() string { -// res := fmt.Sprintf(` -// ----- -// Name: %s -// Title: %s -// Description: %s -// SQL: %s -//`, q.FullName, types.SafeString(q.Title), types.SafeString(q.Description), types.SafeString(q.SQL)) -// -// // add param defs if there are any -// if len(q.Params) > 0 { -// var paramDefsStr = make([]string, len(q.Params)) -// for i, def := range q.Params { -// paramDefsStr[i] = def.String() -// } -// res += fmt.Sprintf("Params:\n\t%s\n ", strings.Join(paramDefsStr, "\n\t")) -// } -// return res -//} -// -//// OnDecoded implements HclResource -//func (q *Query) OnDecoded(*hcl.Block, ResourceMapsProvider) hcl.Diagnostics { -// return nil -//} -// -//// CtyValue implements CtyValueProvider -//func (q *Query) CtyValue() (cty.Value, error) { -// return GetCtyValue(q) -//} -// -//func (q *Query) Diff(other *Query) *DashboardTreeItemDiffs { -// res := &DashboardTreeItemDiffs{ -// Item: q, -// Name: q.Name(), -// } -// -// if !utils.SafeStringsEqual(q.FullName, other.FullName) { -// res.AddPropertyDiff("Name") -// } -// -// res.populateChildDiffs(q, other) -// res.queryProviderDiff(q, other) -// -// return res -//} diff --git a/pkg/steampipeconfig/modconfig/query_args.go b/pkg/steampipeconfig/modconfig/query_args.go deleted file mode 100644 index 495bc30e1..000000000 --- a/pkg/steampipeconfig/modconfig/query_args.go +++ /dev/null @@ -1,386 +0,0 @@ -package modconfig - -import ( - "encoding/json" - "fmt" - "strings" - - typehelpers "github.com/turbot/go-kit/types" - "github.com/turbot/pipe-fittings/utils" -) - -// QueryArgs is a struct which contains the arguments used to invoke a query -// these may either be passed by name, in a map, or as a list of positional args -// NOTE: if both are present the named parameters are used -type QueryArgs struct { - ArgMap map[string]string `cty:"args" json:"args,omitempty"` - // args list may be sparsely populated (in case of runtime dependencies) - // so use *string - ArgList []*string `cty:"args_list" json:"args_list"` - // TACTICAL: map of positional and named args which are strings and therefor do NOT need JSON serialising - // (can be removed when we move to cty) - stringNamedArgs map[string]struct{} - stringPositionalArgs map[int]struct{} -} - -func (q *QueryArgs) String() string { - if q == nil { - return "" - } - if len(q.ArgList) > 0 { - argsStringList := q.ArgsStringList() - return fmt.Sprintf("Args list: %s", strings.Join(argsStringList, ",")) - } - if len(q.ArgMap) > 0 { - var strs = make([]string, len(q.ArgMap)) - idx := 0 - for k, v := range q.ArgMap { - strs[idx] = fmt.Sprintf("%s = %s", k, v) - idx++ - } - return fmt.Sprintf("args:\n\t%s", strings.Join(strs, "\n\t")) - } - return "" -} - -// ArgsStringList convert ArgLists into list of strings -func (q *QueryArgs) ArgsStringList() []string { - var argsStringList = make([]string, len(q.ArgList)) - for i, a := range q.ArgList { - argsStringList[i] = typehelpers.SafeString(a) - } - return argsStringList -} - -// ConvertArgsList convert argList into list of interface{} by unmarshalling -func (q *QueryArgs) ConvertArgsList() ([]any, error) { - var argList = make([]any, len(q.ArgList)) - - for i, a := range q.ArgList { - if a != nil { - // do we need to unmarshal? - if _, stringArg := q.stringPositionalArgs[i]; stringArg { - argList[i] = *a - } else { - // so this arg is stored as json - we need to deserialize - err := json.Unmarshal([]byte(*a), &argList[i]) - if err != nil { - return nil, err - } - } - } - } - return argList, nil -} - -func NewQueryArgs() *QueryArgs { - return &QueryArgs{ - ArgMap: make(map[string]string), - stringNamedArgs: make(map[string]struct{}), - stringPositionalArgs: make(map[int]struct{}), - } -} - -func (q *QueryArgs) Equals(other *QueryArgs) bool { - if other == nil { - return false - } - if q.Empty() { - return other.Empty() - } - if len(other.ArgMap) != len(q.ArgMap) || len(other.ArgList) != len(q.ArgList) { - return false - } - for k, v := range q.ArgMap { - if !utils.SafeStringsEqual(other.ArgMap[k], v) { - return false - } - } - for i, v := range q.ArgList { - if !utils.SafeStringsEqual(other.ArgList[i], v) { - return false - } - } - return true -} - -func (q *QueryArgs) Empty() bool { - return len(q.ArgMap)+len(q.ArgList) == 0 -} - -func (q *QueryArgs) Validate() error { - if len(q.ArgMap) > 0 && len(q.ArgList) > 0 { - return fmt.Errorf("args contain both positional and named parameters") - } - return nil -} - -// -//// Merge merges the other args with ourselves, creating and returning a new QueryArgs with the result -//// NOTE: other has precedence -//func (q *QueryArgs) Merge(other *QueryArgs, source QueryProvider) (*QueryArgs, error) { -// if other == nil { -// return q, nil -// } -// -// // ensure we valid before trying to merge (i.e. cannot define both arg list and arg map) -// if err := q.Validate(); err != nil { -// return nil, fmt.Errorf("argument validation failed for '%s': %s", source.Name(), err.Error()) -// } -// -// // ensure the other args are valid -// if err := other.Validate(); err != nil { -// return nil, fmt.Errorf("runtime argument validation failed for '%s': %s", source.Name(), err.Error()) -// } -// -// // create a new query args to store the merged result -// result := NewQueryArgs() -// result.stringNamedArgs = other.stringNamedArgs -// result.stringPositionalArgs = other.stringPositionalArgs -// -// // named args -// // first set values from other -// for k, v := range other.ArgMap { -// result.ArgMap[k] = v -// -// } -// // now set any unset values from our map -// for k, v := range q.ArgMap { -// if _, ok := result.ArgMap[k]; !ok { -// result.ArgMap[k] = v -// if _, ok := q.stringNamedArgs[k]; ok { -// result.stringNamedArgs[k] = struct{}{} -// } -// } -// } -// -// // positional args -// // so we must have an args list - figure out how long -// listLength := len(q.ArgList) -// if otherLen := len(other.ArgList); otherLen > listLength { -// listLength = otherLen -// } -// if listLength > 0 { -// result.ArgList = make([]*string, listLength) -// -// // first set values from other -// copy(result.ArgList, other.ArgList) -// -// // now set any unset values from base list -// for i, a := range q.ArgList { -// if result.ArgList[i] == nil { -// result.ArgList[i] = a -// if _, ok := q.stringPositionalArgs[i]; ok { -// result.stringPositionalArgs[i] = struct{}{} -// } -// } -// } -// } -// -// // validate the merged result -// // runtime args must specify args in same way as base args (i.e. both must define either map or list) -// if err := result.Validate(); err != nil { -// return nil, fmt.Errorf("runtime argument validation failed when merging runtime args into '%s': %s", source.Name(), err.Error()) -// } -// -// return result, nil -//} - -func (q *QueryArgs) SetNamedArgVal(value any, name string) (err error) { - strVal, ok := value.(string) - if ok { - q.stringNamedArgs[name] = struct{}{} - } else { - strVal, err = q.ToString(value) - if err != nil { - return err - } - } - q.ArgMap[name] = strVal - return nil -} - -func (q *QueryArgs) SetPositionalArgVal(value any, idx int) (err error) { - if idx >= len(q.ArgList) { - return fmt.Errorf("positional arg index %d out of range", idx) - } - strVal, ok := value.(string) - if ok { - // no need to convert toi string - make a note - q.stringPositionalArgs[idx] = struct{}{} - } else { - strVal, err = q.ToString(value) - if err != nil { - return err - } - } - q.ArgList[idx] = &strVal - return nil -} - -func (q *QueryArgs) ToString(value any) (string, error) { - // format the arg value as a JSON string - jsonBytes, err := json.Marshal(value) - if err != nil { - return "", err - } - return string(jsonBytes), nil -} - -func (q *QueryArgs) SetArgMap(argMap map[string]any) error { - for k, v := range argMap { - if err := q.SetNamedArgVal(v, k); err != nil { - return err - } - } - return nil -} - -func (q *QueryArgs) SetArgList(argList []any) error { - q.ArgList = make([]*string, len(argList)) - for i, v := range argList { - if err := q.SetPositionalArgVal(v, i); err != nil { - return err - } - } - return nil -} - -func (q *QueryArgs) GetNamedArg(name string) (interface{}, bool, error) { - argStr, ok := q.ArgMap[name] - if !ok { - return nil, false, nil - } - // do we need to deserialise? - if _, isStringArg := q.stringNamedArgs[name]; isStringArg { - return argStr, true, nil - } - - var res any - if err := json.Unmarshal([]byte(argStr), &res); err != nil { - return nil, false, err - } - return res, true, nil -} - -func (q *QueryArgs) GetPositionalArg(idx int) (interface{}, bool, error) { - if idx > len(q.ArgList) { - return nil, false, fmt.Errorf("positional arg index %d out of range", idx) - } - argStrPtr := q.ArgList[idx] - if argStrPtr == nil { - return nil, false, nil - } - - // do we need to deserialise? - if _, isStringArg := q.stringPositionalArgs[idx]; isStringArg { - return *argStrPtr, true, nil - } - - var res any - if err := json.Unmarshal([]byte(*argStrPtr), &res); err != nil { - return nil, false, err - } - return res, true, nil -} - -//func (q *QueryArgs) resolveNamedParameters(queryProvider QueryProvider) (argVals []any, missingParams []string, err error) { -// // if query params contains both positional and named params, error out -// params := queryProvider.GetParams() -// -// argVals = make([]any, len(params)) -// -// // iterate through each param def and resolve the value -// // build a map of which args have been matched (used to validate all args have param defs) -// argsWithParamDef := make(map[string]bool) -// for i, param := range params { -// // first set default -// defaultValue, err := param.GetDefault() -// if err != nil { -// return nil, nil, err -// } -// -// // can we resolve a value for this param? -// if argVal, ok, err := q.GetNamedArg(param.ShortName); ok { -// if err != nil { -// return nil, nil, err -// } -// argVals[i] = argVal -// argsWithParamDef[param.ShortName] = true -// -// } else if defaultValue != nil { -// // is there a default -// argVals[i] = defaultValue -// } else { -// // no value provided and no default defined - add to missing list -// missingParams = append(missingParams, param.ShortName) -// } -// } -// -// // verify we have param defs for all provided args -// for arg := range q.ArgMap { -// if _, ok := argsWithParamDef[arg]; !ok { -// log.Printf("[TRACE] no parameter definition found for argument '%s'", arg) -// } -// } -// -// return argVals, missingParams, nil -//} - -//func (q *QueryArgs) resolvePositionalParameters(queryProvider QueryProvider) (argValues []any, missingParams []string, err error) { -// // if query params contains both positional and named params, error out -// // if there are param defs - we must be able to resolve all params -// // if there are MORE defs than provided parameters, all remaining defs MUST provide a default -// params := queryProvider.GetParams() -// -// // if no param defs are defined, just use the given values, using runtime dependencies where available -// if len(params) == 0 { -// // no params defined, so we return as many args as are provided -// // (convert arg vals from json) -// argValues, err = q.ConvertArgsList() -// if err != nil { -// return nil, nil, err -// } -// return argValues, nil, nil -// } -// -// // verify we have enough args -// if len(params) < len(q.ArgList) { -// err = fmt.Errorf("resolvePositionalParameters failed for '%s' - %d %s were provided but there %s %d parameter %s", -// queryProvider.Name(), -// len(q.ArgList), -// utils.Pluralize("argument", len(q.ArgList)), -// utils.Pluralize("is", len(params)), -// len(params), -// utils.Pluralize("definition", len(params)), -// ) -// return -// } -// -// // so there are param definitions - use these to populate argValues -// argValues = make([]any, len(params)) -// -// for i, param := range params { -// // first set default -// defaultValue, err := param.GetDefault() -// if err != nil { -// return nil, nil, err -// } -// -// if i < len(q.ArgList) && q.ArgList[i] != nil { -// argVal, _, err := q.GetPositionalArg(i) -// if err != nil { -// return nil, nil, err -// } -// -// argValues[i] = argVal -// } else if defaultValue != nil { -// // so we have run out of provided params - is there a default? -// argValues[i] = defaultValue -// } else { -// // no value provided and no default defined - add to missing list -// missingParams = append(missingParams, param.ShortName) -// } -// } -// return argValues, missingParams, nil -//} diff --git a/pkg/steampipeconfig/modconfig/query_args_helpers.go b/pkg/steampipeconfig/modconfig/query_args_helpers.go deleted file mode 100644 index d7b43a602..000000000 --- a/pkg/steampipeconfig/modconfig/query_args_helpers.go +++ /dev/null @@ -1,85 +0,0 @@ -package modconfig - -// -//// MergeArgs ensures base and runtime args are non nil and merges them into single args -//func MergeArgs(queryProvider QueryProvider, runtimeArgs *QueryArgs) (*QueryArgs, error) { -// -// baseArgs := queryProvider.GetArgs() -// // ensure non nil -// if baseArgs == nil { -// baseArgs = NewQueryArgs() -// } -// if runtimeArgs == nil { -// runtimeArgs = NewQueryArgs() -// } -// -// return baseArgs.Merge(runtimeArgs, queryProvider) -//} -// -//// ResolveArgs resolves the argument values, -//// falling back on defaults from param definitions in the source (if present) -//// it returns the arg values as a csv string which can be used in a query invocation -//// (the arg values and param defaults will already have been converted to postgres format) -//func ResolveArgs(qp QueryProvider, runtimeArgs *QueryArgs) ([]any, error) { -// var argVals []any -// var missingParams []string -// var err error -// // validate args -// if runtimeArgs == nil { -// runtimeArgs = &QueryArgs{} -// } -// -// log.Printf("[TRACE] ResolveArgs: resolving args for %s", qp.Name()) -// -// // merge the query provider args (if any) with the runtime args -// sourceArgs := qp.GetArgs() -// if sourceArgs == nil { -// sourceArgs = &QueryArgs{} -// } -// mergedArgs, err := sourceArgs.Merge(runtimeArgs, qp) -// if err != nil { -// log.Printf("[WARN] ResolveArgs failed to merge args for %s: %s", qp.Name(), err.Error()) -// return nil, err -// } -// -// if namedArgCount := len(mergedArgs.ArgMap); namedArgCount > 0 { -// log.Printf("[TRACE] %s defines %d named %s", qp.Name(), namedArgCount, utils.Pluralize("arg", namedArgCount)) -// // if named args are provided and the query does not define params, we cannot resolve the args -// if len(qp.GetParams()) == 0 { -// log.Printf("[TRACE] %s defines %d named %s but has no parameters definitions", qp.Name(), namedArgCount, utils.Pluralize("arg", namedArgCount)) -// } else { -// // do params contain named params? -// argVals, missingParams, err = mergedArgs.resolveNamedParameters(qp) -// log.Printf("[TRACE] resolved %d named %s for %s", len(argVals), utils.Pluralize("params", len(argVals)), qp.Name()) -// } -// } else { -// // resolve as positional parameters -// // (or fall back to defaults if no positional params are present) -// argVals, missingParams, err = mergedArgs.resolvePositionalParameters(qp) -// log.Printf("[TRACE] resolved %d positional %s for %s", len(argVals), utils.Pluralize("params", len(argVals)), qp.Name()) -// } -// if err != nil { -// log.Printf("[WARN] ResolveArgs failed to resolve args for %s: %s", qp.Name(), err.Error()) -// return nil, err -// } -// -// // did we resolve them all? -// if len(missingParams) > 0 { -// log.Printf("[WARN] ResolveArgs: args missing for %s: %s", qp.Name(), strings.Join(missingParams, ",")) -// // a better error will be constructed by the calling code -// return nil, fmt.Errorf("%s", strings.Join(missingParams, ",")) -// } -// -// // are there any params? -// if len(argVals) == 0 { -// return nil, nil -// } -// -// // convert any array args into a strongly typed array -// for i, v := range argVals { -// argVals[i] = helpers.AnySliceToTypedSlice(v) -// } -// -// // success! -// return argVals, nil -//} diff --git a/pkg/steampipeconfig/modconfig/query_args_test.go b/pkg/steampipeconfig/modconfig/query_args_test.go deleted file mode 100644 index 4678fedc2..000000000 --- a/pkg/steampipeconfig/modconfig/query_args_test.go +++ /dev/null @@ -1,351 +0,0 @@ -package modconfig - -import ( - "github.com/turbot/go-kit/helpers" - "github.com/turbot/pipe-fittings/utils" - "reflect" - "testing" -) - -type resolveParamsTest struct { - baseArgs *QueryArgs - runtimeArgs *QueryArgs - paramDefs []*ParamDef - expected interface{} -} - -// NOTE: all QueryArgs values are Json representations of the arg value -// TODO really we should update the trest to set stringNamedArgs and stringPositionalArgs for each args object -// then we can store the string args as normal strings, not json strings -// TODO add other args types - arrays, json etc. - -var testCasesResolveParams = map[string]resolveParamsTest{ - - "named argsno defs": { - baseArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p1": `"val1"`, - "p2": `"val2"`, - }, - }, - paramDefs: nil, - expected: []any(nil), - }, - "named args with defs": { - baseArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p1": `"val1"`, - "p2": `"val2"`, - }, - }, - paramDefs: []*ParamDef{ - {ShortName: "p1"}, - {ShortName: "p2"}, - }, - expected: []any{"val1", "val2"}, - }, - "named args with defs and partial runtime overrides": { - baseArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p1": `"val1"`, - "p2": `"val2"`, - }, - }, - runtimeArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p2": `"runtime val2"`, - }, - }, - paramDefs: []*ParamDef{ - {ShortName: "p1"}, - {ShortName: "p2"}, - }, - expected: []any{"val1", "runtime val2"}, - }, - - "named args with defs and full runtime overrides": { - baseArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p1": `"val1"`, - "p2": `"val2"`, - }, - }, - runtimeArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p1": `"runtime val1"`, - "p2": `"runtime val2"`, - }, - }, - paramDefs: []*ParamDef{ - {ShortName: "p1"}, - {ShortName: "p2"}, - }, - expected: []any{"runtime val1", "runtime val2"}, - }, - "named args with defs and runtime overrides with additional undefined arg": { - baseArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p1": `"val1"`, - "p2": `"val2"`, - }, - }, - runtimeArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p2": `"runtime val2"`, - "p3": `"runtime val3"`, - }, - }, - paramDefs: []*ParamDef{ - {ShortName: "p1"}, - {ShortName: "p2"}, - }, - expected: []any{"val1", "runtime val2"}, - }, - - "named arg overrides only with defs": { - runtimeArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p1": `"override val1"`, - "p2": `"override val2"`, - }, - }, - paramDefs: []*ParamDef{ - {ShortName: "p1"}, - {ShortName: "p2"}, - }, - expected: []any{"override val1", "override val2"}, - }, - "named param defs with incomplete overrides": { - runtimeArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p2": `"override val2"`, - }, - }, - paramDefs: []*ParamDef{ - {ShortName: "p1"}, - {ShortName: "p2"}, - }, - expected: "ERROR", - }, - "named param defs with incomplete invalid overrides": { - runtimeArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p3": `"override val2"`, - }, - }, - paramDefs: []*ParamDef{ - {ShortName: "p1"}, - {ShortName: "p2"}, - }, - expected: "ERROR", - }, - "named param defs with defaults with incomplete overrides": { - runtimeArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p2": `"override val2"`, - }, - }, - paramDefs: []*ParamDef{ - {ShortName: "p1", Default: utils.ToStringPointer(`"val1"`)}, - {ShortName: "p2", Default: utils.ToStringPointer(`"val2"`)}, - }, - expected: []any{"val1", "override val2"}, - }, - "named param defs with defaults with undefined override": { - runtimeArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p3": `"override val2"`, - }, - }, - paramDefs: []*ParamDef{ - {ShortName: "p1", Default: utils.ToStringPointer(`"val1"`)}, - {ShortName: "p2", Default: utils.ToStringPointer(`"val2"`)}, - }, - expected: []any{"val1", "val2"}, - }, - - "partial named args with defs and defaults": { - baseArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p1": `"val1"`, - }, - }, - paramDefs: []*ParamDef{ - {ShortName: "p1", Default: utils.ToStringPointer(`"def_val1"`)}, - {ShortName: "p2", Default: utils.ToStringPointer(`"def_val2"`)}, - }, - expected: []any{"val1", "def_val2"}, - }, - "partial named args with defs defaults and partial override": { - baseArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p1": `"val1"`, - }, - }, - runtimeArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p2": `"override val2"`, - }, - }, - paramDefs: []*ParamDef{ - {ShortName: "p1", Default: utils.ToStringPointer(`"def_val1"`)}, - {ShortName: "p2", Default: utils.ToStringPointer(`"def_val2"`)}, - {ShortName: "p3", Default: utils.ToStringPointer(`"def_val3"`)}, - }, - - expected: []any{"val1", "override val2", "def_val3"}, - }, - "partial named args with defs and unmatched defaults": { - // only a default for first param, which is populated from the provided positional param - baseArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p1": `"val1"`, - }, - }, - paramDefs: []*ParamDef{ - {ShortName: "p1", Default: utils.ToStringPointer(`"def_val1"`)}, - {ShortName: "p2"}, - }, - expected: "ERROR", - }, - - "positional params no defs": { - baseArgs: &QueryArgs{ - ArgList: []*string{utils.ToStringPointer(`"val1"`), utils.ToStringPointer(`"val2"`)}, - }, - paramDefs: nil, - - expected: []any{"val1", "val2"}, - }, - "positional params with partial runtime override no defs": { - baseArgs: &QueryArgs{ - ArgList: []*string{utils.ToStringPointer(`"val1"`), utils.ToStringPointer(`"val2"`)}, - }, - runtimeArgs: &QueryArgs{ - ArgList: []*string{nil, utils.ToStringPointer(`"override val2"`)}, - }, - paramDefs: nil, - expected: []any{"val1", "override val2"}, - }, - "positional params with full runtime override no defs": { - baseArgs: &QueryArgs{ - ArgList: []*string{utils.ToStringPointer(`"val1"`), utils.ToStringPointer(`"val2"`)}, - }, - runtimeArgs: &QueryArgs{ - ArgList: []*string{utils.ToStringPointer(`"override val1"`), utils.ToStringPointer(`"override val2"`)}, - }, - paramDefs: nil, - expected: []any{"override val1", "override val2"}, - }, - "partial positional args with defs and defaults": { - baseArgs: &QueryArgs{ - ArgList: []*string{utils.ToStringPointer(`"val1"`)}, - }, - paramDefs: []*ParamDef{ - {ShortName: "p1", Default: utils.ToStringPointer(`"def_val1"`)}, - {ShortName: "p2", Default: utils.ToStringPointer(`"def_val2"`)}, - }, - expected: []any{"val1", "def_val2"}, - }, - "partial positional args with defs, overrides and defaults": { - baseArgs: &QueryArgs{ - ArgList: []*string{utils.ToStringPointer(`"val1"`)}, - }, - runtimeArgs: &QueryArgs{ - ArgList: []*string{nil, utils.ToStringPointer(`"override val2"`)}, - }, - paramDefs: []*ParamDef{ - {ShortName: "p1", Default: utils.ToStringPointer(`"def_val1"`)}, - {ShortName: "p2", Default: utils.ToStringPointer(`"def_val2"`)}, - {ShortName: "p3", Default: utils.ToStringPointer(`"def_val3"`)}, - }, - expected: []any{"val1", "override val2", "def_val3"}, - }, - "partial positional args with defs and unmatched defaults": { - // only a default for first param, which is populated from the provided positional param - baseArgs: &QueryArgs{ - ArgList: []*string{utils.ToStringPointer(`"val1"`)}, - }, - paramDefs: []*ParamDef{ - {ShortName: "p1", Default: utils.ToStringPointer(`"def_val1"`)}, - {ShortName: "p2"}, - }, - expected: "ERROR", - }, - - "positional and named args(expect error)": { - baseArgs: &QueryArgs{ - ArgList: []*string{utils.ToStringPointer(`"val1"`), utils.ToStringPointer(`"val2"`)}, - ArgMap: map[string]string{ - "p1": `"val1"`, - "p2": `"val2"`, - }, - }, - paramDefs: nil, - expected: "ERROR", - }, - "positional and override named args (expect error)": { - baseArgs: &QueryArgs{ - ArgList: []*string{utils.ToStringPointer(`"val1"`), utils.ToStringPointer(`"val2"`)}, - }, - runtimeArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p1": `"val1"`, - "p2": `"val2"`, - }, - }, - paramDefs: nil, - expected: "ERROR", - }, - "named and override params (expect error)": { - baseArgs: &QueryArgs{ - ArgMap: map[string]string{ - "p1": `"val1"`, - "p2": `"val2"`, - }, - }, - runtimeArgs: &QueryArgs{ - ArgList: []*string{utils.ToStringPointer(`"val1"`), utils.ToStringPointer(`"val2"`)}, - }, - paramDefs: nil, - expected: "ERROR", - }, -} - -func TestResolveAsString(t *testing.T) { - testsToRun := []string{} - - for name, test := range testCasesResolveParams { - if len(testsToRun) > 0 && !helpers.StringSliceContains(testsToRun, name) { - continue - } - query := &Control{ - QueryProviderImpl: QueryProviderImpl{ - RuntimeDependencyProviderImpl: RuntimeDependencyProviderImpl{ - ModTreeItemImpl: ModTreeItemImpl{ - HclResourceImpl: HclResourceImpl{ - FullName: "control.test_control", - }, - }, - }, - Params: test.paramDefs, - Args: test.baseArgs, - }, - } - res, err := ResolveArgs(query, test.runtimeArgs) - if err != nil { - if test.expected != "ERROR" { - t.Errorf("Test: '%s'' FAILED : \nunexpected error %v", name, err) - } - continue - } - if test.expected == "ERROR" { - t.Errorf("Test: '%s'' FAILED - expected error", name) - continue - } - expected := test.expected.([]any) - if !reflect.DeepEqual(expected, res) { - t.Errorf("Test: '%s'' FAILED : \nexpected:\n %v, \ngot:\n %v\n", name, test.expected, res) - } - } -} diff --git a/pkg/steampipeconfig/modconfig/resolved_query.go b/pkg/steampipeconfig/modconfig/resolved_query.go deleted file mode 100644 index b47e6897d..000000000 --- a/pkg/steampipeconfig/modconfig/resolved_query.go +++ /dev/null @@ -1,31 +0,0 @@ -package modconfig - -import ( - "encoding/json" -) - -// ResolvedQuery contains the execute SQL, raw SQL and args string used to execute a query -type ResolvedQuery struct { - Name string - ExecuteSQL string - RawSQL string - Args []any - IsMetaQuery bool -} - -// QueryArgs converts the ResolvedQuery into QueryArgs -func (r ResolvedQuery) QueryArgs() *QueryArgs { - res := NewQueryArgs() - - res.ArgList = make([]*string, len(r.Args)) - - for i, a := range r.Args { - // TACTICAL convert to JSON representation - jsonBytes, err := json.Marshal(a) - argStr := string(jsonBytes) - if err != nil { - res.ArgList[i] = &argStr - } - } - return res -} diff --git a/pkg/steampipeconfig/modconfig/var_config/doc.go b/pkg/steampipeconfig/modconfig/var_config/doc.go deleted file mode 100644 index 7cd7ff1c3..000000000 --- a/pkg/steampipeconfig/modconfig/var_config/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package variable_config contains code from https://github.com/hashicorp/terraform, with minor modifications - -package var_config diff --git a/pkg/steampipeconfig/modconfig/var_config/named_values.go b/pkg/steampipeconfig/modconfig/var_config/named_values.go deleted file mode 100644 index 4d5a2907b..000000000 --- a/pkg/steampipeconfig/modconfig/var_config/named_values.go +++ /dev/null @@ -1,273 +0,0 @@ -package var_config - -// github.com/hashicorp/terraform/configs/parser_config.go -import ( - "fmt" - "unicode" - - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/gohcl" - "github.com/hashicorp/hcl/v2/hclsyntax" - "github.com/turbot/pipe-fittings/hclhelpers" - "github.com/turbot/steampipe/pkg/steampipeconfig/inputvars/typeexpr" - "github.com/zclconf/go-cty/cty" - "github.com/zclconf/go-cty/cty/convert" -) - -// A consistent detail message for all "not a valid identifier" diagnostics. -const badIdentifierDetail = "A name must start with a letter or underscore and may contain only letters, digits, underscores, and dashes." - -// Variable represents a "variable" block in a module or file. -type Variable struct { - Name string - Description string - Default cty.Value - Type cty.Type - ParsingMode VariableParsingMode - //Validations []*VariableValidation - //Sensitive bool - - DescriptionSet bool - //SensitiveSet bool - - DeclRange hcl.Range -} - -func DecodeVariableBlock(block *hcl.Block, content *hcl.BodyContent, override bool) (*Variable, hcl.Diagnostics) { - v := &Variable{ - Name: block.Labels[0], - DeclRange: hclhelpers.BlockRange(block), - } - var diags hcl.Diagnostics - - // Unless we're building an override, we'll set some defaults - // which we might override with attributes below. We leave these - // as zero-value in the override case so we can recognize whether - // or not they are set when we merge. - if !override { - v.Type = cty.DynamicPseudoType - v.ParsingMode = VariableParseLiteral - } - - if !hclsyntax.ValidIdentifier(v.Name) { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid variable name", - Detail: badIdentifierDetail, - Subject: &block.LabelRanges[0], - }) - } - - if attr, exists := content.Attributes["description"]; exists { - valDiags := gohcl.DecodeExpression(attr.Expr, nil, &v.Description) - diags = append(diags, valDiags...) - v.DescriptionSet = true - } - - if attr, exists := content.Attributes["type"]; exists { - ty, parseMode, tyDiags := decodeVariableType(attr.Expr) - diags = append(diags, tyDiags...) - v.Type = ty - v.ParsingMode = parseMode - } - if attr, exists := content.Attributes["default"]; exists { - val, valDiags := attr.Expr.Value(nil) - diags = append(diags, valDiags...) - - // Convert the default to the expected type so we can catch invalid - // defaults early and allow later code to assume validity. - // Note that this depends on us having already processed any "type" - // attribute above. - // However, we can't do this if we're in an override file where - // the type might not be set; we'll catch that during merge. - if v.Type != cty.NilType { - var err error - val, err = convert.Convert(val, v.Type) - if err != nil { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid default value for variable", - Detail: fmt.Sprintf("This default value is not compatible with the variable's type constraint: %s.", err), - Subject: attr.Expr.Range().Ptr(), - }) - val = cty.DynamicVal - } - } - - v.Default = val - } - - for _, block := range content.Blocks { - switch block.Type { - - default: - // The above cases should be exhaustive for all block types - // defined in variableBlockSchema - panic(fmt.Sprintf("unhandled block type %q", block.Type)) - } - } - - return v, diags -} - -func decodeVariableType(expr hcl.Expression) (cty.Type, VariableParsingMode, hcl.Diagnostics) { - if exprIsNativeQuotedString(expr) { - val, diags := expr.Value(nil) - if diags.HasErrors() { - return cty.DynamicPseudoType, VariableParseHCL, diags - } - str := val.AsString() - switch str { - case "string": - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid quoted type constraints", - Subject: expr.Range().Ptr(), - }) - return cty.DynamicPseudoType, VariableParseLiteral, diags - case "list": - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid quoted type constraints", - Subject: expr.Range().Ptr(), - }) - return cty.DynamicPseudoType, VariableParseHCL, diags - case "map": - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid quoted type constraints", - Subject: expr.Range().Ptr(), - }) - return cty.DynamicPseudoType, VariableParseHCL, diags - default: - return cty.DynamicPseudoType, VariableParseHCL, hcl.Diagnostics{{ - Severity: hcl.DiagError, - Summary: "Invalid legacy variable type hint", - Subject: expr.Range().Ptr(), - }} - } - } - - // First we'll deal with some shorthand forms that the HCL-level type - // expression parser doesn't include. These both emulate pre-0.12 behavior - // of allowing a list or map of any element type as long as all of the - // elements are consistent. This is the same as list(any) or map(any). - switch hcl.ExprAsKeyword(expr) { - case "list": - return cty.List(cty.DynamicPseudoType), VariableParseHCL, nil - case "map": - return cty.Map(cty.DynamicPseudoType), VariableParseHCL, nil - } - - ty, diags := typeexpr.TypeConstraint(expr) - if diags.HasErrors() { - return cty.DynamicPseudoType, VariableParseHCL, diags - } - - switch { - case ty.IsPrimitiveType(): - // Primitive types use literal parsing. - return ty, VariableParseLiteral, diags - default: - // Everything else uses HCL parsing - return ty, VariableParseHCL, diags - } -} - -// Required returns true if this variable is required to be set by the caller, -// or false if there is a default value that will be used when it isn't set. -func (v *Variable) Required() bool { - return v.Default == cty.NilVal -} - -// VariableParsingMode defines how values of a particular variable given by -// text-only mechanisms (command line arguments and environment variables) -// should be parsed to produce the final value. -type VariableParsingMode rune - -// VariableParseLiteral is a variable parsing mode that just takes the given -// string directly as a cty.String value. -const VariableParseLiteral VariableParsingMode = 'L' - -// VariableParseHCL is a variable parsing mode that attempts to parse the given -// string as an HCL expression and returns the result. -const VariableParseHCL VariableParsingMode = 'H' - -// Parse uses the receiving parsing mode to process the given variable value -// string, returning the result along with any diagnostics. -// -// A VariableParsingMode does not know the expected type of the corresponding -// variable, so it's the caller's responsibility to attempt to convert the -// result to the appropriate type and return to the user any diagnostics that -// conversion may produce. -// -// The given name is used to create a synthetic filename in case any diagnostics -// must be generated about the given string value. This should be the name -// of the configuration variable whose value will be populated from the given -// string. -// -// If the returned diagnostics has errors, the returned value may not be -// valid. -func (m VariableParsingMode) Parse(name, value string) (cty.Value, hcl.Diagnostics) { - switch m { - case VariableParseLiteral: - return cty.StringVal(value), nil - case VariableParseHCL: - fakeFilename := fmt.Sprintf("", name) - expr, diags := hclsyntax.ParseExpression([]byte(value), fakeFilename, hcl.Pos{Line: 1, Column: 1}) - if diags.HasErrors() { - return cty.DynamicVal, diags - } - val, valDiags := expr.Value(nil) - diags = append(diags, valDiags...) - return val, diags - default: - // Should never happen - panic(fmt.Errorf("parse called on invalid VariableParsingMode %#v", m)) - } -} - -// VariableValidation represents a configuration-defined validation rule -// for a particular input variable, given as a "validation" block inside -// a "variable" block. -type VariableValidation struct { - // Condition is an expression that refers to the variable being tested - // and contains no other references. The expression must return true - // to indicate that the value is valid or false to indicate that it is - // invalid. If the expression produces an error, that's considered a bug - // in the module defining the validation rule, not an error in the caller. - Condition hcl.Expression - - // ErrorMessage is one or more full sentences, which would need to be in - // English for consistency with the rest of the error message output but - // can in practice be in any language as long as it ends with a period. - // The message should describe what is required for the condition to return - // true in a way that would make sense to a caller of the module. - ErrorMessage string - - DeclRange hcl.Range -} - -// looksLikeSentence is a simple heuristic that encourages writing error -// messages that will be presentable when included as part of a larger error diagnostic -func looksLikeSentences(s string) bool { - if len(s) < 1 { - return false - } - runes := []rune(s) // HCL guarantees that all strings are valid UTF-8 - first := runes[0] - last := runes[len(runes)-1] - - // If the first rune is a letter then it must be an uppercase letter. - // (This will only see the first rune in a multi-rune combining sequence, - // but the first rune is generally the letter if any are, and if not then - // we'll just ignore it because we're primarily expecting English messages - // right now anyway) - if unicode.IsLetter(first) && !unicode.IsUpper(first) { - return false - } - - // The string must be at least one full sentence, which implies having - // sentence-ending punctuation. - return last == '.' || last == '?' || last == '!' -} diff --git a/pkg/steampipeconfig/modconfig/var_config/named_values_test.go b/pkg/steampipeconfig/modconfig/var_config/named_values_test.go deleted file mode 100644 index fd35c2586..000000000 --- a/pkg/steampipeconfig/modconfig/var_config/named_values_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package var_config - -import ( - "testing" -) - -func Test_looksLikeSentences(t *testing.T) { - tests := map[string]struct { - args string - want bool - }{ - "empty sentence": { - args: "", - want: false, - }, - "valid sentence": { - args: "A valid sentence.", - want: true, - }, - "valid sentence with an accent": { - args: `A Valid sentence with an accent "é".`, - want: true, - }, - } - - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - if got := looksLikeSentences(tt.args); got != tt.want { - t.Errorf("looksLikeSentences() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/steampipeconfig/modconfig/var_config/util.go b/pkg/steampipeconfig/modconfig/var_config/util.go deleted file mode 100644 index b58537db0..000000000 --- a/pkg/steampipeconfig/modconfig/var_config/util.go +++ /dev/null @@ -1,18 +0,0 @@ -package var_config - -import ( - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclsyntax" -) - -// exprIsNativeQuotedString determines whether the given expression looks like -// it's a quoted string in the HCL native syntax. -// -// This should be used sparingly only for situations where our legacy HCL -// decoding would've expected a keyword or reference in quotes but our new -// decoding expects the keyword or reference to be provided directly as -// an identifier-based expression. -func exprIsNativeQuotedString(expr hcl.Expression) bool { - _, ok := expr.(*hclsyntax.TemplateExpr) - return ok -} diff --git a/pkg/steampipeconfig/steampipeconfig.go b/pkg/steampipeconfig/steampipeconfig.go index ec3dbb572..aebf74e95 100644 --- a/pkg/steampipeconfig/steampipeconfig.go +++ b/pkg/steampipeconfig/steampipeconfig.go @@ -2,7 +2,6 @@ package steampipeconfig import ( "fmt" - options2 "github.com/turbot/steampipe/pkg/options" "log" "os" "strings" @@ -13,13 +12,14 @@ import ( "github.com/turbot/pipe-fittings/constants" "github.com/turbot/pipe-fittings/error_helpers" "github.com/turbot/pipe-fittings/filepaths" + "github.com/turbot/pipe-fittings/modconfig" "github.com/turbot/pipe-fittings/ociinstaller" - "github.com/turbot/pipe-fittings/options" + poptions "github.com/turbot/pipe-fittings/options" "github.com/turbot/pipe-fittings/plugin" "github.com/turbot/pipe-fittings/versionfile" "github.com/turbot/pipe-fittings/workspace_profile" "github.com/turbot/steampipe-plugin-sdk/v5/sperr" - "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" + "github.com/turbot/steampipe/pkg/options" ) // SteampipeConfig is a struct to hold Connection map and Steampipe options @@ -30,19 +30,19 @@ type SteampipeConfig struct { // map of plugin configs, keyed by plugin instance PluginsInstances map[string]*plugin.Plugin // map of connection name to partially parsed connection config - Connections map[string]*modconfig.Connection + Connections map[string]*modconfig.SteampipeConnection // Steampipe options - DatabaseOptions *options2.Database - GeneralOptions *options2.General - PluginOptions *options2.Plugin + DatabaseOptions *options.Database + GeneralOptions *options.General + PluginOptions *options.Plugin // map of installed plugin versions, keyed by plugin image ref PluginVersions map[string]*versionfile.InstalledVersion } func NewSteampipeConfig(commandName string) *SteampipeConfig { return &SteampipeConfig{ - Connections: make(map[string]*modconfig.Connection), + Connections: make(map[string]*modconfig.SteampipeConnection), Plugins: make(map[string][]*plugin.Plugin), PluginsInstances: make(map[string]*plugin.Plugin), } @@ -92,23 +92,23 @@ func (c *SteampipeConfig) ConfigMap() map[string]interface{} { return res } -func (c *SteampipeConfig) SetOptions(opts options.Options) (errorsAndWarnings error_helpers.ErrorAndWarnings) { +func (c *SteampipeConfig) SetOptions(opts poptions.Options) (errorsAndWarnings error_helpers.ErrorAndWarnings) { errorsAndWarnings = error_helpers.NewErrorsAndWarning(nil) switch o := opts.(type) { - case *options2.Database: + case *options.Database: if c.DatabaseOptions == nil { c.DatabaseOptions = o } else { c.DatabaseOptions.Merge(o) } - case *options2.General: + case *options.General: if c.GeneralOptions == nil { c.GeneralOptions = o } else { c.GeneralOptions.Merge(o) } - case *options2.Plugin: + case *options.Plugin: if c.PluginOptions == nil { c.PluginOptions = o } else { @@ -161,8 +161,8 @@ PluginOptions: return str } -func (c *SteampipeConfig) ConnectionsForPlugin(pluginLongName string, pluginVersion *version.Version) []*modconfig.Connection { - var res []*modconfig.Connection +func (c *SteampipeConfig) ConnectionsForPlugin(pluginLongName string, pluginVersion *version.Version) []*modconfig.SteampipeConnection { + var res []*modconfig.SteampipeConnection for _, con := range c.Connections { // extract constraint from plugin ref := ociinstaller.NewImageRef(con.Plugin) @@ -193,8 +193,8 @@ func (c *SteampipeConfig) ConnectionNames() []string { return res } -func (c *SteampipeConfig) ConnectionList() []*modconfig.Connection { - res := make([]*modconfig.Connection, len(c.Connections)) +func (c *SteampipeConfig) ConnectionList() []*modconfig.SteampipeConnection { + res := make([]*modconfig.SteampipeConnection, len(c.Connections)) idx := 0 for _, c := range c.Connections { res[idx] = c @@ -287,7 +287,7 @@ func (c *SteampipeConfig) initializePlugins() { NOTE: if there is more than one config for the plugin this is an error 5) create a default config for the plugin (with the label set to the image ref) */ -func (c *SteampipeConfig) resolvePluginInstanceForConnection(connection *modconfig.Connection) (*plugin.Plugin, error) { +func (c *SteampipeConfig) resolvePluginInstanceForConnection(connection *modconfig.SteampipeConnection) (*plugin.Plugin, error) { // NOTE: at this point, c.Plugin is NOT populated, only either c.PluginAlias or c.PluginInstance // we populate c.Plugin AFTER resolving the plugin