From 15984137ae31c9d0a7d63fe4e960ee3041eaec79 Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Wed, 11 Jan 2023 16:34:58 +0000 Subject: [PATCH] Add deprecation warnings for deprecated hcl properties. Closes #2973 --- cmd/dashboard.go | 10 +- cmd/list.go | 4 +- pkg/control/init_data.go | 9 +- pkg/query/init_data.go | 8 +- pkg/steampipeconfig/load_mod.go | 46 ++--- pkg/steampipeconfig/load_mod_variables.go | 6 +- .../modconfig/error_and_warnings.go | 31 +++ pkg/steampipeconfig/parse/decode.go | 39 ++-- pkg/steampipeconfig/parse/decode_body.go | 2 +- pkg/steampipeconfig/parse/decode_result.go | 2 +- pkg/steampipeconfig/parse/parser.go | 29 +-- pkg/steampipeconfig/parse/validate.go | 4 +- .../refresh_connections_result.go | 15 +- pkg/workspace/load_workspace.go | 16 +- pkg/workspace/workspace.go | 29 ++- pkg/workspace/workspace_events.go | 20 +- .../node_edge_vals/dashboard.sp | 190 ------------------ tests/manual_testing/node_edge_vals/mod.sp | 3 - .../node_reuse/inputs/dashboard.sp | 4 +- 19 files changed, 157 insertions(+), 310 deletions(-) create mode 100644 pkg/steampipeconfig/modconfig/error_and_warnings.go delete mode 100644 tests/manual_testing/node_edge_vals/dashboard.sp delete mode 100644 tests/manual_testing/node_edge_vals/mod.sp diff --git a/cmd/dashboard.go b/cmd/dashboard.go index 025493e5f..224be23ee 100644 --- a/cmd/dashboard.go +++ b/cmd/dashboard.go @@ -239,13 +239,13 @@ func initDashboard(ctx context.Context) *initialisation.InitData { } func getInitData(ctx context.Context) *initialisation.InitData { - w, err := workspace.LoadWorkspacePromptingForVariables(ctx) - if err != nil { - return initialisation.NewErrorInitData(fmt.Errorf("failed to load workspace: %s", err.Error())) + w, errAndWarnings := workspace.LoadWorkspacePromptingForVariables(ctx) + if errAndWarnings.GetError() != nil { + return initialisation.NewErrorInitData(fmt.Errorf("failed to load workspace: %s", errAndWarnings.GetError().Error())) } - i := initialisation.NewInitData(w). - Init(ctx, constants.InvokerDashboard) + i := initialisation.NewInitData(w).Init(ctx, constants.InvokerDashboard) + i.Result.AddWarnings(errAndWarnings.Warnings...) if len(viper.GetStringSlice(constants.ArgExport)) > 0 { i.RegisterExporters(dashboardExporters()...) diff --git a/cmd/list.go b/cmd/list.go index a41aabbe0..21b50f592 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -45,8 +45,8 @@ func getRunListSubCmd(opts listSubCmdOptions) func(cmd *cobra.Command, args []st return func(cmd *cobra.Command, args []string) { workspacePath := viper.GetString(constants.ArgModLocation) - w, err := workspace.Load(cmd.Context(), workspacePath) - error_helpers.FailOnError(err) + w, errAndWarnings := workspace.Load(cmd.Context(), workspacePath) + error_helpers.FailOnError(errAndWarnings.GetError()) modResources, depResources, err := listResourcesInMod(cmd.Context(), w.Mod, cmd) error_helpers.FailOnErrorWithMessage(err, "could not list resources") diff --git a/pkg/control/init_data.go b/pkg/control/init_data.go index d22fbf09a..b992a3a13 100644 --- a/pkg/control/init_data.go +++ b/pkg/control/init_data.go @@ -30,10 +30,10 @@ func NewInitData(ctx context.Context) *InitData { defer statushooks.Done(ctx) // load the workspace - w, err := workspace.LoadWorkspacePromptingForVariables(ctx) - if err != nil { + w, errAndWarnings := workspace.LoadWorkspacePromptingForVariables(ctx) + if errAndWarnings.GetError() != nil { return &InitData{ - InitData: *initialisation.NewErrorInitData(fmt.Errorf("failed to load workspace: %s", err.Error())), + InitData: *initialisation.NewErrorInitData(fmt.Errorf("failed to load workspace: %s", errAndWarnings.GetError().Error())), } } @@ -41,6 +41,7 @@ func NewInitData(ctx context.Context) *InitData { i := &InitData{ InitData: *initialisation.NewInitData(w), } + i.Result.AddWarnings(errAndWarnings.Warnings...) if !w.ModfileExists() { i.Result.Error = workspace.ErrorNoModDefinition @@ -51,7 +52,7 @@ func NewInitData(ctx context.Context) *InitData { viper.Set(constants.ArgProgress, false) } // set color schema - err = initialiseCheckColorScheme() + err := initialiseCheckColorScheme() if err != nil { i.Result.Error = err return i diff --git a/pkg/query/init_data.go b/pkg/query/init_data.go index 501a090ab..20a0cc01e 100644 --- a/pkg/query/init_data.go +++ b/pkg/query/init_data.go @@ -24,10 +24,10 @@ type InitData struct { // InitData.Done closes after asynchronous initialization completes func NewInitData(ctx context.Context, args []string) *InitData { // load the workspace - w, err := workspace.LoadWorkspacePromptingForVariables(ctx) - if err != nil { + w, errAndWarnings := workspace.LoadWorkspacePromptingForVariables(ctx) + if errAndWarnings.GetError() != nil { return &InitData{ - InitData: *initialisation.NewErrorInitData(fmt.Errorf("failed to load workspace: %s", err.Error())), + InitData: *initialisation.NewErrorInitData(fmt.Errorf("failed to load workspace: %s", errAndWarnings.GetError().Error())), } } @@ -35,6 +35,8 @@ func NewInitData(ctx context.Context, args []string) *InitData { InitData: *initialisation.NewInitData(w), Loaded: make(chan struct{}), } + // add any warnings + i.Result.AddWarnings(errAndWarnings.Warnings...) if len(viper.GetStringSlice(constants.ArgExport)) > 0 { i.RegisterExporters(queryExporters()...) diff --git a/pkg/steampipeconfig/load_mod.go b/pkg/steampipeconfig/load_mod.go index 874ee4ac7..f6394d1cd 100644 --- a/pkg/steampipeconfig/load_mod.go +++ b/pkg/steampipeconfig/load_mod.go @@ -21,27 +21,29 @@ import ( // if CreatePseudoResources flag is set, construct hcl resources for files with specific extensions // NOTE: it is an error if there is more than 1 mod defined, however zero mods is acceptable // - a default mod will be created assuming there are any resource files -func LoadMod(modPath string, parseCtx *parse.ModParseContext) (mod *modconfig.Mod, err error) { +func LoadMod(modPath string, parseCtx *parse.ModParseContext) (mod *modconfig.Mod, errAndWarnings *modconfig.ErrorAndWarnings) { + var err error defer func() { if r := recover(); r != nil { err = helpers.ToError(r) + errAndWarnings = modconfig.NewErrorsAndWarning(err) } }() mod, err = loadModDefinition(modPath, parseCtx) - if err != nil { - return nil, err + if errAndWarnings != nil { + return nil, errAndWarnings } // load the mod dependencies if err := loadModDependencies(mod, parseCtx); err != nil { - return nil, err + return nil, modconfig.NewErrorsAndWarning(err) } // now we have loaded dependencies, set the current mod on the run context parseCtx.CurrentMod = mod // populate the resource maps of the current mod using the dependency mods mod.ResourceMaps = parseCtx.GetResourceMaps() // now load the mod resource hcl - return loadModResources(modPath, parseCtx, mod) + return loadModResources(modPath, parseCtx) } func loadModDefinition(modPath string, parseCtx *parse.ModParseContext) (*modconfig.Mod, error) { @@ -148,9 +150,9 @@ func loadModDependency(modDependency *modconfig.ModVersionConstraint, parseCtx * childRunCtx.BlockTypes = parseCtx.BlockTypes childRunCtx.ParentParseCtx = parseCtx - mod, err := LoadMod(dependencyPath, childRunCtx) - if err != nil { - return err + mod, errAndWarnings := LoadMod(dependencyPath, childRunCtx) + if errAndWarnings.GetError() != nil { + return errAndWarnings.GetError() } // set the version and dependency path of the mod @@ -163,11 +165,11 @@ func loadModDependency(modDependency *modconfig.ModVersionConstraint, parseCtx * parseCtx.ParentParseCtx.LoadedDependencyMods[modDependency.Name] = mod } - return err + return nil } -func loadModResources(modPath string, parseCtx *parse.ModParseContext, mod *modconfig.Mod) (*modconfig.Mod, error) { +func loadModResources(modPath string, parseCtx *parse.ModParseContext) (*modconfig.Mod, *modconfig.ErrorAndWarnings) { // if flag is set, create pseudo resources by mapping files var pseudoResources []modconfig.MappableResource var err error @@ -175,7 +177,7 @@ func loadModResources(modPath string, parseCtx *parse.ModParseContext, mod *modc // now execute any pseudo-resource creations based on file mappings pseudoResources, err = createPseudoResources(modPath, parseCtx) if err != nil { - return nil, err + return nil, modconfig.NewErrorsAndWarning(err) } } @@ -183,28 +185,26 @@ func loadModResources(modPath string, parseCtx *parse.ModParseContext, mod *modc sourcePaths, err := getSourcePaths(modPath, parseCtx.ListOptions) if err != nil { log.Printf("[WARN] LoadMod: failed to get mod file paths: %v\n", err) - return nil, err + return nil, modconfig.NewErrorsAndWarning(err) } // load the raw file data fileData, diags := parse.LoadFileData(sourcePaths...) if diags.HasErrors() { - return nil, plugin.DiagsToError("Failed to load all mod files", diags) + return nil, modconfig.NewErrorsAndWarning(plugin.DiagsToError("Failed to load all mod files", diags)) } // parse all hcl files. - mod, err = parse.ParseMod(modPath, fileData, pseudoResources, parseCtx) - if err != nil { - return nil, err + mod, errAndWarnings := parse.ParseMod(fileData, pseudoResources, parseCtx) + if errAndWarnings.GetError() == nil { + // now add fully populated mod to the parent run context + if parseCtx.ParentParseCtx != nil { + parseCtx.ParentParseCtx.CurrentMod = mod + parseCtx.ParentParseCtx.AddMod(mod) + } } - // now add fully populated mod to the parent run context - if parseCtx.ParentParseCtx != nil { - parseCtx.ParentParseCtx.CurrentMod = mod - parseCtx.ParentParseCtx.AddMod(mod) - } - - return mod, err + return mod, errAndWarnings } // search the parent folder for a mod installatio which satisfied the given mod dependency diff --git a/pkg/steampipeconfig/load_mod_variables.go b/pkg/steampipeconfig/load_mod_variables.go index ceec4d040..89dde2cd4 100644 --- a/pkg/steampipeconfig/load_mod_variables.go +++ b/pkg/steampipeconfig/load_mod_variables.go @@ -18,9 +18,9 @@ import ( func LoadVariableDefinitions(variablePath string, parseCtx *parse.ModParseContext) (*modconfig.ModVariableMap, error) { // only load mod and variables blocks parseCtx.BlockTypes = []string{modconfig.BlockTypeVariable} - mod, err := LoadMod(variablePath, parseCtx) - if err != nil { - return nil, err + mod, errAndWarnings := LoadMod(variablePath, parseCtx) + if errAndWarnings.GetError() != nil { + return nil, errAndWarnings.GetError() } variableMap := modconfig.NewModVariableMap(mod, parseCtx.LoadedDependencyMods) diff --git a/pkg/steampipeconfig/modconfig/error_and_warnings.go b/pkg/steampipeconfig/modconfig/error_and_warnings.go new file mode 100644 index 000000000..95a726e10 --- /dev/null +++ b/pkg/steampipeconfig/modconfig/error_and_warnings.go @@ -0,0 +1,31 @@ +package modconfig + +import "github.com/turbot/steampipe/pkg/error_helpers" + +type ErrorAndWarnings struct { + Error error + Warnings []string +} + +func NewErrorsAndWarning(err error, warnings ...string) *ErrorAndWarnings { + return &ErrorAndWarnings{ + Error: err, Warnings: warnings, + } +} + +func (r *ErrorAndWarnings) AddWarning(warnings ...string) { + r.Warnings = append(r.Warnings, warnings...) +} + +func (r *ErrorAndWarnings) ShowWarnings() { + for _, w := range r.Warnings { + error_helpers.ShowWarning(w) + } +} + +func (r *ErrorAndWarnings) GetError() error { + if r == nil { + return nil + } + return r.Error +} diff --git a/pkg/steampipeconfig/parse/decode.go b/pkg/steampipeconfig/parse/decode.go index 9af3afb08..ba51c5c97 100644 --- a/pkg/steampipeconfig/parse/decode.go +++ b/pkg/steampipeconfig/parse/decode.go @@ -50,11 +50,8 @@ func decode(parseCtx *ModParseContext) hcl.Diagnostics { } } else { resource, res := decodeBlock(block, parseCtx) - if !res.Success() { - diags = append(diags, res.Diags...) - continue - } - if resource == nil { + diags = append(diags, res.Diags...) + if !res.Success() || resource == nil { continue } @@ -725,7 +722,7 @@ func validateName(block *hcl.Block) hcl.Diagnostics { // We use partial decoding so that we can automatically decode as many properties as possible // and only manually decode properties requiring special logic. // The problem is the partial decode does not return errors for invalid attributes/blocks, so we must implement our own -func validateHcl(body *hclsyntax.Body, schema *hcl.BodySchema) hcl.Diagnostics { +func validateHcl(blockType string, body *hclsyntax.Body, schema *hcl.BodySchema) hcl.Diagnostics { var diags hcl.Diagnostics // identify any blocks specified by hcl tags @@ -743,21 +740,39 @@ func validateHcl(body *hclsyntax.Body, schema *hcl.BodySchema) hcl.Diagnostics { if _, ok := supportedBlocks[block.Type]; !ok { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, - Summary: fmt.Sprintf(`Unsupported block type: Blocks of type "%s" are not expected here.`, block.Type), + Summary: fmt.Sprintf(`Unsupported block type: Blocks of type '%s' are not expected here.`, block.Type), Subject: &block.TypeRange, }) } } for _, attribute := range body.Attributes { if _, ok := supportedAttributes[attribute.Name]; !ok { + // special case code for deprecated properties subject := attribute.Range() - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: fmt.Sprintf(`Unsupported attribute: Attribute "%s" not expected here.`, attribute.Name), - Subject: &subject, - }) + if isDeprecated(attribute, blockType) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: fmt.Sprintf(`Deprecated attribute: '%s' is deprecated for '%s' blocks and will be ignored.`, attribute.Name, blockType), + Subject: &subject, + }) + } else { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf(`Unsupported attribute: '%s' not expected here.`, attribute.Name), + Subject: &subject, + }) + } } } return diags } + +func isDeprecated(attribute *hclsyntax.Attribute, blockType string) bool { + switch attribute.Name { + case "search_path", "search_path_prefix": + return blockType == modconfig.BlockTypeQuery || blockType == modconfig.BlockTypeControl + default: + return false + } +} diff --git a/pkg/steampipeconfig/parse/decode_body.go b/pkg/steampipeconfig/parse/decode_body.go index cb1e99d43..ec1dda239 100644 --- a/pkg/steampipeconfig/parse/decode_body.go +++ b/pkg/steampipeconfig/parse/decode_body.go @@ -26,7 +26,7 @@ func decodeHclBody(body hcl.Body, evalCtx *hcl.EvalContext, resourceProvider mod // get the schema for this resource schema := getResourceSchema(resource, nestedStructs) // handle invalid block types - moreDiags = validateHcl(body.(*hclsyntax.Body), schema) + moreDiags = validateHcl(resource.BlockType(), body.(*hclsyntax.Body), schema) diags = append(diags, moreDiags...) moreDiags = decodeHclBodyIntoStruct(body, evalCtx, resourceProvider, resource) diff --git a/pkg/steampipeconfig/parse/decode_result.go b/pkg/steampipeconfig/parse/decode_result.go index c01c0fc53..31e0b8355 100644 --- a/pkg/steampipeconfig/parse/decode_result.go +++ b/pkg/steampipeconfig/parse/decode_result.go @@ -40,7 +40,7 @@ func (p *decodeResult) handleDecodeDiags(diags hcl.Diagnostics) { } } // only register errors if there are NOT any missing variables - if diags.HasErrors() && len(p.Depends) == 0 { + if len(p.Depends) == 0 { p.addDiags(diags) } } diff --git a/pkg/steampipeconfig/parse/parser.go b/pkg/steampipeconfig/parse/parser.go index 37f072b31..6bfd94f73 100644 --- a/pkg/steampipeconfig/parse/parser.go +++ b/pkg/steampipeconfig/parse/parser.go @@ -155,32 +155,32 @@ func ParseModDefinition(modPath string) (*modconfig.Mod, error) { // ParseMod parses all source hcl files for the mod path and associated resources, and returns the mod object // NOTE: the mod definition has already been parsed (or a default created) and is in opts.RunCtx.RootMod -func ParseMod(modPath string, fileData map[string][]byte, pseudoResources []modconfig.MappableResource, parseCtx *ModParseContext) (*modconfig.Mod, error) { +func ParseMod(fileData map[string][]byte, pseudoResources []modconfig.MappableResource, parseCtx *ModParseContext) (*modconfig.Mod, *modconfig.ErrorAndWarnings) { body, diags := ParseHclFiles(fileData) if diags.HasErrors() { - return nil, plugin.DiagsToError("Failed to load all mod source files", diags) + return nil, modconfig.NewErrorsAndWarning(plugin.DiagsToError("Failed to load all mod source files", diags)) } content, moreDiags := body.Content(WorkspaceBlockSchema) if moreDiags.HasErrors() { diags = append(diags, moreDiags...) - return nil, plugin.DiagsToError("Failed to load mod", diags) + return nil, modconfig.NewErrorsAndWarning(plugin.DiagsToError("Failed to load mod", diags)) } mod := parseCtx.CurrentMod if mod == nil { - return nil, fmt.Errorf("ParseMod called with no Current Mod set in ModParseContext") + return nil, modconfig.NewErrorsAndWarning(fmt.Errorf("ParseMod called with no Current Mod set in ModParseContext")) } // get names of all resources defined in hcl which may also be created as pseudo resources hclResources, err := loadMappableResourceNames(content) if err != nil { - return nil, err + return nil, modconfig.NewErrorsAndWarning(err) } // if variables were passed in runcontext, add to the mod for _, v := range parseCtx.Variables { if diags = mod.AddResource(v); diags.HasErrors() { - return nil, plugin.DiagsToError("Failed to add resource to mod", diags) + return nil, modconfig.NewErrorsAndWarning(plugin.DiagsToError("Failed to add resource to mod", diags)) } } @@ -193,17 +193,22 @@ func ParseMod(modPath string, fileData map[string][]byte, pseudoResources []modc // add the mod to the run context // - this it to ensure all pseudo resources get added and build the eval context with the variables we just added if diags = parseCtx.AddMod(mod); diags.HasErrors() { - return nil, plugin.DiagsToError("Failed to add mod to run context", diags) + return nil, modconfig.NewErrorsAndWarning(plugin.DiagsToError("Failed to add mod to run context", diags)) } + // collect warnings as we parse + var res = &modconfig.ErrorAndWarnings{} + // we may need to decode more than once as we gather dependencies as we go // continue decoding as long as the number of unresolved blocks decreases prevUnresolvedBlocks := 0 for attempts := 0; ; attempts++ { diags = decode(parseCtx) if diags.HasErrors() { - return nil, plugin.DiagsToError("Failed to decode all mod hcl files", diags) + return nil, modconfig.NewErrorsAndWarning(plugin.DiagsToError("Failed to decode all mod hcl files", diags)) } + // now retrieve the warning strings + res.AddWarning(plugin.DiagsToWarnings(diags)...) // if there are no unresolved blocks, we are done unresolvedBlocks := len(parseCtx.UnresolvedBlocks) @@ -214,18 +219,16 @@ func ParseMod(modPath string, fileData map[string][]byte, pseudoResources []modc // if the number of unresolved blocks has NOT reduced, fail if prevUnresolvedBlocks != 0 && unresolvedBlocks >= prevUnresolvedBlocks { str := parseCtx.FormatDependencies() - return nil, fmt.Errorf("failed to resolve mod dependencies after %d attempts\nDependencies:\n%s", attempts+1, str) + return nil, modconfig.NewErrorsAndWarning(fmt.Errorf("failed to resolve mod dependencies after %d attempts\nDependencies:\n%s", attempts+1, str)) } // update prevUnresolvedBlocks prevUnresolvedBlocks = unresolvedBlocks } // now tell mod to build tree of controls. - if err := mod.BuildResourceTree(parseCtx.LoadedDependencyMods); err != nil { - return nil, err - } + res.Error = mod.BuildResourceTree(parseCtx.LoadedDependencyMods) - return mod, nil + return mod, res } // parse a yaml file into a hcl.File object diff --git a/pkg/steampipeconfig/parse/validate.go b/pkg/steampipeconfig/parse/validate.go index f20d033b0..f4bf1da8c 100644 --- a/pkg/steampipeconfig/parse/validate.go +++ b/pkg/steampipeconfig/parse/validate.go @@ -99,8 +99,8 @@ func validateParamAndQueryNotBothSet(resource modconfig.QueryProvider) hcl.Diagn } if !resource.IsTopLevel() && !resource.ParamsInheritedFromBase() { diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Only top level resources can have 'param' blocks", + Severity: hcl.DiagWarning, + Summary: "Deprecated usage: Only top level resources can have 'param' blocks", Detail: fmt.Sprintf("%s contains 'param' blocks but is not a top level resource.", resource.Name()), Subject: resource.GetDeclRange(), }) diff --git a/pkg/steampipeconfig/refresh_connections_result.go b/pkg/steampipeconfig/refresh_connections_result.go index 90534f405..2d0ce60e2 100644 --- a/pkg/steampipeconfig/refresh_connections_result.go +++ b/pkg/steampipeconfig/refresh_connections_result.go @@ -2,30 +2,19 @@ package steampipeconfig import ( "fmt" + "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" "strings" - "github.com/turbot/steampipe/pkg/error_helpers" "github.com/turbot/steampipe/pkg/utils" ) // RefreshConnectionResult is a structure used to contain the result of either a RefreshConnections or a NewLocalClient operation type RefreshConnectionResult struct { + modconfig.ErrorAndWarnings UpdatedConnections bool - Warnings []string - Error error Updates *ConnectionUpdates } -func (r *RefreshConnectionResult) AddWarning(warning string) { - r.Warnings = append(r.Warnings, warning) -} - -func (r *RefreshConnectionResult) ShowWarnings() { - for _, w := range r.Warnings { - error_helpers.ShowWarning(w) - } -} - func (r *RefreshConnectionResult) Merge(other *RefreshConnectionResult) { if other == nil { return diff --git a/pkg/workspace/load_workspace.go b/pkg/workspace/load_workspace.go index e4d64ae79..b1b35c8db 100644 --- a/pkg/workspace/load_workspace.go +++ b/pkg/workspace/load_workspace.go @@ -14,31 +14,31 @@ import ( "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" ) -func LoadWorkspacePromptingForVariables(ctx context.Context) (*Workspace, error) { +func LoadWorkspacePromptingForVariables(ctx context.Context) (*Workspace, *modconfig.ErrorAndWarnings) { workspacePath := viper.GetString(constants.ArgModLocation) t := time.Now() defer func() { log.Printf("[TRANCE] Workspace load took %dms\n", time.Since(t).Milliseconds()) }() - w, err := Load(ctx, workspacePath) - if err == nil { - return w, nil + w, errAndWarnings := Load(ctx, workspacePath) + if errAndWarnings.GetError() == nil { + return w, errAndWarnings } - missingVariablesError, ok := err.(modconfig.MissingVariableError) + missingVariablesError, ok := errAndWarnings.GetError().(modconfig.MissingVariableError) // if there was an error which is NOT a MissingVariableError, return it if !ok { - return nil, err + return nil, errAndWarnings } // if interactive input is disabled, return the missing variables error if !viper.GetBool(constants.ArgInput) { - return nil, missingVariablesError + return nil, modconfig.NewErrorsAndWarning(missingVariablesError) } // so we have missing variables - prompt for them // first hide spinner if it is there statushooks.Done(ctx) if err := promptForMissingVariables(ctx, missingVariablesError.MissingVariables, workspacePath); err != nil { log.Printf("[TRACE] Interactive variables prompting returned error %v", err) - return nil, err + return nil, modconfig.NewErrorsAndWarning(err) } // ok we should have all variables now - reload workspace return Load(ctx, workspacePath) diff --git a/pkg/workspace/workspace.go b/pkg/workspace/workspace.go index 4f2ba4675..a6dbbf316 100644 --- a/pkg/workspace/workspace.go +++ b/pkg/workspace/workspace.go @@ -60,23 +60,18 @@ type Workspace struct { } // Load creates a Workspace and loads the workspace mod -func Load(ctx context.Context, workspacePath string) (*Workspace, error) { +func Load(ctx context.Context, workspacePath string) (*Workspace, *modconfig.ErrorAndWarnings) { utils.LogTime("workspace.Load start") defer utils.LogTime("workspace.Load end") workspace, err := createShellWorkspace(workspacePath) if err != nil { - return nil, err + return nil, modconfig.NewErrorsAndWarning(err) } // load the workspace mod - if err := workspace.loadWorkspaceMod(ctx); err != nil { - log.Printf("[TRACE] loadWorkspaceMod failed: %s", err.Error()) - return nil, err - } - - // return context error so calling code can handle cancellations - return workspace, nil + errAndWarnings := workspace.loadWorkspaceMod(ctx) + return workspace, errAndWarnings } // LoadVariables creates a Workspace and uses it to load all variables, ignoring any value resolution errors @@ -245,13 +240,13 @@ func (w *Workspace) findModFilePath(folder string) (string, error) { return modFilePath, nil } -func (w *Workspace) loadWorkspaceMod(ctx context.Context) error { +func (w *Workspace) loadWorkspaceMod(ctx context.Context) *modconfig.ErrorAndWarnings { // resolve values of all input variables // we WILL validate missing variables when loading validateMissing := true inputVariables, err := w.getInputVariables(ctx, validateMissing) if err != nil { - return err + return modconfig.NewErrorsAndWarning(err) } // populate the parsed variable values w.VariableValues = inputVariables.VariableValues @@ -259,7 +254,7 @@ func (w *Workspace) loadWorkspaceMod(ctx context.Context) error { // build run context which we use to load the workspace parseCtx, err := w.getParseContext() if err != nil { - return err + return modconfig.NewErrorsAndWarning(err) } // add variables to runContext parseCtx.AddInputVariables(inputVariables) @@ -267,9 +262,9 @@ func (w *Workspace) loadWorkspaceMod(ctx context.Context) error { parseCtx.BlockTypeExclusions = []string{modconfig.BlockTypeVariable} // load the workspace mod - m, err := steampipeconfig.LoadMod(w.Path, parseCtx) - if err != nil { - return err + m, errorAndWarning := steampipeconfig.LoadMod(w.Path, parseCtx) + if errorAndWarning.Error != nil { + return errorAndWarning } // now set workspace properties @@ -282,7 +277,9 @@ func (w *Workspace) loadWorkspaceMod(ctx context.Context) error { w.Mods[w.Mod.Name()] = w.Mod // verify all runtime dependencies can be resolved - return w.verifyResourceRuntimeDependencies() + errorAndWarning.Error = w.verifyResourceRuntimeDependencies() + + return errorAndWarning } func (w *Workspace) getInputVariables(ctx context.Context, validateMissing bool) (*modconfig.ModVariableMap, error) { diff --git a/pkg/workspace/workspace_events.go b/pkg/workspace/workspace_events.go index d481a01ce..e889f168e 100644 --- a/pkg/workspace/workspace_events.go +++ b/pkg/workspace/workspace_events.go @@ -78,12 +78,12 @@ func (w *Workspace) handleDashboardEvent() { func (w *Workspace) handleFileWatcherEvent(ctx context.Context, client db_common.Client, ev []fsnotify.Event) { log.Printf("[TRACE] handleFileWatcherEvent") - prevResourceMaps, resourceMaps, err := w.reloadResourceMaps(ctx) + prevResourceMaps, resourceMaps, errAndWarnings := w.reloadResourceMaps(ctx) - if err != nil { + if errAndWarnings.GetError() != nil { log.Printf("[TRACE] handleFileWatcherEvent reloadResourceMaps returned error - call PublishDashboardEvent") // publish error event - w.PublishDashboardEvent(ctx, &dashboardevents.WorkspaceError{Error: err}) + w.PublishDashboardEvent(ctx, &dashboardevents.WorkspaceError{Error: errAndWarnings.GetError()}) log.Printf("[TRACE] back from PublishDashboardEvent") return } @@ -102,7 +102,7 @@ func (w *Workspace) handleFileWatcherEvent(ctx context.Context, client db_common w.raiseDashboardChangedEvents(ctx, resourceMaps, prevResourceMaps) } -func (w *Workspace) reloadResourceMaps(ctx context.Context) (*modconfig.ResourceMaps, *modconfig.ResourceMaps, error) { +func (w *Workspace) reloadResourceMaps(ctx context.Context) (*modconfig.ResourceMaps, *modconfig.ResourceMaps, *modconfig.ErrorAndWarnings) { w.loadLock.Lock() defer w.loadLock.Unlock() @@ -115,15 +115,15 @@ func (w *Workspace) reloadResourceMaps(ctx context.Context) (*modconfig.Resource } // now reload the workspace - err := w.loadWorkspaceMod(ctx) - if err != nil { + errAndWarnings := w.loadWorkspaceMod(ctx) + if errAndWarnings.GetError() != nil { // check the existing watcher error - if we are already in an error state, do not show error if w.watcherError == nil { - w.fileWatcherErrorHandler(ctx, error_helpers.PrefixError(err, "failed to reload workspace")) + w.fileWatcherErrorHandler(ctx, error_helpers.PrefixError(errAndWarnings.GetError(), "failed to reload workspace")) } // now set watcher error to new error - w.watcherError = err - return nil, nil, err + w.watcherError = errAndWarnings.GetError() + return nil, nil, errAndWarnings } // clear watcher error w.watcherError = nil @@ -131,7 +131,7 @@ func (w *Workspace) reloadResourceMaps(ctx context.Context) (*modconfig.Resource // reload the resource maps resourceMaps := w.Mod.ResourceMaps - return prevResourceMaps, resourceMaps, nil + return prevResourceMaps, resourceMaps, errAndWarnings } diff --git a/tests/manual_testing/node_edge_vals/dashboard.sp b/tests/manual_testing/node_edge_vals/dashboard.sp deleted file mode 100644 index d01bfadc3..000000000 --- a/tests/manual_testing/node_edge_vals/dashboard.sp +++ /dev/null @@ -1,190 +0,0 @@ -dashboard "bug5_multiple_inputs" { - title = "bug5: multiple inputs" - input "bucket_arn" { - title = "Select a bucket:" - sql = query.bug5_s3_bucket_input.sql - width = 4 - } - graph { - title = "Relationships 1" - type = "graph" - //direction = "left_right" - with "policy_std" { - sql = <<-EOQ - select - policy_std - from - aws_s3_bucket - where - arn = $1 - EOQ - args = [self.input.bucket_arn.value] - } - nodes = [ - node.bug5_me_node, - node.bug5_iam_policy_statement_nodes, - ] - edges = [ - edge.bug5_bucket_policy_statement_edges, - ] - args = { - arn = self.input.bucket_arn.value - policy_std = with.policy_std.rows[0].policy_std - bucket_arns = [self.input.bucket_arn.value] - } - } - table { - # sql = <<-EOQ - sql = <<-EOQ - select - concat('statement:', i) as id, - coalesce ( - t.stmt ->> 'Sid', - concat('[', i::text, ']') - ) as title - from - aws_s3_bucket, - jsonb_array_elements(policy_std -> 'Statement') with ordinality as t(stmt,i) - where - arn = $1 - EOQ - args = [self.input.bucket_arn.value] - } - //*************** - input "lambda_arn" { - title = "Select a lambda function:" - sql = query.bug5_lambda_function_input.sql - width = 4 - } - graph { - title = "Relationships 2" - type = "graph" - //direction = "left_right" - with "policy_std" { - sql = <<-EOQ - select - policy_std - from - aws_lambda_function - where - arn = $1 - EOQ - args = [self.input.lambda_arn.value] - } - nodes = [ - node.bug5_me_node, - node.bug5_iam_policy_statement_nodes, - ] - edges = [ - edge.bug5_lambda_function_policy_statement_edges, - ] - args = { - arn = self.input.lambda_arn.value - policy_std = with.policy_std.rows[0].policy_std - lambda_arns = [self.input.lambda_arn.value] - } - } - table { - sql = <<-EOQ - select - concat('statement:', i) as id, - coalesce ( - t.stmt ->> 'Sid', - concat('[', i::text, ']') - ) as title - from - aws_lambda_function, - jsonb_array_elements(policy_std -> 'Statement') with ordinality as t(stmt,i) - where - arn = $1 - EOQ - - args = [self.input.lambda_arn.value] - } - -} -query "bug5_s3_bucket_input" { - sql = <<-EOQ - select - title as label, - arn as value, - json_build_object( - 'account_id', account_id, - 'region', region - ) as tags - from - aws_s3_bucket - order by - title; - EOQ -} -query "bug5_lambda_function_input" { - sql = <<-EOQ - select - title as label, - arn as value, - json_build_object( - 'account_id', account_id, - 'region', region - ) as tags - from - aws_lambda_function - order by - title; - EOQ -} -node "bug5_me_node" { - //category = category.aws_iam_policy - sql = <<-EOQ - select - $1 as id, - $1 as title - EOQ - param "arn" {} -} -node "bug5_iam_policy_statement_nodes" { - //category = category.aws_iam_policy_statement - sql = <<-EOQ - select - concat('statement:', i) as id, - coalesce ( - t.stmt ->> 'Sid', - concat('[', i::text, ']') - ) as title - from - jsonb_array_elements(($1 :: jsonb) -> 'Statement') with ordinality as t(stmt,i) - EOQ - param "policy_std" {} -} -edge "bug5_bucket_policy_statement_edges" { - title = "statement" - sql = <<-EOQ - - select - distinct on (arn,i) - arn as from_id, - concat('statement:', i) as to_id - from - aws_s3_bucket, - jsonb_array_elements(policy_std -> 'Statement') with ordinality as t(stmt,i) - where - arn = any($1) - EOQ - param "bucket_arns" {} -} -edge "bug5_lambda_function_policy_statement_edges" { - title = "statement" - sql = <<-EOQ - - select - distinct on (arn,i) - arn as from_id, - concat('statement:', i) as to_id - from - aws_lambda_function, - jsonb_array_elements(policy_std -> 'Statement') with ordinality as t(stmt,i) - where - arn = any($1) - EOQ - param "lambda_arns" {} -} \ No newline at end of file diff --git a/tests/manual_testing/node_edge_vals/mod.sp b/tests/manual_testing/node_edge_vals/mod.sp deleted file mode 100644 index 5cf8b0382..000000000 --- a/tests/manual_testing/node_edge_vals/mod.sp +++ /dev/null @@ -1,3 +0,0 @@ -mod reports_poc { - title = "Reports POC" -} \ No newline at end of file diff --git a/tests/manual_testing/node_reuse/inputs/dashboard.sp b/tests/manual_testing/node_reuse/inputs/dashboard.sp index bd502e6cb..cf15682e1 100644 --- a/tests/manual_testing/node_reuse/inputs/dashboard.sp +++ b/tests/manual_testing/node_reuse/inputs/dashboard.sp @@ -13,7 +13,8 @@ dashboard "inputs" { } table { - query = query.q1 + param "foo" {} + sql = "select $1" args = { arn = self.input.i1.value } @@ -30,4 +31,5 @@ dashboard "inputs" { query "q1"{ sql = "select arn from aws_account where arn = $1" param "arn" { } + search_path="test" }