diff --git a/control/controldisplay/export_template.go b/control/controldisplay/export_template.go index 6f5aa2748..da69ba78f 100644 --- a/control/controldisplay/export_template.go +++ b/control/controldisplay/export_template.go @@ -62,7 +62,7 @@ func ResolveExportTemplate(export string, allowFilenameEvaluation bool) (format } if !allowFilenameEvaluation { - return nil, "", ErrTemplateNotFound + return nil, "", fmt.Errorf("template %s not found", export) } // if the above didn't match, then the input argument is a file name @@ -85,7 +85,7 @@ func findTemplateByFilename(export string, available []*ExportTemplate) (format extension := filepath.Ext(export) if len(extension) == 0 { // we don't have anything to work with - return nil, ErrTemplateNotFound + return nil, fmt.Errorf("template %s not found", export) } matchingTemplates := []*ExportTemplate{} @@ -97,21 +97,23 @@ func findTemplateByFilename(export string, available []*ExportTemplate) (format } if len(matchingTemplates) > 1 { + matchNames := []string{} // find out if any of them has preference for _, match := range matchingTemplates { if match.DefaultTemplateForExtension { return match, nil } + matchNames = append(matchNames, match.FormatName) } // there's ambiguity - we have more than one matching templates based on extension - return nil, ErrAmbiguousTemplate + return nil, fmt.Errorf("ambiguous templates found: %v", matchNames) } if len(matchingTemplates) == 1 { return matchingTemplates[0], nil } - return nil, ErrTemplateNotFound + return nil, fmt.Errorf("template %s not found", export) } func loadAvailableTemplates() ([]*ExportTemplate, error) { diff --git a/control/controldisplay/formatter.go b/control/controldisplay/formatter.go index 8b836f12d..06da65fff 100644 --- a/control/controldisplay/formatter.go +++ b/control/controldisplay/formatter.go @@ -3,14 +3,13 @@ package controldisplay import ( "context" "errors" - "fmt" "io" "os" - "reflect" "strings" "text/template" "time" + "github.com/MasterMinds/sprig" "github.com/turbot/steampipe/constants" "github.com/turbot/steampipe/control/controlexecute" "github.com/turbot/steampipe/version" @@ -92,73 +91,28 @@ func (j *NullFormatter) FileExtension() string { return "" } +func templateFuncs() template.FuncMap { + useFromSprigMap := []string{"upper", "toJson", "quote", "dict", "add", "now"} + + var funcs template.FuncMap = template.FuncMap{} + sprigMap := sprig.TxtFuncMap() + for _, use := range useFromSprigMap { + f, found := sprigMap[use] + if found { + funcs[use] = f + } + } + for k, v := range formatterTemplateFuncMap { + funcs[k] = v + } + + return funcs +} + var formatterTemplateFuncMap template.FuncMap = template.FuncMap{ "steampipeversion": func() string { return version.SteampipeVersion.String() }, "workingdir": func() string { wd, _ := os.Getwd(); return wd }, - "asstr": func(i reflect.Value) string { return fmt.Sprintf("%v", i) }, - "dict": func(values ...interface{}) (map[string]interface{}, error) { - if len(values)%2 != 0 { - return nil, errors.New("invalid dict call") - } - dict := make(map[string]interface{}, len(values)/2) - for i := 0; i < len(values); i += 2 { - key, ok := values[i].(string) - if !ok { - return nil, errors.New("dict keys must be strings") - } - dict[key] = values[i+1] - } - return dict, nil - }, - "summarystatusclass": func(status string, total int) string { - switch strings.ToLower(status) { - case "ok": - if total > 0 { - return "summary-total-ok highlight" - } - return "summary-total-ok" - case "skip": - if total > 0 { - return "summary-total-skip highlight" - } - return "summary-total-skip" - case "info": - if total > 0 { - return "summary-total-info highlight" - } - return "summary-total-info" - case "alarm": - if total > 0 { - return "summary-total-alarm highlight" - } - return "summary-total-alarm" - case "error": - if total > 0 { - return "summary-total-error highlight" - } - return "summary-total-error" - } - return "" - }, - "ToUpper": func(text string) string { - return strings.ToUpper(text) - }, - "timenow": func() string { - return time.Now().Format(time.RFC3339) - }, - "GetDimensionRegion": func(row *controlexecute.ResultRow) string { - if row.Dimensions[0].Key == "region" { - return row.Dimensions[0].Value - } - return "ap-south-1" - }, - "GetDimensionAccount": func(row *controlexecute.ResultRow) string { - if row.Dimensions[0].Key == "account_id" { - return row.Dimensions[0].Value - } - return row.Dimensions[1].Value - }, - "DurationInFloat": func(t time.Duration) float64 { + "DurationInSeconds": func(t time.Duration) float64 { return t.Seconds() }, } diff --git a/control/controldisplay/formatter_template.go b/control/controldisplay/formatter_template.go index 26c4ca2d0..6e98f5f54 100644 --- a/control/controldisplay/formatter_template.go +++ b/control/controldisplay/formatter_template.go @@ -2,16 +2,29 @@ package controldisplay import ( "context" - "errors" + "fmt" "io" "os" "text/template" + "github.com/spf13/viper" + "github.com/turbot/steampipe/constants" "github.com/turbot/steampipe/control/controlexecute" + "github.com/turbot/steampipe/version" ) -var ErrAmbiguousTemplate = errors.New("ambiguous templates found") -var ErrTemplateNotFound = errors.New("template not found") +type TemplateRenderConfig struct { + RenderHeader bool +} +type TemplateRenderConstants struct { + SteampipeVersion string +} + +type TemplateRenderContext struct { + Constants TemplateRenderConstants + Config TemplateRenderConfig + Data *controlexecute.ExecutionTree +} // TemplateFormatter implements the 'Formatter' interface and exposes a generic template based output mechanism // for 'check' execution trees @@ -23,7 +36,17 @@ type TemplateFormatter struct { func (tf TemplateFormatter) Format(ctx context.Context, tree *controlexecute.ExecutionTree) (io.Reader, error) { reader, writer := io.Pipe() go func() { - if err := tf.template.ExecuteTemplate(writer, "outlet", tree); err != nil { + renderContext := TemplateRenderContext{ + Constants: TemplateRenderConstants{ + SteampipeVersion: version.SteampipeVersion.String(), + }, + Config: TemplateRenderConfig{ + RenderHeader: viper.GetBool(constants.ArgHeader), + }, + Data: tree, + } + + if err := tf.template.ExecuteTemplate(writer, "output", renderContext); err != nil { writer.CloseWithError(err) } else { writer.Close() @@ -38,17 +61,14 @@ func (tf TemplateFormatter) FileExtension() string { return tf.exportFormat.OutputExtension } else { // otherwise return the fullname - return tf.exportFormat.FormatFullName + return fmt.Sprintf(".%s", tf.exportFormat.FormatFullName) } } func NewTemplateFormatter(input ExportTemplate) (*TemplateFormatter, error) { - t, err := template.New("outlet"). - Funcs(formatterTemplateFuncMap). - ParseFS(os.DirFS(input.TemplatePath), "*") + t := template.Must(template.New("outlet"). + Funcs(templateFuncs()). + ParseFS(os.DirFS(input.TemplatePath), "*")) - if err != nil { - return nil, err - } return &TemplateFormatter{exportFormat: input, template: t}, nil } diff --git a/control/controldisplay/formatter_test.go b/control/controldisplay/formatter_test.go index 4ade86b95..777122de7 100644 --- a/control/controldisplay/formatter_test.go +++ b/control/controldisplay/formatter_test.go @@ -70,7 +70,7 @@ var tree = &controlexecute.ExecutionTree{ Reason: "is pretty insecure", Resource: "some other resource", Dimensions: []controlexecute.Dimension{}, - Control: &c11, + Run: &controlexecute.ControlRun{Control: &c11}, }, }, }, @@ -82,7 +82,7 @@ var tree = &controlexecute.ExecutionTree{ Reason: "is pretty insecure", Resource: "some other resource", Dimensions: []controlexecute.Dimension{}, - Control: &c12, + Run: &controlexecute.ControlRun{Control: &c12}, }, }, }, @@ -99,7 +99,7 @@ var tree = &controlexecute.ExecutionTree{ Reason: "is pretty insecure", Resource: "some other resource", Dimensions: []controlexecute.Dimension{}, - Control: &c21, + Run: &controlexecute.ControlRun{Control: &c21}, }, }, }, @@ -111,7 +111,7 @@ var tree = &controlexecute.ExecutionTree{ Reason: "is pretty insecure", Resource: "some other resource", Dimensions: []controlexecute.Dimension{}, - Control: &c22, + Run: &controlexecute.ControlRun{Control: &c22}, }, }, }, diff --git a/control/controldisplay/templates/asff.json/001.asff.tmpl.json b/control/controldisplay/templates/asff.json/001.asff.tmpl.json deleted file mode 100644 index 1068a3ab5..000000000 --- a/control/controldisplay/templates/asff.json/001.asff.tmpl.json +++ /dev/null @@ -1,38 +0,0 @@ -{{ define "outlet" }}[ {{ range .ControlRuns -}} {{ range .Rows }} {{ template "control_run_template" . -}} {{ end -}} {{ end }} -] {{ end }} -{{ define "control_run_template" }} -{ - "SchemaVersion": "2018-10-08", - "Id": "{{ .Control.FullName }}", - "ProductArn": "arn:aws:securityhub:{{ . | GetDimensionRegion }}:{{ . | GetDimensionAccount }}:product/{{ . | GetDimensionAccount }}/default", - "ProductFields": { - "ProviderName": "Steampipe", - "ProviderVersion": "{{ steampipeversion }}" - }, - "GeneratorId": "steampipe-{{ .Control.ShortName }}", - "AwsAccountId": "{{ . | GetDimensionAccount }}", - "Types": [ - "automated" - ], - "UpdatedAt": "{{ timenow }}", - "CreatedAt": "{{ timenow }}", - "Title": "{{ .Control.Title }}", - "Description": "{{ .Control.Description }}",{{ with .Control.Severity }} - "Severity": { - "Label": "{{ . | ToUpper}}" - },{{ else }} - "Severity": { - "Label": "INFORMATIONAL" - },{{ end }} - "Resources": [ - { - "Type": "Other", - "Id": "{{ .Resource }}" - } - ], - "Compliance": { - "Status": "{{ template "statusmap" .Status -}}" - } -},{{ end -}} - -{{ define "statusmap" }}{{ if eq . "ok" }}PASSED{{ end -}}{{ if eq . "error" }}WARNING{{ end -}}{{ if eq . "alarm" }}FAILED{{ end -}}{{ if eq . "skip" }}NOT_AVAILABLE{{ end -}}{{ if eq . "info" }}NOT_AVAILABLE{{ end -}}{{ end -}} \ No newline at end of file diff --git a/control/controldisplay/templates/asff.json/output.tmpl b/control/controldisplay/templates/asff.json/output.tmpl new file mode 100644 index 000000000..c6314df84 --- /dev/null +++ b/control/controldisplay/templates/asff.json/output.tmpl @@ -0,0 +1,38 @@ +{{ define "output" }}[ {{ range $runIdx,$run := .Data.ControlRuns -}} {{ range $rowIdx,$row := $run.Rows }} {{ if gt (add $runIdx $rowIdx) 0 }} {{ if $row.Run.MatchTag "plugin" "aws" }} , {{ end }} {{end}} {{ template "control_row_template" $row -}} {{ end -}} {{ end }} +] {{ end }} +{{ define "control_row_template" }} {{ if .Run.MatchTag "plugin" "aws" }} +{ + "SchemaVersion": "2018-10-08", + "Id": "{{ .Run.Control.FullName }}", + "ProductArn": "arn:aws:securityhub:{{ .GetDimensionValue "region" }}:{{ .GetDimensionValue "account_id" }}:product/{{ .GetDimensionValue "account_id" }}/default", + "ProductFields": { + "ProviderName": "Steampipe", + "ProviderVersion": "{{ steampipeversion }}" + }, + "GeneratorId": "steampipe-{{ .Run.Control.ShortName }}", + "AwsAccountId": "{{ .GetDimensionValue "account_id" }}", + "Types": [ + "automated" + ], + "UpdatedAt": "{{ now.Format "2006-01-02T15:04:05Z07:00" }}", + "CreatedAt": "{{ now.Format "2006-01-02T15:04:05Z07:00" }}", + "Title": {{ toJson .Run.Control.Title }}, + "Description": {{ toJson .Run.Control.Description }},{{ with .Run.Control.Severity }} + "Severity": { + "Label": "{{ upper . }}" + },{{ else }} + "Severity": { + "Label": "INFORMATIONAL" + },{{ end }} + "Resources": [ + { + "Type": "Other", + "Id": "{{ .Resource }}" + } + ], + "Compliance": { + "Status": "{{ template "statusmap" .Status -}}" + } +}{{ end -}} {{ end -}} + +{{ define "statusmap" }}{{ if eq . "ok" }}PASSED{{ end -}}{{ if eq . "error" }}WARNING{{ end -}}{{ if eq . "alarm" }}FAILED{{ end -}}{{ if eq . "skip" }}NOT_AVAILABLE{{ end -}}{{ if eq . "info" }}NOT_AVAILABLE{{ end -}}{{ end -}} diff --git a/control/controldisplay/templates/html.html/favicon.b64 b/control/controldisplay/templates/html/favicon.b64 similarity index 100% rename from control/controldisplay/templates/html.html/favicon.b64 rename to control/controldisplay/templates/html/favicon.b64 diff --git a/control/controldisplay/templates/html.html/logo.b64 b/control/controldisplay/templates/html/logo.b64 similarity index 100% rename from control/controldisplay/templates/html.html/logo.b64 rename to control/controldisplay/templates/html/logo.b64 diff --git a/control/controldisplay/templates/html.html/normalize.tmpl.css b/control/controldisplay/templates/html/normalize.tmpl.css similarity index 100% rename from control/controldisplay/templates/html.html/normalize.tmpl.css rename to control/controldisplay/templates/html/normalize.tmpl.css diff --git a/control/controldisplay/templates/html.html/001.index.tmpl.html b/control/controldisplay/templates/html/output.tmpl similarity index 67% rename from control/controldisplay/templates/html.html/001.index.tmpl.html rename to control/controldisplay/templates/html/output.tmpl index 442895bd9..75f132c0a 100644 --- a/control/controldisplay/templates/html.html/001.index.tmpl.html +++ b/control/controldisplay/templates/html/output.tmpl @@ -1,4 +1,4 @@ -{{ define "outlet" }} +{{ define "output" }} @@ -19,15 +19,15 @@
{{/* we expect 0 or 1 root control runs */}} - {{ range .Root.ControlRuns -}} + {{ range .Data.Root.ControlRuns -}} {{ template "control_run_template" . -}} {{ end }} {{/* we expect 0 or 1 root groups */}} - {{ range .Root.Groups -}} + {{ range .Data.Root.Groups -}} {{ template "root_group_template" . -}} {{ end }} -
@@ -48,27 +48,27 @@ ✅ OK - {{ .Ok }} + {{ .Ok }} ⇨ Skip - {{ .Skip }} + {{ .Skip }} ℹ Info - {{ .Info }} + {{ .Info }} ❌ Alarm - {{ .Alarm }} + {{ .Alarm }} ❗ Error - {{ .Error }} + {{ .Error }} @@ -88,11 +88,11 @@ - {{ .Ok }} - {{ .Skip }} - {{ .Info }} - {{ .Alarm }} - {{ .Error }} + {{ .Ok }} + {{ .Skip }} + {{ .Info }} + {{ .Alarm }} + {{ .Error }} {{ .TotalCount }} @@ -186,4 +186,10 @@ {{ end }} -{{ define "statusicon" }}{{ if eq . "ok" }}✅{{ end -}}{{ if eq . "skip" }}⇨{{ end -}}{{ if eq . "info" }}ℹ{{ end -}}{{ if eq . "alarm" }}❌{{ end -}}{{ if eq . "error" }}❗{{ end -}}{{ end -}} \ No newline at end of file +{{ define "statusicon" }}{{ if eq . "ok" }}✅{{ end -}}{{ if eq . "skip" }}⇨{{ end -}}{{ if eq . "info" }}ℹ{{ end -}}{{ if eq . "alarm" }}❌{{ end -}}{{ if eq . "error" }}❗{{ end -}}{{ end -}} + +{{ define "summaryokclass" }}{{ if gt . 0 }}summary-total-ok highlight{{ end -}}{{ if eq . 0 }}summary-total-ok{{ end -}}{{ end -}} +{{ define "summaryskipclass" }}{{ if gt . 0 }}summary-total-skip highlight{{ end -}}{{ if eq . 0 }}summary-total-skip{{ end -}}{{ end -}} +{{ define "summaryinfoclass" }}{{ if gt . 0 }}summary-total-info highlight{{ end -}}{{ if eq . 0 }}summary-total-info{{ end -}}{{ end -}} +{{ define "summaryalarmclass" }}{{ if gt . 0 }}summary-total-alarm highlight{{ end -}}{{ if eq . 0 }}summary-total-alarm{{ end -}}{{ end -}} +{{ define "summaryerrorclass" }}{{ if gt . 0 }}summary-total-error highlight{{ end -}}{{ if eq . 0 }}summary-total-error{{ end -}}{{ end -}} diff --git a/control/controldisplay/templates/html.html/style.tmpl.css b/control/controldisplay/templates/html/style.tmpl.css similarity index 100% rename from control/controldisplay/templates/html.html/style.tmpl.css rename to control/controldisplay/templates/html/style.tmpl.css diff --git a/control/controldisplay/templates/markdown.md/001.index.tmpl.md b/control/controldisplay/templates/md/output.tmpl similarity index 88% rename from control/controldisplay/templates/markdown.md/001.index.tmpl.md rename to control/controldisplay/templates/md/output.tmpl index 97366f788..bac82e76e 100644 --- a/control/controldisplay/templates/markdown.md/001.index.tmpl.md +++ b/control/controldisplay/templates/md/output.tmpl @@ -1,15 +1,15 @@ -{{ define "outlet" }} +{{ define "output" }} {{/* we expect 0 or 1 root control runs */}} -{{ range .Root.ControlRuns -}} +{{ range .Data.Root.ControlRuns -}} {{ template "control_run_template" . -}} {{ end }} {{/* we expect 0 or 1 root groups */}} -{{ range .Root.Groups -}} +{{ range .Data.Root.Groups -}} {{ template "root_group_template" . -}} {{ end }} \ -_Report run at `{{ .StartTime.Format "2006-01-02 15:04:05" }}` using [`Steampipe {{ steampipeversion }}`](https://steampipe.io) in dir `{{ workingdir }}`._ +_Report run at `{{ .Data.StartTime.Format "2006-01-02 15:04:05" }}` using [`Steampipe {{ .Constants.SteampipeVersion }}`](https://steampipe.io) in dir `{{ workingdir }}`._ {{ end }} {{/* templates */}} diff --git a/control/controldisplay/templates/nunit3.xml/001.nunit3.tmpl.xml b/control/controldisplay/templates/nunit3.xml/output.tmpl similarity index 65% rename from control/controldisplay/templates/nunit3.xml/001.nunit3.tmpl.xml rename to control/controldisplay/templates/nunit3.xml/output.tmpl index a90fa7005..7f993f08b 100644 --- a/control/controldisplay/templates/nunit3.xml/001.nunit3.tmpl.xml +++ b/control/controldisplay/templates/nunit3.xml/output.tmpl @@ -1,17 +1,17 @@ -{{ define "outlet" }} - - {{ range .Root.Groups }} +{{ define "output" }} + + {{ range .Data.Root.Groups }} {{ template "group_template" . }} {{ end }} - {{ range .Root.ControlRuns -}} + {{ range .Data.Root.ControlRuns -}} {{ template "control_run_template" . -}} {{ end -}} {{ end }} {{ define "group_template" }} - + {{ range .Groups }} {{ template "group_template" . }} {{ end }} @@ -23,7 +23,7 @@ {{ end }} {{ define "control_run_template" }} - + {{ range $index,$row := .Rows }} {{ template "control_row_template" dict "idx" $index "row" $row }} {{ end }} diff --git a/control/controlexecute/control_run.go b/control/controlexecute/control_run.go index 451f39b08..121015c22 100644 --- a/control/controlexecute/control_run.go +++ b/control/controlexecute/control_run.go @@ -46,13 +46,16 @@ type ControlRun struct { BackendPid int64 `json:"-"` // the result - ControlId string `json:"control_id"` - Description string `json:"description"` - Severity string `json:"severity"` - Tags map[string]string `json:"tags"` - Title string `json:"title"` - RowMap map[string][]*ResultRow `json:"-"` - Rows []*ResultRow `json:"results"` + ControlId string `json:"control_id"` + Description string `json:"description"` + Severity string `json:"severity"` + Tags map[string]string `json:"tags"` + Title string `json:"title"` + RowMap map[string][]*ResultRow `json:"-"` + Rows []*ResultRow `json:"results"` + DimensionKeys []string `json:"-"` + Group *ResultGroup `json:"-"` + Tree *ExecutionTree `json:"-"` // the query result stream queryResult *queryresult.Result @@ -60,9 +63,7 @@ type ControlRun struct { stateLock sync.Mutex doneChan chan bool - group *ResultGroup - executionTree *ExecutionTree - attempts int + attempts int } func NewControlRun(control *modconfig.Control, group *ResultGroup, executionTree *ExecutionTree) *ControlRun { @@ -83,16 +84,47 @@ func NewControlRun(control *modconfig.Control, group *ResultGroup, executionTree Lifecycle: utils.NewLifecycleTimer(), - executionTree: executionTree, - runStatus: ControlRunReady, + Tree: executionTree, + runStatus: ControlRunReady, - group: group, + Group: group, doneChan: make(chan bool, 1), } res.Lifecycle.Add("constructed") return res } +func (r *ControlRun) GetRunStatus() ControlRunStatus { + r.stateLock.Lock() + defer r.stateLock.Unlock() + return r.runStatus +} + +func (r *ControlRun) Finished() bool { + status := r.GetRunStatus() + return status == ControlRunComplete || status == ControlRunError +} + +func (r *ControlRun) MatchTag(key string, value string) bool { + val, found := r.Tags[key] + return found && (val == value) +} + +func (r *ControlRun) GetError() error { + return r.runError +} + +func (r *ControlRun) setError(ctx context.Context, err error) { + if err == nil { + return + } + r.runError = utils.TransformErrorToSteampipe(err) + + // update error count + r.Summary.Error++ + r.setRunStatus(ctx, ControlRunError) +} + func (r *ControlRun) skip(ctx context.Context) { r.setRunStatus(ctx, ControlRunComplete) } @@ -165,16 +197,16 @@ func (r *ControlRun) execute(ctx context.Context, client db_common.Client) { // function to cleanup and update status after control run completion defer func() { // update the result group status with our status - this will be passed all the way up the execution tree - r.group.updateSummary(r.Summary) + r.Group.updateSummary(r.Summary) if len(r.Severity) != 0 { - r.group.updateSeverityCounts(r.Severity, r.Summary) + r.Group.updateSeverityCounts(r.Severity, r.Summary) } r.Lifecycle.Add("execute_end") r.Duration = time.Since(startTime) - if r.group != nil { - r.group.addDuration(r.Duration) + if r.Group != nil { + r.Group.addDuration(r.Duration) } - log.Printf("[TRACE] finishing with concurrency, %s, , %d\n", r.Control.Name(), r.executionTree.progress.executing) + log.Printf("[TRACE] finishing with concurrency, %s, , %d\n", r.Control.Name(), r.Tree.progress.executing) }() // get a db connection @@ -197,14 +229,14 @@ func (r *ControlRun) execute(ctx context.Context, client db_common.Client) { r.runStatus = ControlRunStarted // update the current running control in the Progress renderer - r.executionTree.progress.OnControlStart(ctx, control) - defer r.executionTree.progress.OnControlFinish(ctx) + r.Tree.progress.OnControlStart(ctx, control) + defer r.Tree.progress.OnControlFinish(ctx) // resolve the control query r.Lifecycle.Add("query_resolution_start") query, err := r.resolveControlQuery(control) if err != nil { - r.SetError(ctx, err) + r.setError(ctx, err) return } r.Lifecycle.Add("query_resolution_finish") @@ -212,7 +244,7 @@ func (r *ControlRun) execute(ctx context.Context, client db_common.Client) { log.Printf("[TRACE] setting search path %s\n", control.Name()) r.Lifecycle.Add("set_search_path_start") if err := r.setSearchPath(ctx, dbSession, client); err != nil { - r.SetError(ctx, err) + r.setError(ctx, err) return } r.Lifecycle.Add("set_search_path_finish") @@ -244,7 +276,7 @@ func (r *ControlRun) execute(ctx context.Context, client db_common.Client) { log.Printf("[TRACE] control %s query failed again with plugin connectivity error %s - NOT retrying...", r.Control.Name(), err) } } - r.SetError(ctx, err) + r.setError(ctx, err) return } @@ -256,21 +288,6 @@ func (r *ControlRun) execute(ctx context.Context, client db_common.Client) { log.Printf("[TRACE] finish result for, %s\n", control.Name()) } -func (r *ControlRun) SetError(ctx context.Context, err error) { - if err == nil { - return - } - r.runError = utils.TransformErrorToSteampipe(err) - - // update error count - r.Summary.Error++ - r.setRunStatus(ctx, ControlRunError) -} - -func (r *ControlRun) GetError() error { - return r.runError -} - // create a context with a deadline, and with status updates disabled (we do not want to show 'loading' results) func (r *ControlRun) getControlQueryContext(ctx context.Context) context.Context { // create a context with a deadline @@ -284,19 +301,8 @@ func (r *ControlRun) getControlQueryContext(ctx context.Context) context.Context return newCtx } -func (r *ControlRun) GetRunStatus() ControlRunStatus { - r.stateLock.Lock() - defer r.stateLock.Unlock() - return r.runStatus -} - -func (r *ControlRun) Finished() bool { - status := r.GetRunStatus() - return status == ControlRunComplete || status == ControlRunError -} - func (r *ControlRun) resolveControlQuery(control *modconfig.Control) (string, error) { - query, err := r.executionTree.workspace.ResolveControlQuery(control, nil) + query, err := r.Tree.workspace.ResolveControlQuery(control, nil) if err != nil { return "", fmt.Errorf(`cannot run %s - failed to resolve query "%s": %s`, control.Name(), typehelpers.SafeString(control.SQL), err.Error()) } @@ -317,7 +323,7 @@ func (r *ControlRun) waitForResults(ctx context.Context) { select { // check for cancellation case <-ctx.Done(): - r.SetError(ctx, ctx.Err()) + r.setError(ctx, ctx.Err()) case <-gatherDoneChan: // do nothing } @@ -326,6 +332,17 @@ func (r *ControlRun) waitForResults(ctx context.Context) { func (r *ControlRun) gatherResults(ctx context.Context) { r.Lifecycle.Add("gather_start") defer func() { r.Lifecycle.Add("gather_finish") }() + + defer func() { + for _, row := range r.Rows { + for _, dim := range row.Dimensions { + r.DimensionKeys = append(r.DimensionKeys, dim.Key) + } + } + r.DimensionKeys = utils.StringSliceDistinct(r.DimensionKeys) + r.Group.addDimensionKeys(r.DimensionKeys...) + }() + for { select { case row := <-*r.queryResult.RowChan: @@ -339,16 +356,16 @@ func (r *ControlRun) gatherResults(ctx context.Context) { // if the row is in error then we terminate the run if row.Error != nil { // set error status and summary - r.SetError(ctx, row.Error) + r.setError(ctx, row.Error) // update the result group status with our status - this will be passed all the way up the execution tree - r.group.updateSummary(r.Summary) + r.Group.updateSummary(r.Summary) return } // so all is ok - create another result row - result, err := NewResultRow(r.Control, row, r.queryResult.ColTypes) + result, err := NewResultRow(r, row, r.queryResult.ColTypes) if err != nil { - r.SetError(ctx, err) + r.setError(ctx, err) return } r.addResultRow(result) @@ -394,9 +411,9 @@ func (r *ControlRun) setRunStatus(ctx context.Context, status ControlRunStatus) if r.Finished() { // update Progress if status == ControlRunError { - r.executionTree.progress.OnControlError(ctx) + r.Tree.progress.OnControlError(ctx) } else { - r.executionTree.progress.OnControlComplete(ctx) + r.Tree.progress.OnControlComplete(ctx) } r.doneChan <- true diff --git a/control/controlexecute/result_group.go b/control/controlexecute/result_group.go index fb17dc0eb..c294b814b 100644 --- a/control/controlexecute/result_group.go +++ b/control/controlexecute/result_group.go @@ -3,6 +3,7 @@ package controlexecute import ( "context" "log" + "sort" "sync" "time" @@ -31,9 +32,10 @@ type ResultGroup struct { Severity map[string]StatusSummary `json:"-"` // the control tree item associated with this group(i.e. a mod/benchmark) - GroupItem modconfig.ModTreeItem `json:"-"` - Parent *ResultGroup `json:"-"` - Duration time.Duration `json:"-"` + GroupItem modconfig.ModTreeItem `json:"-"` + Parent *ResultGroup `json:"-"` + Duration time.Duration `json:"-"` + DimensionKeys []string `json:"-"` // lock to prevent multiple control_runs updating this updateLock *sync.Mutex @@ -113,6 +115,75 @@ func NewResultGroup(ctx context.Context, executionTree *ExecutionTree, treeItem return group } +func (r *ResultGroup) AllTagKeys() []string { + tags := []string{} + for k := range r.Tags { + tags = append(tags, k) + } + for _, child := range r.Groups { + tags = append(tags, child.AllTagKeys()...) + } + for _, run := range r.ControlRuns { + for k := range run.Control.Tags { + tags = append(tags, k) + } + } + tags = utils.StringSliceDistinct(tags) + sort.Strings(tags) + return tags +} + +// GetGroupByName finds an immediate child ResultGroup with a specific name +func (r *ResultGroup) GetGroupByName(name string) *ResultGroup { + for _, group := range r.Groups { + if group.GroupId == name { + return group + } + } + return nil +} + +// GetChildGroupByName finds a nested child ResultGroup with a specific name +func (r *ResultGroup) GetChildGroupByName(name string) *ResultGroup { + for _, group := range r.Groups { + if group.GroupId == name { + return group + } + if child := group.GetChildGroupByName(name); child != nil { + return child + } + } + return nil +} + +// GetControlRunByName finds a child ControlRun with a specific control name +func (r *ResultGroup) GetControlRunByName(name string) *ControlRun { + for _, run := range r.ControlRuns { + if run.Control.Name() == name { + return run + } + } + return nil +} + +func (r *ResultGroup) ControlRunCount() int { + count := len(r.ControlRuns) + for _, g := range r.Groups { + count += g.ControlRunCount() + } + return count +} + +func (r *ResultGroup) addDimensionKeys(keys ...string) { + r.updateLock.Lock() + defer r.updateLock.Unlock() + r.DimensionKeys = append(r.DimensionKeys, keys...) + if r.Parent != nil { + r.Parent.addDimensionKeys(keys...) + } + r.DimensionKeys = utils.StringSliceDistinct(r.DimensionKeys) +} + // populateGroupMap mutates the passed in a map to return all child result groups func (r *ResultGroup) populateGroupMap(groupMap map[string]*ResultGroup) { if groupMap == nil { @@ -182,7 +253,7 @@ func (r *ResultGroup) execute(ctx context.Context, client db_common.Client, para for _, controlRun := range r.ControlRuns { if utils.IsContextCancelled(ctx) { - controlRun.SetError(ctx, ctx.Err()) + controlRun.setError(ctx, ctx.Err()) continue } @@ -193,7 +264,7 @@ func (r *ResultGroup) execute(ctx context.Context, client db_common.Client, para err := parallelismLock.Acquire(ctx, 1) if err != nil { - controlRun.SetError(ctx, err) + controlRun.setError(ctx, err) continue } @@ -201,7 +272,7 @@ func (r *ResultGroup) execute(ctx context.Context, client db_common.Client, para defer func() { if r := recover(); r != nil { // if the Execute panic'ed, set it as an error - run.SetError(ctx, helpers.ToError(r)) + run.setError(ctx, helpers.ToError(r)) } // Release in defer, so that we don't retain the lock even if there's a panic inside parallelismLock.Release(1) @@ -213,44 +284,3 @@ func (r *ResultGroup) execute(ctx context.Context, client db_common.Client, para child.execute(ctx, client, parallelismLock) } } - -// GetGroupByName finds an immediate child ResultGroup with a specific name -func (r *ResultGroup) GetGroupByName(name string) *ResultGroup { - for _, group := range r.Groups { - if group.GroupId == name { - return group - } - } - return nil -} - -// GetChildGroupByName finds a nested child ResultGroup with a specific name -func (r *ResultGroup) GetChildGroupByName(name string) *ResultGroup { - for _, group := range r.Groups { - if group.GroupId == name { - return group - } - if child := group.GetChildGroupByName(name); child != nil { - return child - } - } - return nil -} - -// GetControlRunByName finds a child ControlRun with a specific control name -func (r *ResultGroup) GetControlRunByName(name string) *ControlRun { - for _, run := range r.ControlRuns { - if run.Control.Name() == name { - return run - } - } - return nil -} - -func (r *ResultGroup) ControlRunCount() int { - count := len(r.ControlRuns) - for _, g := range r.Groups { - count += g.ControlRunCount() - } - return count -} diff --git a/control/controlexecute/result_row.go b/control/controlexecute/result_row.go index ab3135591..dcd010843 100644 --- a/control/controlexecute/result_row.go +++ b/control/controlexecute/result_row.go @@ -19,9 +19,20 @@ type ResultRow struct { Resource string `json:"resource" csv:"resource"` Status string `json:"status" csv:"status"` Dimensions []Dimension `json:"dimensions"` + Run *ControlRun `json:"-"` Control *modconfig.Control `json:"-" csv:"control_id:UnqualifiedName,control_title:Title,control_description:Description"` } +// GetDimensionValue returns the value for a dimension key. Returns an empty string with 'false' if not found +func (r *ResultRow) GetDimensionValue(key string) string { + for _, dim := range r.Dimensions { + if dim.Key == key { + return dim.Value + } + } + return "" +} + // AddDimension checks whether a column value is a scalar type, and if so adds it to the Dimensions map func (r *ResultRow) AddDimension(c *sql.ColumnType, val interface{}) { switch c.ScanType().Kind() { @@ -32,13 +43,14 @@ func (r *ResultRow) AddDimension(c *sql.ColumnType, val interface{}) { } } -func NewResultRow(control *modconfig.Control, row *queryresult.RowResult, colTypes []*sql.ColumnType) (*ResultRow, error) { +func NewResultRow(run *ControlRun, row *queryresult.RowResult, colTypes []*sql.ColumnType) (*ResultRow, error) { // validate the required columns exist in the result if err := validateColumns(colTypes); err != nil { return nil, err } res := &ResultRow{ - Control: control, + Run: run, + Control: run.Control, } // was there a SQL error _executing the control diff --git a/filepaths/steampipe.go b/filepaths/steampipe.go index 29d3e8924..b4bf72aa5 100644 --- a/filepaths/steampipe.go +++ b/filepaths/steampipe.go @@ -35,7 +35,7 @@ func steampipeSubDir(dirName string) string { // TemplateDir returns the path to the templates directory (creates if missing) func TemplateDir() string { - return steampipeSubDir("templates") + return steampipeSubDir(filepath.Join("check", "templates")) } // PluginDir returns the path to the plugins directory (creates if missing) diff --git a/go.mod b/go.mod index b2f2beb3e..2fdaee114 100644 --- a/go.mod +++ b/go.mod @@ -58,6 +58,14 @@ require ( ) require ( + github.com/Masterminds/goutils v1.1.0 // indirect + github.com/huandu/xstrings v1.3.2 // indirect + github.com/mitchellh/copystructure v1.0.0 // indirect + github.com/mitchellh/reflectwalk v1.0.1 // indirect +) + +require ( + github.com/MasterMinds/sprig v2.22.0+incompatible github.com/Microsoft/go-winio v0.5.0 // indirect github.com/Microsoft/hcsshim v0.8.22 // indirect github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect diff --git a/go.sum b/go.sum index d8e4bdbd0..36a94aa17 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,9 @@ github.com/ChrisTrenkamp/goxpath v0.0.0-20170922090931-c385f95c6022/go.mod h1:nu github.com/ChrisTrenkamp/goxpath v0.0.0-20190607011252-c5096ec8773d/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4= github.com/Machiel/slugify v1.0.1 h1:EfWSlRWstMadsgzmiV7d0yVd2IFlagWH68Q+DcYCm4E= github.com/Machiel/slugify v1.0.1/go.mod h1:fTFGn5uWEynW4CUMG7sWkYXOf1UgDxyTM3DbR6Qfg3k= +github.com/MasterMinds/sprig v2.22.0+incompatible h1:bu+/znjYSisHsySkFnXCL5Ropbm2ElSWkpwhwBhgwr0= +github.com/MasterMinds/sprig v2.22.0+incompatible/go.mod h1:RVIjNhXONBqvB4TwWE/BuzAtF5cCTSpS/pcZEs6HkpU= +github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RPBiVg= github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= @@ -576,6 +579,7 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/iancoleman/strcase v0.1.2 h1:gnomlvw9tnV3ITTAxzKSgTF+8kFWcU/f+TgttpXGz1U= github.com/iancoleman/strcase v0.1.2/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= diff --git a/utils/string_slice.go b/utils/string_slice.go new file mode 100644 index 000000000..8be86cd42 --- /dev/null +++ b/utils/string_slice.go @@ -0,0 +1,14 @@ +package utils + +// TODO: investigate turbot/go-kit/helpers +func StringSliceDistinct(slice []string) []string { + var res []string + occurenceMap := make(map[string]struct{}) + for _, item := range slice { + occurenceMap[item] = struct{}{} + } + for item := range occurenceMap { + res = append(res, item) + } + return res +}