From be67b786f4e2e9be562879d3ac050787606a144d Mon Sep 17 00:00:00 2001
From: Puskar Basu <45908484+pskrbasu@users.noreply.github.com>
Date: Wed, 19 Jan 2022 22:06:11 +0530
Subject: [PATCH] Update check templates to remove functions. Closes #1320
---
control/controldisplay/export_template.go | 10 +-
control/controldisplay/formatter.go | 86 +++---------
control/controldisplay/formatter_template.go | 42 ++++--
control/controldisplay/formatter_test.go | 8 +-
.../templates/asff.json/001.asff.tmpl.json | 38 -----
.../templates/asff.json/output.tmpl | 38 +++++
.../templates/{html.html => html}/favicon.b64 | 0
.../templates/{html.html => html}/logo.b64 | 0
.../{html.html => html}/normalize.tmpl.css | 0
.../001.index.tmpl.html => html/output.tmpl} | 38 ++---
.../{html.html => html}/style.tmpl.css | 0
.../001.index.tmpl.md => md/output.tmpl} | 8 +-
.../{001.nunit3.tmpl.xml => output.tmpl} | 12 +-
control/controlexecute/control_run.go | 131 ++++++++++--------
control/controlexecute/result_group.go | 124 ++++++++++-------
control/controlexecute/result_row.go | 16 ++-
filepaths/steampipe.go | 2 +-
go.mod | 8 ++
go.sum | 4 +
utils/string_slice.go | 14 ++
20 files changed, 323 insertions(+), 256 deletions(-)
delete mode 100644 control/controldisplay/templates/asff.json/001.asff.tmpl.json
create mode 100644 control/controldisplay/templates/asff.json/output.tmpl
rename control/controldisplay/templates/{html.html => html}/favicon.b64 (100%)
rename control/controldisplay/templates/{html.html => html}/logo.b64 (100%)
rename control/controldisplay/templates/{html.html => html}/normalize.tmpl.css (100%)
rename control/controldisplay/templates/{html.html/001.index.tmpl.html => html/output.tmpl} (67%)
rename control/controldisplay/templates/{html.html => html}/style.tmpl.css (100%)
rename control/controldisplay/templates/{markdown.md/001.index.tmpl.md => md/output.tmpl} (88%)
rename control/controldisplay/templates/nunit3.xml/{001.nunit3.tmpl.xml => output.tmpl} (65%)
create mode 100644 utils/string_slice.go
diff --git a/control/controldisplay/export_template.go b/control/controldisplay/export_template.go
index 6f5aa2748..da69ba78f 100644
--- a/control/controldisplay/export_template.go
+++ b/control/controldisplay/export_template.go
@@ -62,7 +62,7 @@ func ResolveExportTemplate(export string, allowFilenameEvaluation bool) (format
}
if !allowFilenameEvaluation {
- return nil, "", ErrTemplateNotFound
+ return nil, "", fmt.Errorf("template %s not found", export)
}
// if the above didn't match, then the input argument is a file name
@@ -85,7 +85,7 @@ func findTemplateByFilename(export string, available []*ExportTemplate) (format
extension := filepath.Ext(export)
if len(extension) == 0 {
// we don't have anything to work with
- return nil, ErrTemplateNotFound
+ return nil, fmt.Errorf("template %s not found", export)
}
matchingTemplates := []*ExportTemplate{}
@@ -97,21 +97,23 @@ func findTemplateByFilename(export string, available []*ExportTemplate) (format
}
if len(matchingTemplates) > 1 {
+ matchNames := []string{}
// find out if any of them has preference
for _, match := range matchingTemplates {
if match.DefaultTemplateForExtension {
return match, nil
}
+ matchNames = append(matchNames, match.FormatName)
}
// there's ambiguity - we have more than one matching templates based on extension
- return nil, ErrAmbiguousTemplate
+ return nil, fmt.Errorf("ambiguous templates found: %v", matchNames)
}
if len(matchingTemplates) == 1 {
return matchingTemplates[0], nil
}
- return nil, ErrTemplateNotFound
+ return nil, fmt.Errorf("template %s not found", export)
}
func loadAvailableTemplates() ([]*ExportTemplate, error) {
diff --git a/control/controldisplay/formatter.go b/control/controldisplay/formatter.go
index 8b836f12d..06da65fff 100644
--- a/control/controldisplay/formatter.go
+++ b/control/controldisplay/formatter.go
@@ -3,14 +3,13 @@ package controldisplay
import (
"context"
"errors"
- "fmt"
"io"
"os"
- "reflect"
"strings"
"text/template"
"time"
+ "github.com/MasterMinds/sprig"
"github.com/turbot/steampipe/constants"
"github.com/turbot/steampipe/control/controlexecute"
"github.com/turbot/steampipe/version"
@@ -92,73 +91,28 @@ func (j *NullFormatter) FileExtension() string {
return ""
}
+func templateFuncs() template.FuncMap {
+ useFromSprigMap := []string{"upper", "toJson", "quote", "dict", "add", "now"}
+
+ var funcs template.FuncMap = template.FuncMap{}
+ sprigMap := sprig.TxtFuncMap()
+ for _, use := range useFromSprigMap {
+ f, found := sprigMap[use]
+ if found {
+ funcs[use] = f
+ }
+ }
+ for k, v := range formatterTemplateFuncMap {
+ funcs[k] = v
+ }
+
+ return funcs
+}
+
var formatterTemplateFuncMap template.FuncMap = template.FuncMap{
"steampipeversion": func() string { return version.SteampipeVersion.String() },
"workingdir": func() string { wd, _ := os.Getwd(); return wd },
- "asstr": func(i reflect.Value) string { return fmt.Sprintf("%v", i) },
- "dict": func(values ...interface{}) (map[string]interface{}, error) {
- if len(values)%2 != 0 {
- return nil, errors.New("invalid dict call")
- }
- dict := make(map[string]interface{}, len(values)/2)
- for i := 0; i < len(values); i += 2 {
- key, ok := values[i].(string)
- if !ok {
- return nil, errors.New("dict keys must be strings")
- }
- dict[key] = values[i+1]
- }
- return dict, nil
- },
- "summarystatusclass": func(status string, total int) string {
- switch strings.ToLower(status) {
- case "ok":
- if total > 0 {
- return "summary-total-ok highlight"
- }
- return "summary-total-ok"
- case "skip":
- if total > 0 {
- return "summary-total-skip highlight"
- }
- return "summary-total-skip"
- case "info":
- if total > 0 {
- return "summary-total-info highlight"
- }
- return "summary-total-info"
- case "alarm":
- if total > 0 {
- return "summary-total-alarm highlight"
- }
- return "summary-total-alarm"
- case "error":
- if total > 0 {
- return "summary-total-error highlight"
- }
- return "summary-total-error"
- }
- return ""
- },
- "ToUpper": func(text string) string {
- return strings.ToUpper(text)
- },
- "timenow": func() string {
- return time.Now().Format(time.RFC3339)
- },
- "GetDimensionRegion": func(row *controlexecute.ResultRow) string {
- if row.Dimensions[0].Key == "region" {
- return row.Dimensions[0].Value
- }
- return "ap-south-1"
- },
- "GetDimensionAccount": func(row *controlexecute.ResultRow) string {
- if row.Dimensions[0].Key == "account_id" {
- return row.Dimensions[0].Value
- }
- return row.Dimensions[1].Value
- },
- "DurationInFloat": func(t time.Duration) float64 {
+ "DurationInSeconds": func(t time.Duration) float64 {
return t.Seconds()
},
}
diff --git a/control/controldisplay/formatter_template.go b/control/controldisplay/formatter_template.go
index 26c4ca2d0..6e98f5f54 100644
--- a/control/controldisplay/formatter_template.go
+++ b/control/controldisplay/formatter_template.go
@@ -2,16 +2,29 @@ package controldisplay
import (
"context"
- "errors"
+ "fmt"
"io"
"os"
"text/template"
+ "github.com/spf13/viper"
+ "github.com/turbot/steampipe/constants"
"github.com/turbot/steampipe/control/controlexecute"
+ "github.com/turbot/steampipe/version"
)
-var ErrAmbiguousTemplate = errors.New("ambiguous templates found")
-var ErrTemplateNotFound = errors.New("template not found")
+type TemplateRenderConfig struct {
+ RenderHeader bool
+}
+type TemplateRenderConstants struct {
+ SteampipeVersion string
+}
+
+type TemplateRenderContext struct {
+ Constants TemplateRenderConstants
+ Config TemplateRenderConfig
+ Data *controlexecute.ExecutionTree
+}
// TemplateFormatter implements the 'Formatter' interface and exposes a generic template based output mechanism
// for 'check' execution trees
@@ -23,7 +36,17 @@ type TemplateFormatter struct {
func (tf TemplateFormatter) Format(ctx context.Context, tree *controlexecute.ExecutionTree) (io.Reader, error) {
reader, writer := io.Pipe()
go func() {
- if err := tf.template.ExecuteTemplate(writer, "outlet", tree); err != nil {
+ renderContext := TemplateRenderContext{
+ Constants: TemplateRenderConstants{
+ SteampipeVersion: version.SteampipeVersion.String(),
+ },
+ Config: TemplateRenderConfig{
+ RenderHeader: viper.GetBool(constants.ArgHeader),
+ },
+ Data: tree,
+ }
+
+ if err := tf.template.ExecuteTemplate(writer, "output", renderContext); err != nil {
writer.CloseWithError(err)
} else {
writer.Close()
@@ -38,17 +61,14 @@ func (tf TemplateFormatter) FileExtension() string {
return tf.exportFormat.OutputExtension
} else {
// otherwise return the fullname
- return tf.exportFormat.FormatFullName
+ return fmt.Sprintf(".%s", tf.exportFormat.FormatFullName)
}
}
func NewTemplateFormatter(input ExportTemplate) (*TemplateFormatter, error) {
- t, err := template.New("outlet").
- Funcs(formatterTemplateFuncMap).
- ParseFS(os.DirFS(input.TemplatePath), "*")
+ t := template.Must(template.New("outlet").
+ Funcs(templateFuncs()).
+ ParseFS(os.DirFS(input.TemplatePath), "*"))
- if err != nil {
- return nil, err
- }
return &TemplateFormatter{exportFormat: input, template: t}, nil
}
diff --git a/control/controldisplay/formatter_test.go b/control/controldisplay/formatter_test.go
index 4ade86b95..777122de7 100644
--- a/control/controldisplay/formatter_test.go
+++ b/control/controldisplay/formatter_test.go
@@ -70,7 +70,7 @@ var tree = &controlexecute.ExecutionTree{
Reason: "is pretty insecure",
Resource: "some other resource",
Dimensions: []controlexecute.Dimension{},
- Control: &c11,
+ Run: &controlexecute.ControlRun{Control: &c11},
},
},
},
@@ -82,7 +82,7 @@ var tree = &controlexecute.ExecutionTree{
Reason: "is pretty insecure",
Resource: "some other resource",
Dimensions: []controlexecute.Dimension{},
- Control: &c12,
+ Run: &controlexecute.ControlRun{Control: &c12},
},
},
},
@@ -99,7 +99,7 @@ var tree = &controlexecute.ExecutionTree{
Reason: "is pretty insecure",
Resource: "some other resource",
Dimensions: []controlexecute.Dimension{},
- Control: &c21,
+ Run: &controlexecute.ControlRun{Control: &c21},
},
},
},
@@ -111,7 +111,7 @@ var tree = &controlexecute.ExecutionTree{
Reason: "is pretty insecure",
Resource: "some other resource",
Dimensions: []controlexecute.Dimension{},
- Control: &c22,
+ Run: &controlexecute.ControlRun{Control: &c22},
},
},
},
diff --git a/control/controldisplay/templates/asff.json/001.asff.tmpl.json b/control/controldisplay/templates/asff.json/001.asff.tmpl.json
deleted file mode 100644
index 1068a3ab5..000000000
--- a/control/controldisplay/templates/asff.json/001.asff.tmpl.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{{ define "outlet" }}[ {{ range .ControlRuns -}} {{ range .Rows }} {{ template "control_run_template" . -}} {{ end -}} {{ end }}
-] {{ end }}
-{{ define "control_run_template" }}
-{
- "SchemaVersion": "2018-10-08",
- "Id": "{{ .Control.FullName }}",
- "ProductArn": "arn:aws:securityhub:{{ . | GetDimensionRegion }}:{{ . | GetDimensionAccount }}:product/{{ . | GetDimensionAccount }}/default",
- "ProductFields": {
- "ProviderName": "Steampipe",
- "ProviderVersion": "{{ steampipeversion }}"
- },
- "GeneratorId": "steampipe-{{ .Control.ShortName }}",
- "AwsAccountId": "{{ . | GetDimensionAccount }}",
- "Types": [
- "automated"
- ],
- "UpdatedAt": "{{ timenow }}",
- "CreatedAt": "{{ timenow }}",
- "Title": "{{ .Control.Title }}",
- "Description": "{{ .Control.Description }}",{{ with .Control.Severity }}
- "Severity": {
- "Label": "{{ . | ToUpper}}"
- },{{ else }}
- "Severity": {
- "Label": "INFORMATIONAL"
- },{{ end }}
- "Resources": [
- {
- "Type": "Other",
- "Id": "{{ .Resource }}"
- }
- ],
- "Compliance": {
- "Status": "{{ template "statusmap" .Status -}}"
- }
-},{{ end -}}
-
-{{ define "statusmap" }}{{ if eq . "ok" }}PASSED{{ end -}}{{ if eq . "error" }}WARNING{{ end -}}{{ if eq . "alarm" }}FAILED{{ end -}}{{ if eq . "skip" }}NOT_AVAILABLE{{ end -}}{{ if eq . "info" }}NOT_AVAILABLE{{ end -}}{{ end -}}
\ No newline at end of file
diff --git a/control/controldisplay/templates/asff.json/output.tmpl b/control/controldisplay/templates/asff.json/output.tmpl
new file mode 100644
index 000000000..c6314df84
--- /dev/null
+++ b/control/controldisplay/templates/asff.json/output.tmpl
@@ -0,0 +1,38 @@
+{{ define "output" }}[ {{ range $runIdx,$run := .Data.ControlRuns -}} {{ range $rowIdx,$row := $run.Rows }} {{ if gt (add $runIdx $rowIdx) 0 }} {{ if $row.Run.MatchTag "plugin" "aws" }} , {{ end }} {{end}} {{ template "control_row_template" $row -}} {{ end -}} {{ end }}
+] {{ end }}
+{{ define "control_row_template" }} {{ if .Run.MatchTag "plugin" "aws" }}
+{
+ "SchemaVersion": "2018-10-08",
+ "Id": "{{ .Run.Control.FullName }}",
+ "ProductArn": "arn:aws:securityhub:{{ .GetDimensionValue "region" }}:{{ .GetDimensionValue "account_id" }}:product/{{ .GetDimensionValue "account_id" }}/default",
+ "ProductFields": {
+ "ProviderName": "Steampipe",
+ "ProviderVersion": "{{ steampipeversion }}"
+ },
+ "GeneratorId": "steampipe-{{ .Run.Control.ShortName }}",
+ "AwsAccountId": "{{ .GetDimensionValue "account_id" }}",
+ "Types": [
+ "automated"
+ ],
+ "UpdatedAt": "{{ now.Format "2006-01-02T15:04:05Z07:00" }}",
+ "CreatedAt": "{{ now.Format "2006-01-02T15:04:05Z07:00" }}",
+ "Title": {{ toJson .Run.Control.Title }},
+ "Description": {{ toJson .Run.Control.Description }},{{ with .Run.Control.Severity }}
+ "Severity": {
+ "Label": "{{ upper . }}"
+ },{{ else }}
+ "Severity": {
+ "Label": "INFORMATIONAL"
+ },{{ end }}
+ "Resources": [
+ {
+ "Type": "Other",
+ "Id": "{{ .Resource }}"
+ }
+ ],
+ "Compliance": {
+ "Status": "{{ template "statusmap" .Status -}}"
+ }
+}{{ end -}} {{ end -}}
+
+{{ define "statusmap" }}{{ if eq . "ok" }}PASSED{{ end -}}{{ if eq . "error" }}WARNING{{ end -}}{{ if eq . "alarm" }}FAILED{{ end -}}{{ if eq . "skip" }}NOT_AVAILABLE{{ end -}}{{ if eq . "info" }}NOT_AVAILABLE{{ end -}}{{ end -}}
diff --git a/control/controldisplay/templates/html.html/favicon.b64 b/control/controldisplay/templates/html/favicon.b64
similarity index 100%
rename from control/controldisplay/templates/html.html/favicon.b64
rename to control/controldisplay/templates/html/favicon.b64
diff --git a/control/controldisplay/templates/html.html/logo.b64 b/control/controldisplay/templates/html/logo.b64
similarity index 100%
rename from control/controldisplay/templates/html.html/logo.b64
rename to control/controldisplay/templates/html/logo.b64
diff --git a/control/controldisplay/templates/html.html/normalize.tmpl.css b/control/controldisplay/templates/html/normalize.tmpl.css
similarity index 100%
rename from control/controldisplay/templates/html.html/normalize.tmpl.css
rename to control/controldisplay/templates/html/normalize.tmpl.css
diff --git a/control/controldisplay/templates/html.html/001.index.tmpl.html b/control/controldisplay/templates/html/output.tmpl
similarity index 67%
rename from control/controldisplay/templates/html.html/001.index.tmpl.html
rename to control/controldisplay/templates/html/output.tmpl
index 442895bd9..75f132c0a 100644
--- a/control/controldisplay/templates/html.html/001.index.tmpl.html
+++ b/control/controldisplay/templates/html/output.tmpl
@@ -1,4 +1,4 @@
-{{ define "outlet" }}
+{{ define "output" }}
@@ -19,15 +19,15 @@
{{/* we expect 0 or 1 root control runs */}}
- {{ range .Root.ControlRuns -}}
+ {{ range .Data.Root.ControlRuns -}}
{{ template "control_run_template" . -}}
{{ end }}
{{/* we expect 0 or 1 root groups */}}
- {{ range .Root.Groups -}}
+ {{ range .Data.Root.Groups -}}
{{ template "root_group_template" . -}}
{{ end }}
-
@@ -48,27 +48,27 @@
| ✅ |
OK |
- {{ .Ok }} |
+ {{ .Ok }} |
| ⇨ |
Skip |
- {{ .Skip }} |
+ {{ .Skip }} |
| ℹ |
Info |
- {{ .Info }} |
+ {{ .Info }} |
| ❌ |
Alarm |
- {{ .Alarm }} |
+ {{ .Alarm }} |
| ❗ |
Error |
- {{ .Error }} |
+ {{ .Error }} |
@@ -88,11 +88,11 @@
- | {{ .Ok }} |
- {{ .Skip }} |
- {{ .Info }} |
- {{ .Alarm }} |
- {{ .Error }} |
+ {{ .Ok }} |
+ {{ .Skip }} |
+ {{ .Info }} |
+ {{ .Alarm }} |
+ {{ .Error }} |
{{ .TotalCount }} |
@@ -186,4 +186,10 @@
{{ end }}
-{{ define "statusicon" }}{{ if eq . "ok" }}✅{{ end -}}{{ if eq . "skip" }}⇨{{ end -}}{{ if eq . "info" }}ℹ{{ end -}}{{ if eq . "alarm" }}❌{{ end -}}{{ if eq . "error" }}❗{{ end -}}{{ end -}}
\ No newline at end of file
+{{ define "statusicon" }}{{ if eq . "ok" }}✅{{ end -}}{{ if eq . "skip" }}⇨{{ end -}}{{ if eq . "info" }}ℹ{{ end -}}{{ if eq . "alarm" }}❌{{ end -}}{{ if eq . "error" }}❗{{ end -}}{{ end -}}
+
+{{ define "summaryokclass" }}{{ if gt . 0 }}summary-total-ok highlight{{ end -}}{{ if eq . 0 }}summary-total-ok{{ end -}}{{ end -}}
+{{ define "summaryskipclass" }}{{ if gt . 0 }}summary-total-skip highlight{{ end -}}{{ if eq . 0 }}summary-total-skip{{ end -}}{{ end -}}
+{{ define "summaryinfoclass" }}{{ if gt . 0 }}summary-total-info highlight{{ end -}}{{ if eq . 0 }}summary-total-info{{ end -}}{{ end -}}
+{{ define "summaryalarmclass" }}{{ if gt . 0 }}summary-total-alarm highlight{{ end -}}{{ if eq . 0 }}summary-total-alarm{{ end -}}{{ end -}}
+{{ define "summaryerrorclass" }}{{ if gt . 0 }}summary-total-error highlight{{ end -}}{{ if eq . 0 }}summary-total-error{{ end -}}{{ end -}}
diff --git a/control/controldisplay/templates/html.html/style.tmpl.css b/control/controldisplay/templates/html/style.tmpl.css
similarity index 100%
rename from control/controldisplay/templates/html.html/style.tmpl.css
rename to control/controldisplay/templates/html/style.tmpl.css
diff --git a/control/controldisplay/templates/markdown.md/001.index.tmpl.md b/control/controldisplay/templates/md/output.tmpl
similarity index 88%
rename from control/controldisplay/templates/markdown.md/001.index.tmpl.md
rename to control/controldisplay/templates/md/output.tmpl
index 97366f788..bac82e76e 100644
--- a/control/controldisplay/templates/markdown.md/001.index.tmpl.md
+++ b/control/controldisplay/templates/md/output.tmpl
@@ -1,15 +1,15 @@
-{{ define "outlet" }}
+{{ define "output" }}
{{/* we expect 0 or 1 root control runs */}}
-{{ range .Root.ControlRuns -}}
+{{ range .Data.Root.ControlRuns -}}
{{ template "control_run_template" . -}}
{{ end }}
{{/* we expect 0 or 1 root groups */}}
-{{ range .Root.Groups -}}
+{{ range .Data.Root.Groups -}}
{{ template "root_group_template" . -}}
{{ end }}
\
-_Report run at `{{ .StartTime.Format "2006-01-02 15:04:05" }}` using [`Steampipe {{ steampipeversion }}`](https://steampipe.io) in dir `{{ workingdir }}`._
+_Report run at `{{ .Data.StartTime.Format "2006-01-02 15:04:05" }}` using [`Steampipe {{ .Constants.SteampipeVersion }}`](https://steampipe.io) in dir `{{ workingdir }}`._
{{ end }}
{{/* templates */}}
diff --git a/control/controldisplay/templates/nunit3.xml/001.nunit3.tmpl.xml b/control/controldisplay/templates/nunit3.xml/output.tmpl
similarity index 65%
rename from control/controldisplay/templates/nunit3.xml/001.nunit3.tmpl.xml
rename to control/controldisplay/templates/nunit3.xml/output.tmpl
index a90fa7005..7f993f08b 100644
--- a/control/controldisplay/templates/nunit3.xml/001.nunit3.tmpl.xml
+++ b/control/controldisplay/templates/nunit3.xml/output.tmpl
@@ -1,17 +1,17 @@
-{{ define "outlet" }}
-
- {{ range .Root.Groups }}
+{{ define "output" }}
+
+ {{ range .Data.Root.Groups }}
{{ template "group_template" . }}
{{ end }}
- {{ range .Root.ControlRuns -}}
+ {{ range .Data.Root.ControlRuns -}}
{{ template "control_run_template" . -}}
{{ end -}}
{{ end }}
{{ define "group_template" }}
-
+
{{ range .Groups }}
{{ template "group_template" . }}
{{ end }}
@@ -23,7 +23,7 @@
{{ end }}
{{ define "control_run_template" }}
-
+
{{ range $index,$row := .Rows }}
{{ template "control_row_template" dict "idx" $index "row" $row }}
{{ end }}
diff --git a/control/controlexecute/control_run.go b/control/controlexecute/control_run.go
index 451f39b08..121015c22 100644
--- a/control/controlexecute/control_run.go
+++ b/control/controlexecute/control_run.go
@@ -46,13 +46,16 @@ type ControlRun struct {
BackendPid int64 `json:"-"`
// the result
- ControlId string `json:"control_id"`
- Description string `json:"description"`
- Severity string `json:"severity"`
- Tags map[string]string `json:"tags"`
- Title string `json:"title"`
- RowMap map[string][]*ResultRow `json:"-"`
- Rows []*ResultRow `json:"results"`
+ ControlId string `json:"control_id"`
+ Description string `json:"description"`
+ Severity string `json:"severity"`
+ Tags map[string]string `json:"tags"`
+ Title string `json:"title"`
+ RowMap map[string][]*ResultRow `json:"-"`
+ Rows []*ResultRow `json:"results"`
+ DimensionKeys []string `json:"-"`
+ Group *ResultGroup `json:"-"`
+ Tree *ExecutionTree `json:"-"`
// the query result stream
queryResult *queryresult.Result
@@ -60,9 +63,7 @@ type ControlRun struct {
stateLock sync.Mutex
doneChan chan bool
- group *ResultGroup
- executionTree *ExecutionTree
- attempts int
+ attempts int
}
func NewControlRun(control *modconfig.Control, group *ResultGroup, executionTree *ExecutionTree) *ControlRun {
@@ -83,16 +84,47 @@ func NewControlRun(control *modconfig.Control, group *ResultGroup, executionTree
Lifecycle: utils.NewLifecycleTimer(),
- executionTree: executionTree,
- runStatus: ControlRunReady,
+ Tree: executionTree,
+ runStatus: ControlRunReady,
- group: group,
+ Group: group,
doneChan: make(chan bool, 1),
}
res.Lifecycle.Add("constructed")
return res
}
+func (r *ControlRun) GetRunStatus() ControlRunStatus {
+ r.stateLock.Lock()
+ defer r.stateLock.Unlock()
+ return r.runStatus
+}
+
+func (r *ControlRun) Finished() bool {
+ status := r.GetRunStatus()
+ return status == ControlRunComplete || status == ControlRunError
+}
+
+func (r *ControlRun) MatchTag(key string, value string) bool {
+ val, found := r.Tags[key]
+ return found && (val == value)
+}
+
+func (r *ControlRun) GetError() error {
+ return r.runError
+}
+
+func (r *ControlRun) setError(ctx context.Context, err error) {
+ if err == nil {
+ return
+ }
+ r.runError = utils.TransformErrorToSteampipe(err)
+
+ // update error count
+ r.Summary.Error++
+ r.setRunStatus(ctx, ControlRunError)
+}
+
func (r *ControlRun) skip(ctx context.Context) {
r.setRunStatus(ctx, ControlRunComplete)
}
@@ -165,16 +197,16 @@ func (r *ControlRun) execute(ctx context.Context, client db_common.Client) {
// function to cleanup and update status after control run completion
defer func() {
// update the result group status with our status - this will be passed all the way up the execution tree
- r.group.updateSummary(r.Summary)
+ r.Group.updateSummary(r.Summary)
if len(r.Severity) != 0 {
- r.group.updateSeverityCounts(r.Severity, r.Summary)
+ r.Group.updateSeverityCounts(r.Severity, r.Summary)
}
r.Lifecycle.Add("execute_end")
r.Duration = time.Since(startTime)
- if r.group != nil {
- r.group.addDuration(r.Duration)
+ if r.Group != nil {
+ r.Group.addDuration(r.Duration)
}
- log.Printf("[TRACE] finishing with concurrency, %s, , %d\n", r.Control.Name(), r.executionTree.progress.executing)
+ log.Printf("[TRACE] finishing with concurrency, %s, , %d\n", r.Control.Name(), r.Tree.progress.executing)
}()
// get a db connection
@@ -197,14 +229,14 @@ func (r *ControlRun) execute(ctx context.Context, client db_common.Client) {
r.runStatus = ControlRunStarted
// update the current running control in the Progress renderer
- r.executionTree.progress.OnControlStart(ctx, control)
- defer r.executionTree.progress.OnControlFinish(ctx)
+ r.Tree.progress.OnControlStart(ctx, control)
+ defer r.Tree.progress.OnControlFinish(ctx)
// resolve the control query
r.Lifecycle.Add("query_resolution_start")
query, err := r.resolveControlQuery(control)
if err != nil {
- r.SetError(ctx, err)
+ r.setError(ctx, err)
return
}
r.Lifecycle.Add("query_resolution_finish")
@@ -212,7 +244,7 @@ func (r *ControlRun) execute(ctx context.Context, client db_common.Client) {
log.Printf("[TRACE] setting search path %s\n", control.Name())
r.Lifecycle.Add("set_search_path_start")
if err := r.setSearchPath(ctx, dbSession, client); err != nil {
- r.SetError(ctx, err)
+ r.setError(ctx, err)
return
}
r.Lifecycle.Add("set_search_path_finish")
@@ -244,7 +276,7 @@ func (r *ControlRun) execute(ctx context.Context, client db_common.Client) {
log.Printf("[TRACE] control %s query failed again with plugin connectivity error %s - NOT retrying...", r.Control.Name(), err)
}
}
- r.SetError(ctx, err)
+ r.setError(ctx, err)
return
}
@@ -256,21 +288,6 @@ func (r *ControlRun) execute(ctx context.Context, client db_common.Client) {
log.Printf("[TRACE] finish result for, %s\n", control.Name())
}
-func (r *ControlRun) SetError(ctx context.Context, err error) {
- if err == nil {
- return
- }
- r.runError = utils.TransformErrorToSteampipe(err)
-
- // update error count
- r.Summary.Error++
- r.setRunStatus(ctx, ControlRunError)
-}
-
-func (r *ControlRun) GetError() error {
- return r.runError
-}
-
// create a context with a deadline, and with status updates disabled (we do not want to show 'loading' results)
func (r *ControlRun) getControlQueryContext(ctx context.Context) context.Context {
// create a context with a deadline
@@ -284,19 +301,8 @@ func (r *ControlRun) getControlQueryContext(ctx context.Context) context.Context
return newCtx
}
-func (r *ControlRun) GetRunStatus() ControlRunStatus {
- r.stateLock.Lock()
- defer r.stateLock.Unlock()
- return r.runStatus
-}
-
-func (r *ControlRun) Finished() bool {
- status := r.GetRunStatus()
- return status == ControlRunComplete || status == ControlRunError
-}
-
func (r *ControlRun) resolveControlQuery(control *modconfig.Control) (string, error) {
- query, err := r.executionTree.workspace.ResolveControlQuery(control, nil)
+ query, err := r.Tree.workspace.ResolveControlQuery(control, nil)
if err != nil {
return "", fmt.Errorf(`cannot run %s - failed to resolve query "%s": %s`, control.Name(), typehelpers.SafeString(control.SQL), err.Error())
}
@@ -317,7 +323,7 @@ func (r *ControlRun) waitForResults(ctx context.Context) {
select {
// check for cancellation
case <-ctx.Done():
- r.SetError(ctx, ctx.Err())
+ r.setError(ctx, ctx.Err())
case <-gatherDoneChan:
// do nothing
}
@@ -326,6 +332,17 @@ func (r *ControlRun) waitForResults(ctx context.Context) {
func (r *ControlRun) gatherResults(ctx context.Context) {
r.Lifecycle.Add("gather_start")
defer func() { r.Lifecycle.Add("gather_finish") }()
+
+ defer func() {
+ for _, row := range r.Rows {
+ for _, dim := range row.Dimensions {
+ r.DimensionKeys = append(r.DimensionKeys, dim.Key)
+ }
+ }
+ r.DimensionKeys = utils.StringSliceDistinct(r.DimensionKeys)
+ r.Group.addDimensionKeys(r.DimensionKeys...)
+ }()
+
for {
select {
case row := <-*r.queryResult.RowChan:
@@ -339,16 +356,16 @@ func (r *ControlRun) gatherResults(ctx context.Context) {
// if the row is in error then we terminate the run
if row.Error != nil {
// set error status and summary
- r.SetError(ctx, row.Error)
+ r.setError(ctx, row.Error)
// update the result group status with our status - this will be passed all the way up the execution tree
- r.group.updateSummary(r.Summary)
+ r.Group.updateSummary(r.Summary)
return
}
// so all is ok - create another result row
- result, err := NewResultRow(r.Control, row, r.queryResult.ColTypes)
+ result, err := NewResultRow(r, row, r.queryResult.ColTypes)
if err != nil {
- r.SetError(ctx, err)
+ r.setError(ctx, err)
return
}
r.addResultRow(result)
@@ -394,9 +411,9 @@ func (r *ControlRun) setRunStatus(ctx context.Context, status ControlRunStatus)
if r.Finished() {
// update Progress
if status == ControlRunError {
- r.executionTree.progress.OnControlError(ctx)
+ r.Tree.progress.OnControlError(ctx)
} else {
- r.executionTree.progress.OnControlComplete(ctx)
+ r.Tree.progress.OnControlComplete(ctx)
}
r.doneChan <- true
diff --git a/control/controlexecute/result_group.go b/control/controlexecute/result_group.go
index fb17dc0eb..c294b814b 100644
--- a/control/controlexecute/result_group.go
+++ b/control/controlexecute/result_group.go
@@ -3,6 +3,7 @@ package controlexecute
import (
"context"
"log"
+ "sort"
"sync"
"time"
@@ -31,9 +32,10 @@ type ResultGroup struct {
Severity map[string]StatusSummary `json:"-"`
// the control tree item associated with this group(i.e. a mod/benchmark)
- GroupItem modconfig.ModTreeItem `json:"-"`
- Parent *ResultGroup `json:"-"`
- Duration time.Duration `json:"-"`
+ GroupItem modconfig.ModTreeItem `json:"-"`
+ Parent *ResultGroup `json:"-"`
+ Duration time.Duration `json:"-"`
+ DimensionKeys []string `json:"-"`
// lock to prevent multiple control_runs updating this
updateLock *sync.Mutex
@@ -113,6 +115,75 @@ func NewResultGroup(ctx context.Context, executionTree *ExecutionTree, treeItem
return group
}
+func (r *ResultGroup) AllTagKeys() []string {
+ tags := []string{}
+ for k := range r.Tags {
+ tags = append(tags, k)
+ }
+ for _, child := range r.Groups {
+ tags = append(tags, child.AllTagKeys()...)
+ }
+ for _, run := range r.ControlRuns {
+ for k := range run.Control.Tags {
+ tags = append(tags, k)
+ }
+ }
+ tags = utils.StringSliceDistinct(tags)
+ sort.Strings(tags)
+ return tags
+}
+
+// GetGroupByName finds an immediate child ResultGroup with a specific name
+func (r *ResultGroup) GetGroupByName(name string) *ResultGroup {
+ for _, group := range r.Groups {
+ if group.GroupId == name {
+ return group
+ }
+ }
+ return nil
+}
+
+// GetChildGroupByName finds a nested child ResultGroup with a specific name
+func (r *ResultGroup) GetChildGroupByName(name string) *ResultGroup {
+ for _, group := range r.Groups {
+ if group.GroupId == name {
+ return group
+ }
+ if child := group.GetChildGroupByName(name); child != nil {
+ return child
+ }
+ }
+ return nil
+}
+
+// GetControlRunByName finds a child ControlRun with a specific control name
+func (r *ResultGroup) GetControlRunByName(name string) *ControlRun {
+ for _, run := range r.ControlRuns {
+ if run.Control.Name() == name {
+ return run
+ }
+ }
+ return nil
+}
+
+func (r *ResultGroup) ControlRunCount() int {
+ count := len(r.ControlRuns)
+ for _, g := range r.Groups {
+ count += g.ControlRunCount()
+ }
+ return count
+}
+
+func (r *ResultGroup) addDimensionKeys(keys ...string) {
+ r.updateLock.Lock()
+ defer r.updateLock.Unlock()
+ r.DimensionKeys = append(r.DimensionKeys, keys...)
+ if r.Parent != nil {
+ r.Parent.addDimensionKeys(keys...)
+ }
+ r.DimensionKeys = utils.StringSliceDistinct(r.DimensionKeys)
+}
+
// populateGroupMap mutates the passed in a map to return all child result groups
func (r *ResultGroup) populateGroupMap(groupMap map[string]*ResultGroup) {
if groupMap == nil {
@@ -182,7 +253,7 @@ func (r *ResultGroup) execute(ctx context.Context, client db_common.Client, para
for _, controlRun := range r.ControlRuns {
if utils.IsContextCancelled(ctx) {
- controlRun.SetError(ctx, ctx.Err())
+ controlRun.setError(ctx, ctx.Err())
continue
}
@@ -193,7 +264,7 @@ func (r *ResultGroup) execute(ctx context.Context, client db_common.Client, para
err := parallelismLock.Acquire(ctx, 1)
if err != nil {
- controlRun.SetError(ctx, err)
+ controlRun.setError(ctx, err)
continue
}
@@ -201,7 +272,7 @@ func (r *ResultGroup) execute(ctx context.Context, client db_common.Client, para
defer func() {
if r := recover(); r != nil {
// if the Execute panic'ed, set it as an error
- run.SetError(ctx, helpers.ToError(r))
+ run.setError(ctx, helpers.ToError(r))
}
// Release in defer, so that we don't retain the lock even if there's a panic inside
parallelismLock.Release(1)
@@ -213,44 +284,3 @@ func (r *ResultGroup) execute(ctx context.Context, client db_common.Client, para
child.execute(ctx, client, parallelismLock)
}
}
-
-// GetGroupByName finds an immediate child ResultGroup with a specific name
-func (r *ResultGroup) GetGroupByName(name string) *ResultGroup {
- for _, group := range r.Groups {
- if group.GroupId == name {
- return group
- }
- }
- return nil
-}
-
-// GetChildGroupByName finds a nested child ResultGroup with a specific name
-func (r *ResultGroup) GetChildGroupByName(name string) *ResultGroup {
- for _, group := range r.Groups {
- if group.GroupId == name {
- return group
- }
- if child := group.GetChildGroupByName(name); child != nil {
- return child
- }
- }
- return nil
-}
-
-// GetControlRunByName finds a child ControlRun with a specific control name
-func (r *ResultGroup) GetControlRunByName(name string) *ControlRun {
- for _, run := range r.ControlRuns {
- if run.Control.Name() == name {
- return run
- }
- }
- return nil
-}
-
-func (r *ResultGroup) ControlRunCount() int {
- count := len(r.ControlRuns)
- for _, g := range r.Groups {
- count += g.ControlRunCount()
- }
- return count
-}
diff --git a/control/controlexecute/result_row.go b/control/controlexecute/result_row.go
index ab3135591..dcd010843 100644
--- a/control/controlexecute/result_row.go
+++ b/control/controlexecute/result_row.go
@@ -19,9 +19,20 @@ type ResultRow struct {
Resource string `json:"resource" csv:"resource"`
Status string `json:"status" csv:"status"`
Dimensions []Dimension `json:"dimensions"`
+ Run *ControlRun `json:"-"`
Control *modconfig.Control `json:"-" csv:"control_id:UnqualifiedName,control_title:Title,control_description:Description"`
}
+// GetDimensionValue returns the value for a dimension key. Returns an empty string with 'false' if not found
+func (r *ResultRow) GetDimensionValue(key string) string {
+ for _, dim := range r.Dimensions {
+ if dim.Key == key {
+ return dim.Value
+ }
+ }
+ return ""
+}
+
// AddDimension checks whether a column value is a scalar type, and if so adds it to the Dimensions map
func (r *ResultRow) AddDimension(c *sql.ColumnType, val interface{}) {
switch c.ScanType().Kind() {
@@ -32,13 +43,14 @@ func (r *ResultRow) AddDimension(c *sql.ColumnType, val interface{}) {
}
}
-func NewResultRow(control *modconfig.Control, row *queryresult.RowResult, colTypes []*sql.ColumnType) (*ResultRow, error) {
+func NewResultRow(run *ControlRun, row *queryresult.RowResult, colTypes []*sql.ColumnType) (*ResultRow, error) {
// validate the required columns exist in the result
if err := validateColumns(colTypes); err != nil {
return nil, err
}
res := &ResultRow{
- Control: control,
+ Run: run,
+ Control: run.Control,
}
// was there a SQL error _executing the control
diff --git a/filepaths/steampipe.go b/filepaths/steampipe.go
index 29d3e8924..b4bf72aa5 100644
--- a/filepaths/steampipe.go
+++ b/filepaths/steampipe.go
@@ -35,7 +35,7 @@ func steampipeSubDir(dirName string) string {
// TemplateDir returns the path to the templates directory (creates if missing)
func TemplateDir() string {
- return steampipeSubDir("templates")
+ return steampipeSubDir(filepath.Join("check", "templates"))
}
// PluginDir returns the path to the plugins directory (creates if missing)
diff --git a/go.mod b/go.mod
index b2f2beb3e..2fdaee114 100644
--- a/go.mod
+++ b/go.mod
@@ -58,6 +58,14 @@ require (
)
require (
+ github.com/Masterminds/goutils v1.1.0 // indirect
+ github.com/huandu/xstrings v1.3.2 // indirect
+ github.com/mitchellh/copystructure v1.0.0 // indirect
+ github.com/mitchellh/reflectwalk v1.0.1 // indirect
+)
+
+require (
+ github.com/MasterMinds/sprig v2.22.0+incompatible
github.com/Microsoft/go-winio v0.5.0 // indirect
github.com/Microsoft/hcsshim v0.8.22 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
diff --git a/go.sum b/go.sum
index d8e4bdbd0..36a94aa17 100644
--- a/go.sum
+++ b/go.sum
@@ -80,6 +80,9 @@ github.com/ChrisTrenkamp/goxpath v0.0.0-20170922090931-c385f95c6022/go.mod h1:nu
github.com/ChrisTrenkamp/goxpath v0.0.0-20190607011252-c5096ec8773d/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4=
github.com/Machiel/slugify v1.0.1 h1:EfWSlRWstMadsgzmiV7d0yVd2IFlagWH68Q+DcYCm4E=
github.com/Machiel/slugify v1.0.1/go.mod h1:fTFGn5uWEynW4CUMG7sWkYXOf1UgDxyTM3DbR6Qfg3k=
+github.com/MasterMinds/sprig v2.22.0+incompatible h1:bu+/znjYSisHsySkFnXCL5Ropbm2ElSWkpwhwBhgwr0=
+github.com/MasterMinds/sprig v2.22.0+incompatible/go.mod h1:RVIjNhXONBqvB4TwWE/BuzAtF5cCTSpS/pcZEs6HkpU=
+github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RPBiVg=
github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
@@ -576,6 +579,7 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe
github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ=
github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/iancoleman/strcase v0.1.2 h1:gnomlvw9tnV3ITTAxzKSgTF+8kFWcU/f+TgttpXGz1U=
github.com/iancoleman/strcase v0.1.2/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE=
diff --git a/utils/string_slice.go b/utils/string_slice.go
new file mode 100644
index 000000000..8be86cd42
--- /dev/null
+++ b/utils/string_slice.go
@@ -0,0 +1,14 @@
+package utils
+
+// TODO: investigate turbot/go-kit/helpers
+func StringSliceDistinct(slice []string) []string {
+ var res []string
+ occurenceMap := make(map[string]struct{})
+ for _, item := range slice {
+ occurenceMap[item] = struct{}{}
+ }
+ for item := range occurenceMap {
+ res = append(res, item)
+ }
+ return res
+}