Files
steampipe/pkg/steampipeconfig/parse/decode.go

698 lines
23 KiB
Go

package parse
import (
"fmt"
"reflect"
"strings"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/turbot/go-kit/helpers"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig/var_config"
"github.com/turbot/steampipe/pkg/utils"
)
// A consistent detail message for all "not a valid identifier" diagnostics.
const badIdentifierDetail = "A name must start with a letter or underscore and may contain only letters, digits, underscores, and dashes."
var missingVariableErrors = []string{
// returned when the context variables does not have top level 'type' node (locals/control/etc)
"Unknown variable",
// returned when the variables have the type object but a field has not yet been populated
"Unsupported attribute",
"Missing map element",
}
func decode(runCtx *RunContext) hcl.Diagnostics {
var diags hcl.Diagnostics
// build list of blocks to decode
blocks, err := runCtx.BlocksToDecode()
// now clear dependencies from run context - they will be rebuilt
runCtx.ClearDependencies()
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "failed to determine required dependency order",
Detail: err.Error()})
}
for _, block := range blocks {
resources, res := decodeBlock(block, runCtx)
if !res.Success() {
diags = append(diags, res.Diags...)
continue
}
resourceDiags := addResourcesToMod(runCtx, resources...)
diags = append(diags, resourceDiags...)
}
return diags
}
func addResourcesToMod(runCtx *RunContext, resources ...modconfig.HclResource) hcl.Diagnostics {
var diags hcl.Diagnostics
for _, resource := range resources {
if _, ok := resource.(*modconfig.Mod); !ok {
moreDiags := runCtx.CurrentMod.AddResource(resource)
diags = append(diags, moreDiags...)
}
}
return diags
}
func decodeBlock(block *hcl.Block, runCtx *RunContext) ([]modconfig.HclResource, *decodeResult) {
var resource modconfig.HclResource
var resources []modconfig.HclResource
var res = newDecodeResult()
// if opts specifies block types, check whether this type is included
if !runCtx.ShouldIncludeBlock(block) {
return nil, res
}
// has this block already been decoded?
// (this could happen if it is a child block and has been decoded before its parent as part of second decode phase)
if resource, ok := runCtx.GetDecodedResourceForBlock(block); ok {
return []modconfig.HclResource{resource}, res
}
// check name is valid
diags := validateName(block)
if diags.HasErrors() {
res.addDiags(diags)
return nil, res
}
// now do the actual decode
if helpers.StringSliceContains(modconfig.QueryProviderBlocks, block.Type) {
resource, res = decodeQueryProvider(block, runCtx)
resources = append(resources, resource)
} else {
switch block.Type {
case modconfig.BlockTypeMod:
var mod *modconfig.Mod
// decodeMode has slightly different args as this code is shared with ParseModDefinition
mod, res = decodeMod(block, runCtx.EvalCtx, runCtx.CurrentMod)
resources = append(resources, mod)
case modconfig.BlockTypeLocals:
// special case decode logic for locals
var locals []*modconfig.Local
locals, res = decodeLocals(block, runCtx)
for _, local := range locals {
resources = append(resources, local)
}
case modconfig.BlockTypeDashboard:
resource, res = decodeDashboard(block, runCtx)
resources = append(resources, resource)
case modconfig.BlockTypeContainer:
resource, res = decodeDashboardContainer(block, runCtx)
resources = append(resources, resource)
case modconfig.BlockTypeVariable:
resource, res = decodeVariable(block, runCtx)
resources = append(resources, resource)
case modconfig.BlockTypeBenchmark:
resource, res = decodeBenchmark(block, runCtx)
resources = append(resources, resource)
default:
// all other blocks are treated the same:
resource, res = decodeResource(block, runCtx)
resources = append(resources, resource)
}
}
for _, resource := range resources {
// handle the result
// - if there are dependencies, add to run context
handleDecodeResult(resource, res, block, runCtx)
}
return resources, res
}
// generic decode function for any resource we do not have custom decode logic for
func decodeResource(block *hcl.Block, runCtx *RunContext) (modconfig.HclResource, *decodeResult) {
res := newDecodeResult()
// get shell resource
resource, diags := resourceForBlock(block, runCtx)
res.handleDecodeDiags(diags)
if diags.HasErrors() {
return nil, res
}
diags = gohcl.DecodeBody(block.Body, runCtx.EvalCtx, resource)
if len(diags) > 0 {
res.handleDecodeDiags(diags)
}
return resource, res
}
// return a shell resource for the given block
func resourceForBlock(block *hcl.Block, runCtx *RunContext) (modconfig.HclResource, hcl.Diagnostics) {
var resource modconfig.HclResource
// runCtx already contains the current mod
mod := runCtx.CurrentMod
blockName := runCtx.DetermineBlockName(block)
switch block.Type {
case modconfig.BlockTypeMod:
resource = mod
case modconfig.BlockTypeQuery:
resource = modconfig.NewQuery(block, mod, blockName)
case modconfig.BlockTypeControl:
resource = modconfig.NewControl(block, mod, blockName)
case modconfig.BlockTypeBenchmark:
resource = modconfig.NewBenchmark(block, mod, blockName)
case modconfig.BlockTypeDashboard:
resource = modconfig.NewDashboard(block, mod, blockName)
case modconfig.BlockTypeContainer:
resource = modconfig.NewDashboardContainer(block, mod, blockName)
case modconfig.BlockTypeChart:
resource = modconfig.NewDashboardChart(block, mod, blockName)
case modconfig.BlockTypeCard:
resource = modconfig.NewDashboardCard(block, mod, blockName)
case modconfig.BlockTypeFlow:
resource = modconfig.NewDashboardFlow(block, mod, blockName)
case modconfig.BlockTypeGraph:
resource = modconfig.NewDashboardGraph(block, mod, blockName)
case modconfig.BlockTypeHierarchy:
resource = modconfig.NewDashboardHierarchy(block, mod, blockName)
case modconfig.BlockTypeImage:
resource = modconfig.NewDashboardImage(block, mod, blockName)
case modconfig.BlockTypeInput:
resource = modconfig.NewDashboardInput(block, mod, blockName)
case modconfig.BlockTypeTable:
resource = modconfig.NewDashboardTable(block, mod, blockName)
case modconfig.BlockTypeText:
resource = modconfig.NewDashboardText(block, mod, blockName)
default:
return nil, hcl.Diagnostics{&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("resourceForBlock called for unsupported block type %s", block.Type),
Subject: &block.DefRange,
},
}
}
return resource, nil
}
func decodeLocals(block *hcl.Block, runCtx *RunContext) ([]*modconfig.Local, *decodeResult) {
res := newDecodeResult()
attrs, diags := block.Body.JustAttributes()
if len(attrs) == 0 {
res.Diags = diags
return nil, res
}
// build list of locals
locals := make([]*modconfig.Local, 0, len(attrs))
for name, attr := range attrs {
if !hclsyntax.ValidIdentifier(name) {
res.Diags = append(res.Diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid local value name",
Detail: badIdentifierDetail,
Subject: &attr.NameRange,
})
continue
}
// try to evaluate expression
val, diags := attr.Expr.Value(runCtx.EvalCtx)
// handle any resulting diags, which may specify dependencies
res.handleDecodeDiags(diags)
// add to our list
locals = append(locals, modconfig.NewLocal(name, val, attr.Range, runCtx.CurrentMod))
}
return locals, res
}
func decodeVariable(block *hcl.Block, runCtx *RunContext) (*modconfig.Variable, *decodeResult) {
res := newDecodeResult()
var variable *modconfig.Variable
content, diags := block.Body.Content(VariableBlockSchema)
res.handleDecodeDiags(diags)
v, diags := var_config.DecodeVariableBlock(block, content, false)
res.handleDecodeDiags(diags)
if res.Success() {
variable = modconfig.NewVariable(v, runCtx.CurrentMod)
}
return variable, res
}
func decodeParam(block *hcl.Block, runCtx *RunContext, parentName string) (*modconfig.ParamDef, hcl.Diagnostics) {
def := modconfig.NewParamDef(block)
content, diags := block.Body.Content(ParamDefBlockSchema)
if attr, exists := content.Attributes["description"]; exists {
moreDiags := gohcl.DecodeExpression(attr.Expr, runCtx.EvalCtx, &def.Description)
diags = append(diags, moreDiags...)
}
if attr, exists := content.Attributes["default"]; exists {
v, moreDiags := attr.Expr.Value(runCtx.EvalCtx)
diags = append(diags, moreDiags...)
if !moreDiags.HasErrors() {
// convert the raw default into a postgres representation
if valStr, err := utils.CtyToPostgresString(v); err == nil {
def.Default = utils.ToStringPointer(valStr)
} else {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("%s has invalid parameter config", parentName),
Detail: err.Error(),
Subject: &attr.Range,
})
}
}
}
return def, diags
}
func decodeQueryProvider(block *hcl.Block, runCtx *RunContext) (modconfig.HclResource, *decodeResult) {
res := newDecodeResult()
// get shell resource
resource, diags := resourceForBlock(block, runCtx)
res.handleDecodeDiags(diags)
if diags.HasErrors() {
return nil, res
}
// do a partial decode using QueryProviderBlockSchema
// this will be used to pull out attributes which need manual decoding
content, remain, diags := block.Body.PartialContent(QueryProviderBlockSchema)
res.handleDecodeDiags(diags)
if !res.Success() {
return nil, res
}
// handle invalid block types
res.addDiags(validateBlocks(remain.(*hclsyntax.Body), QueryProviderBlockSchema, resource))
// decode the body into 'resource' to populate all properties that can be automatically decoded
diags = gohcl.DecodeBody(remain, runCtx.EvalCtx, resource)
// handle any resulting diags, which may specify dependencies
res.handleDecodeDiags(diags)
// cast resource to a QueryProvider
queryProvider, ok := resource.(modconfig.QueryProvider)
if !ok {
// coding error
panic(fmt.Sprintf("block type %s not convertible to a QueryProvider", block.Type))
}
sqlAttr, sqlPropertySet := content.Attributes["sql"]
_, queryPropertySet := content.Attributes["query"]
if sqlPropertySet && queryPropertySet {
// either Query or SQL property may be set - if Query property already set, error
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("%s has both 'SQL' and 'query' property set - only 1 of these may be set", resource.Name()),
Subject: &sqlAttr.Range,
})
res.addDiags(diags)
}
if attr, exists := content.Attributes["args"]; exists {
args, runtimeDependencies, diags := decodeArgs(attr, runCtx.EvalCtx, queryProvider)
if diags.HasErrors() {
// handle dependencies
res.handleDecodeDiags(diags)
} else {
queryProvider.SetArgs(args)
queryProvider.AddRuntimeDependencies(runtimeDependencies)
}
}
var params []*modconfig.ParamDef
for _, block := range content.Blocks {
// only paramdefs are defined in the schema
if block.Type != modconfig.BlockTypeParam {
panic(fmt.Sprintf("invalid child block type %s", block.Type))
}
// param block cannot be set if a query property is set - it is only valid if inline SQL ids defined
if queryPropertySet {
diags = append(diags, invalidParamDiags(resource, block))
}
paramDef, moreDiags := decodeParam(block, runCtx, resource.Name())
if !moreDiags.HasErrors() {
params = append(params, paramDef)
// add and references contained in the param block to the control refs
moreDiags = AddReferences(resource, block, runCtx)
}
diags = append(diags, moreDiags...)
}
queryProvider.SetParams(params)
// handle any resulting diags, which may specify dependencies
res.handleDecodeDiags(diags)
return resource, res
}
func invalidParamDiags(resource modconfig.HclResource, block *hcl.Block) *hcl.Diagnostic {
return &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("%s has 'query' property set so cannot define param blocks", resource.Name()),
Subject: &block.DefRange,
}
}
func decodeDashboard(block *hcl.Block, runCtx *RunContext) (*modconfig.Dashboard, *decodeResult) {
res := newDecodeResult()
dashboard := modconfig.NewDashboard(block, runCtx.CurrentMod, runCtx.DetermineBlockName(block))
// do a partial decode using an empty schema - use to pull out all body content in the remain block
_, remain, diags := block.Body.PartialContent(&hcl.BodySchema{})
res.handleDecodeDiags(diags)
// handle invalid block types
res.addDiags(validateBlocks(remain.(*hclsyntax.Body), DashboardBlockSchema, dashboard))
if !res.Success() {
return nil, res
}
// decode the body into 'dashboardContainer' to populate all properties that can be automatically decoded
diags = gohcl.DecodeBody(remain, runCtx.EvalCtx, dashboard)
// handle any resulting diags, which may specify dependencies
res.handleDecodeDiags(diags)
if dashboard.Base != nil && len(dashboard.Base.ChildNames) > 0 {
supportedChildren := []string{modconfig.BlockTypeContainer, modconfig.BlockTypeChart, modconfig.BlockTypeControl, modconfig.BlockTypeCard, modconfig.BlockTypeFlow, modconfig.BlockTypeGraph, modconfig.BlockTypeHierarchy, modconfig.BlockTypeImage, modconfig.BlockTypeInput, modconfig.BlockTypeTable, modconfig.BlockTypeText}
// TODO: we should be passing in the block for the Base resource - but this is only used for diags
// and we do not expect to get any (as this function has already succeeded when the base was originally parsed)
children, _ := resolveChildrenFromNames(dashboard.Base.ChildNames, block, supportedChildren, runCtx)
dashboard.Base.SetChildren(children)
}
if !res.Success() {
return dashboard, res
}
// now decode child blocks
body := remain.(*hclsyntax.Body)
if len(body.Blocks) > 0 {
blocksRes := decodeDashboardBlocks(body, dashboard, runCtx)
res.Merge(blocksRes)
}
return dashboard, res
}
func decodeDashboardBlocks(content *hclsyntax.Body, dashboard *modconfig.Dashboard, runCtx *RunContext) *decodeResult {
var res = newDecodeResult()
var inputs []*modconfig.DashboardInput
// set dashboard as parent on the run context - this is used when generating names for anonymous blocks
runCtx.PushParent(dashboard)
defer func() {
runCtx.PopParent()
}()
for _, b := range content.Blocks {
// decode block
resources, blockRes := decodeBlock(b.AsHCLBlock(), runCtx)
res.Merge(blockRes)
if !blockRes.Success() {
continue
}
// we expect either inputs or child report nodes
for _, resource := range resources {
if b.Type == modconfig.BlockTypeInput {
input := resource.(*modconfig.DashboardInput)
inputs = append(inputs, input)
dashboard.AddChild(input)
// inputs get added to the mod in SetInputs
} else {
// add the resource to the mod
res.addDiags(addResourcesToMod(runCtx, resource))
// add to the dashboard children
// (we expect this cast to always succeed)
if child, ok := resource.(modconfig.ModTreeItem); ok {
dashboard.AddChild(child)
}
}
}
}
moreDiags := dashboard.SetInputs(inputs)
res.addDiags(moreDiags)
return res
}
func decodeDashboardContainer(block *hcl.Block, runCtx *RunContext) (*modconfig.DashboardContainer, *decodeResult) {
res := newDecodeResult()
container := modconfig.NewDashboardContainer(block, runCtx.CurrentMod, runCtx.DetermineBlockName(block))
// do a partial decode using an empty schema - use to pull out all body content in the remain block
_, remain, diags := block.Body.PartialContent(&hcl.BodySchema{})
res.handleDecodeDiags(diags)
if !res.Success() {
return nil, res
}
// handle invalid block types
res.addDiags(validateBlocks(remain.(*hclsyntax.Body), DashboardContainerBlockSchema, container))
// decode the body into 'dashboardContainer' to populate all properties that can be automatically decoded
diags = gohcl.DecodeBody(remain, runCtx.EvalCtx, container)
// handle any resulting diags, which may specify dependencies
res.handleDecodeDiags(diags)
// now decode child blocks
body := remain.(*hclsyntax.Body)
if len(body.Blocks) > 0 {
blocksRes := decodeDashboardContainerBlocks(body, container, runCtx)
res.Merge(blocksRes)
}
return container, res
}
func decodeDashboardContainerBlocks(content *hclsyntax.Body, dashboardContainer *modconfig.DashboardContainer, runCtx *RunContext) *decodeResult {
var res = newDecodeResult()
// set container as parent on the run context - this is used when generating names for anonymous blocks
runCtx.PushParent(dashboardContainer)
defer func() {
runCtx.PopParent()
}()
for _, b := range content.Blocks {
resources, blockRes := decodeBlock(b.AsHCLBlock(), runCtx)
res.Merge(blockRes)
if !blockRes.Success() {
continue
}
for _, resource := range resources {
// special handling for inputs
if b.Type == modconfig.BlockTypeInput {
input := resource.(*modconfig.DashboardInput)
dashboardContainer.Inputs = append(dashboardContainer.Inputs, input)
dashboardContainer.AddChild(input)
// the input will be added to the mod by the parent dashboard
} else {
// for all other children, add to mod and children
res.addDiags(addResourcesToMod(runCtx, resource))
if child, ok := resource.(modconfig.ModTreeItem); ok {
dashboardContainer.AddChild(child)
}
}
}
}
return res
}
func decodeBenchmark(block *hcl.Block, runCtx *RunContext) (*modconfig.Benchmark, *decodeResult) {
res := newDecodeResult()
benchmark := modconfig.NewBenchmark(block, runCtx.CurrentMod, runCtx.DetermineBlockName(block))
content, diags := block.Body.Content(BenchmarkBlockSchema)
res.handleDecodeDiags(diags)
diags = decodeProperty(content, "children", &benchmark.ChildNames, runCtx)
res.handleDecodeDiags(diags)
diags = decodeProperty(content, "description", &benchmark.Description, runCtx)
res.handleDecodeDiags(diags)
diags = decodeProperty(content, "documentation", &benchmark.Documentation, runCtx)
res.handleDecodeDiags(diags)
diags = decodeProperty(content, "tags", &benchmark.Tags, runCtx)
res.handleDecodeDiags(diags)
diags = decodeProperty(content, "title", &benchmark.Title, runCtx)
res.handleDecodeDiags(diags)
diags = decodeProperty(content, "type", &benchmark.Type, runCtx)
res.handleDecodeDiags(diags)
diags = decodeProperty(content, "display", &benchmark.Display, runCtx)
res.handleDecodeDiags(diags)
// now add children
if res.Success() {
supportedChildren := []string{modconfig.BlockTypeBenchmark, modconfig.BlockTypeControl}
children, diags := resolveChildrenFromNames(benchmark.ChildNames.StringList(), block, supportedChildren, runCtx)
res.handleDecodeDiags(diags)
// now set children and child name strings
benchmark.Children = children
benchmark.ChildNameStrings = getChildNameStringsFromModTreeItem(children)
}
// decode report specific properties
diags = decodeProperty(content, "base", &benchmark.Base, runCtx)
res.handleDecodeDiags(diags)
if benchmark.Base != nil && len(benchmark.Base.ChildNames) > 0 {
supportedChildren := []string{modconfig.BlockTypeBenchmark, modconfig.BlockTypeControl}
// TODO: we should be passing in the block for the Base resource - but this is only used for diags
// and we do not expect to get any (as this function has already succeeded when the base was originally parsed)
children, _ := resolveChildrenFromNames(benchmark.Base.ChildNameStrings, block, supportedChildren, runCtx)
benchmark.Base.Children = children
}
diags = decodeProperty(content, "width", &benchmark.Width, runCtx)
res.handleDecodeDiags(diags)
return benchmark, res
}
func decodeProperty(content *hcl.BodyContent, property string, dest interface{}, runCtx *RunContext) hcl.Diagnostics {
var diags hcl.Diagnostics
if title, ok := content.Attributes[property]; ok {
diags = gohcl.DecodeExpression(title.Expr, runCtx.EvalCtx, dest)
}
return diags
}
// handleDecodeResult
// if decode was successful:
// - generate and set resource metadata
// - add resource to RunContext (which adds it to the mod)handleDecodeResult
func handleDecodeResult(resource modconfig.HclResource, res *decodeResult, block *hcl.Block, runCtx *RunContext) {
if res.Success() {
anonymousResource := resourceIsAnonymous(resource)
// call post decode hook
// NOTE: must do this BEFORE adding resource to run context to ensure we respect the base property
moreDiags := resource.OnDecoded(block, runCtx)
res.addDiags(moreDiags)
// add references
moreDiags = AddReferences(resource, block, runCtx)
res.addDiags(moreDiags)
// if resource is NOT anonymous, add into the run context
if !anonymousResource {
moreDiags = runCtx.AddResource(resource)
res.addDiags(moreDiags)
}
// if resource supports metadata, save it
if resourceWithMetadata, ok := resource.(modconfig.ResourceWithMetadata); ok {
body := block.Body.(*hclsyntax.Body)
moreDiags = addResourceMetadata(resourceWithMetadata, body.SrcRange, runCtx)
res.addDiags(moreDiags)
}
} else {
if len(res.Depends) > 0 {
moreDiags := runCtx.AddDependencies(block, resource.GetUnqualifiedName(), res.Depends)
res.addDiags(moreDiags)
}
}
}
func resourceIsAnonymous(resource modconfig.HclResource) bool {
// (if a resource anonymous it must support ResourceWithMetadata)
resourceWithMetadata, ok := resource.(modconfig.ResourceWithMetadata)
anonymousResource := ok && resourceWithMetadata.IsAnonymous()
return anonymousResource
}
func addResourceMetadata(resourceWithMetadata modconfig.ResourceWithMetadata, srcRange hcl.Range, runCtx *RunContext) hcl.Diagnostics {
metadata, err := GetMetadataForParsedResource(resourceWithMetadata.Name(), srcRange, runCtx.FileData, runCtx.CurrentMod)
if err != nil {
return hcl.Diagnostics{&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: err.Error(),
Subject: &srcRange,
}}
}
// set on resource
resourceWithMetadata.SetMetadata(metadata)
return nil
}
func validateName(block *hcl.Block) hcl.Diagnostics {
if len(block.Labels) == 0 {
return nil
}
if !hclsyntax.ValidIdentifier(block.Labels[0]) {
return hcl.Diagnostics{&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid name",
Detail: badIdentifierDetail,
Subject: &block.LabelRanges[0],
}}
}
return nil
}
// Validate all blocks are supported
// We use partial decoding so that we can automatically decode as many properties as possible
// and only manually decode properties requiring special logic.
// The problem is the partial decode does not return errors for invalid blocks, so we must implement our own
func validateBlocks(body *hclsyntax.Body, schema *hcl.BodySchema, resource modconfig.HclResource) hcl.Diagnostics {
var diags hcl.Diagnostics
// identify any blocks specified by hcl tags
var supportedBlocks []string
v := reflect.TypeOf(helpers.DereferencePointer(resource))
for i := 0; i < v.NumField(); i++ {
tag := v.FieldByIndex([]int{i}).Tag.Get("hcl")
if idx := strings.LastIndex(tag, ",block"); idx != -1 {
supportedBlocks = append(supportedBlocks, tag[:idx])
}
}
// ad din blocks specified in the schema
for _, b := range schema.Blocks {
supportedBlocks = append(supportedBlocks, b.Type)
}
// now check for invalid blocks
for _, block := range body.Blocks {
if !helpers.StringSliceContains(supportedBlocks, block.Type) {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf(`Unsupported block type: Blocks of type "%s" are not expected here.`, block.Type),
Subject: &block.TypeRange,
})
}
}
return diags
}