jprdonnelly/steampipe
1
0
Fork 0
mirror of https://github.com/turbot/steampipe.git synced 2026-04-05 16:00:06 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update mod parsing to support controls. update mods and queries to use string pointers for properties to allow optional hcl properties. Reenable mod loading and workspace exclusions. Closes #410

Browse Source
This commit is contained in:
kai
2021-04-19 15:59:32 +01:00
parent 4b7bc7eac4
commit 7f80f0f69f
22 changed files with 374 additions and 289 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
steampipeconfig/load_config_test.go
View File

@@ -66,7 +66,6 @@ secret_key = "aws_dmi_002_secret_key"`,
steampipeDir: "test_data/connection_config/single_connection",
expected: &SteampipeConfig{
Connections: map[string]*Connection{
// todo normalise plugin names here?
"a": {
Name: "a",
Plugin: "hub.steampipe.io/plugins/test_data/connection-test-1@latest",
@@ -82,7 +81,6 @@ secret_key = "aws_dmi_002_secret_key"`,
steampipeDir: "test_data/connection_config/single_connection_with_default_options",
expected: &SteampipeConfig{
Connections: map[string]*Connection{
// todo normalise plugin names here?
"a": {
Name: "a",
Plugin: "hub.steampipe.io/plugins/test_data/connection-test-1@latest",
@@ -120,7 +118,6 @@ secret_key = "aws_dmi_002_secret_key"`,
workspaceDir: "test_data/workspaces/search_path_prefix",
expected: &SteampipeConfig{
Connections: map[string]*Connection{
// todo normalise plugin names here?
"a": {
Name: "a",
Plugin: "hub.steampipe.io/plugins/test_data/connection-test-1@latest",
@@ -154,7 +151,6 @@ secret_key = "aws_dmi_002_secret_key"`,
workspaceDir: "test_data/workspaces/override_terminal_config",
expected: &SteampipeConfig{
Connections: map[string]*Connection{
// todo normalise plugin names here?
"a": {
Name: "a",
Plugin: "hub.steampipe.io/plugins/test_data/connection-test-1@latest",

142
steampipeconfig/load_mod.go
View File

@@ -46,80 +46,48 @@ func LoadMod(modPath string, opts *LoadModOptions) (mod *modconfig.Mod, err erro
}
}()
var parseResult = newModParseResult()
// NOTE: for now sp file loading is disabled
enableModLoading := false
if enableModLoading {
if opts == nil {
opts = &LoadModOptions{}
}
// verify the mod folder exists
if _, err := os.Stat(modPath); os.IsNotExist(err) {
return nil, fmt.Errorf("mod folder %s does not exist", modPath)
}
// build list of all filepaths we need to parse/load
// NOTE: pseudo resource creation is handled separately below
var include = filehelpers.InclusionsFromExtensions([]string{constants.ModDataExtension})
sourcePaths, err := getSourcePaths(modPath, include, opts.Exclude)
if err != nil {
log.Printf("[WARN] LoadMod: failed to get mod file paths: %v\n", err)
return nil, err
}
// load the raw data
fileData, diags := loadFileData(sourcePaths)
if diags.HasErrors() {
log.Printf("[WARN] LoadMod: failed to load all mod files: %v\n", err)
return nil, plugin.DiagsToError("Failed to load all mod files", diags)
}
// parse all hcl files
parseResult, err = parseModHcl(modPath, fileData)
if err != nil {
return nil, err
}
if opts == nil {
opts = &LoadModOptions{}
}
// verify the mod folder exists
if _, err := os.Stat(modPath); os.IsNotExist(err) {
return nil, fmt.Errorf("mod folder %s does not exist", modPath)
}
// is there a mod resource definition?
if parseResult.mod != nil {
mod = parseResult.mod
} else {
// so there is no mod resource definition - check flag for our response
if !opts.CreateDefaultMod() {
// CreateDefaultMod flag NOT set - fail
return nil, fmt.Errorf("mod folder %s does not contain a mod resource definition", modPath)
}
// just create a default mod
mod = defaultWorkspaceMod()
// build list of all filepaths we need to parse/load
// NOTE: pseudo resource creation is handled separately below
var include = filehelpers.InclusionsFromExtensions([]string{constants.ModDataExtension})
sourcePaths, err := getSourcePaths(modPath, include, opts.Exclude)
if err != nil {
log.Printf("[WARN] LoadMod: failed to get mod file paths: %v\n", err)
return nil, err
}
// load the raw data
fileData, diags := loadFileData(sourcePaths)
if diags.HasErrors() {
log.Printf("[WARN] LoadMod: failed to load all mod files: %v\n", err)
return nil, plugin.DiagsToError("Failed to load all mod files", diags)
}
// parse all hcl files.
mod, err = parseModHcl(modPath, fileData, opts)
if err != nil {
return nil, err
}
// if flag is set, create pseudo resources by mapping files
if opts.CreatePseudoResources() {
// now execute any pseudo-resource creations based on file mappings
err = createPseudoResources(modPath, parseResult, opts)
err = createPseudoResources(modPath, mod, opts)
if err != nil {
return nil, err
}
}
// now convert query map into an array and set on the mod object
mod.PopulateQueries(parseResult.queryMap)
return
}
type modParseResult struct {
queryMap map[string]*modconfig.Query
mod *modconfig.Mod
}
func newModParseResult() *modParseResult {
return &modParseResult{
queryMap: make(map[string]*modconfig.Query),
}
}
// GetModFileExtensions :: return list of all file extensions we care about
// this will be the mod data extension, plus any registered extensions registered in fileToResourceMap
func GetModFileExtensions() []string {
@@ -149,7 +117,7 @@ func getSourcePaths(modPath string, include, exclude []string) ([]string, error)
}
// parse all source hcl files for the mod and associated resources
func parseModHcl(modPath string, fileData map[string][]byte) (*modParseResult, error) {
func parseModHcl(modPath string, fileData map[string][]byte, opts *LoadModOptions) (*modconfig.Mod, error) {
var mod *modconfig.Mod
body, diags := parseHclFiles(fileData)
@@ -164,6 +132,7 @@ func parseModHcl(modPath string, fileData map[string][]byte) (*modParseResult, e
}
var queries = make(map[string]*modconfig.Query)
var controls = make(map[string]*modconfig.Control)
for _, block := range content.Blocks {
switch block.Type {
case "variable":
@@ -184,15 +153,31 @@ func parseModHcl(modPath string, fileData map[string][]byte) (*modParseResult, e
diags = append(diags, moreDiags...)
break
}
if _, ok := queries[query.Name]; ok {
if _, ok := queries[*query.Name]; ok {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("mod defines more that one query named %s", query.Name),
Summary: fmt.Sprintf("mod defines more that one query named %s", *query.Name),
Subject: &block.DefRange,
})
continue
}
queries[query.Name] = query
queries[*query.Name] = query
case "control":
control, moreDiags := parseControl(block)
if moreDiags.HasErrors() {
diags = append(diags, moreDiags...)
break
}
if _, ok := controls[*control.Name]; ok {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("mod defines more that one control named %s", *control.Name),
Subject: &block.DefRange,
})
continue
}
controls[*control.Name] = control
}
}
@@ -200,12 +185,26 @@ func parseModHcl(modPath string, fileData map[string][]byte) (*modParseResult, e
return nil, plugin.DiagsToError("Failed to parse all mod hcl files", diags)
}
return &modParseResult{queries, mod}, nil
// is there a mod resource definition?
if mod == nil {
// should we creaste a default mod?
if !opts.CreateDefaultMod() {
// CreateDefaultMod flag NOT set - fail
return nil, fmt.Errorf("mod folder %s does not contain a mod resource definition", modPath)
}
// just create a default mod
mod = defaultWorkspaceMod()
}
mod.Queries = queries
mod.Controls = controls
return mod, nil
}
// create pseudo-resources for any files whose extensions are registered
// NOTE: this mutates parseResults
func createPseudoResources(modPath string, parseResults *modParseResult, opts *LoadModOptions) error {
func createPseudoResources(modPath string, mod *modconfig.Mod, opts *LoadModOptions) error {
// list all registered files
var include = filehelpers.InclusionsFromExtensions(modconfig.RegisteredFileExtensions())
sourcePaths, err := getSourcePaths(modPath, include, opts.Exclude)
@@ -228,7 +227,7 @@ func createPseudoResources(modPath string, parseResults *modParseResult, opts *L
continue
}
if resource != nil {
if err := addResourceIfUnique(resource, parseResults, path); err != nil {
if err := addResourceIfUnique(resource, mod, path); err != nil {
errors = append(errors, err)
}
}
@@ -245,19 +244,20 @@ func createPseudoResources(modPath string, parseResults *modParseResult, opts *L
}
// add resource to parse results, if there is no resource of same name
func addResourceIfUnique(resource modconfig.MappableResource, parseResults *modParseResult, path string) error {
func addResourceIfUnique(resource modconfig.MappableResource, mod *modconfig.Mod, path string) error {
switch r := resource.(type) {
case *modconfig.Query:
// check there is not already a query with the same name
if _, ok := parseResults.queryMap[r.Name]; ok {
if _, ok := mod.Queries[*r.Name]; ok {
// we have already created a query with this name - skip!
return fmt.Errorf("not creating resource for '%s' as there is already a query '%s' defined", path, r.Name)
return fmt.Errorf("not creating resource for '%s' as there is already a query '%s' defined", path, *r.Name)
}
parseResults.queryMap[r.Name] = r
mod.Queries[*r.Name] = r
}
return nil
}
func defaultWorkspaceMod() *modconfig.Mod {
return &modconfig.Mod{Name: "local"}
defaultName := "local"
return &modconfig.Mod{Name: &defaultName}
}

325
steampipeconfig/load_mod_test.go
View File

@@ -16,179 +16,179 @@ type loadModTest struct {
expected interface{}
}
var alias = "_m2"
var loadWorkspaceOptions = &LoadModOptions{
Exclude: []string{fmt.Sprintf("**/%s*", constants.WorkspaceDataDir)},
Flags: CreatePseudoResources | CreateDefaultMod,
}
var testCasesLoadMod = map[string]loadModTest{
var testCasesLoadMod map[string]loadModTest = map[string]loadModTest{
"no_mod_sql_files": {
source: "test_data/mods/no_mod_sql_files",
expected: &modconfig.Mod{
Name: "local",
Queries: []*modconfig.Query{
{
Name: "q1", SQL: "select 1",
Name: toStringPointer("local"),
Queries: map[string]*modconfig.Query{
"q1": {
Name: toStringPointer("q1"), SQL: toStringPointer("select 1"),
},
{
Name: "q2", SQL: "select 2",
"q2": {
Name: toStringPointer("q2"), SQL: toStringPointer("select 2"),
},
}},
},
// commented out as we do not support SP file loading yet
"no_mod_hcl_queries": {
source: "test_data/mods/no_mod_hcl_queries",
expected: &modconfig.Mod{
Name: toStringPointer("local"),
Queries: map[string]*modconfig.Query{
"q1": {
toStringPointer("q1"), toStringPointer("Q1"), toStringPointer("THIS IS QUERY 1"), toStringPointer("select 1"),
},
"q2": {
toStringPointer("q2"), toStringPointer("Q2"), toStringPointer("THIS IS QUERY 2"), toStringPointer("select 2"),
},
},
},
},
"single_mod_duplicate_query": {
source: "test_data/mods/single_mod_duplicate_query",
expected: "ERROR",
},
"single_mod_no_query": {
source: "test_data/mods/single_mod_no_query",
expected: &modconfig.Mod{
Name: toStringPointer("m1"),
Title: toStringPointer("M1"),
Description: toStringPointer("THIS IS M1"),
ModDepends: []*modconfig.ModVersion{
{"github.com/turbot/m2", "0.0.0", toStringPointer("_m1")},
},
},
},
"single_mod_one_query": {
source: "test_data/mods/single_mod_one_query",
expected: &modconfig.Mod{
Name: toStringPointer("m1"),
Title: toStringPointer("M1"),
Description: toStringPointer("THIS IS M1"),
ModDepends: []*modconfig.ModVersion{
{"github.com/turbot/m2", "0.0.0", toStringPointer("_m1")},
},
Queries: map[string]*modconfig.Query{
"q1": {
Name: toStringPointer("q1"), Title: toStringPointer("Q1"), Description: toStringPointer("THIS IS QUERY 1"), SQL: toStringPointer("select 1"),
},
},
},
},
"single_mod_one_query_one_control": {
source: "test_data/mods/single_mod_one_query_one_control",
expected: &modconfig.Mod{
Name: toStringPointer("m1"),
Title: toStringPointer("M1"),
Description: toStringPointer("THIS IS M1"),
ModDepends: []*modconfig.ModVersion{
{"github.com/turbot/m2", "0.0.0", toStringPointer("_m1")},
},
Queries: map[string]*modconfig.Query{
"q1": {
//"no_mod_hcl_queries": {
// source: "test_data/mods/no_mod_hcl_queries",
// expected: &modconfig.Mod{
// Name: "local",
// Queries: []*modconfig.Query{
// {
// "q1", "Q1", "THIS IS QUERY 1", "select 1",
// },
// {
// "q2", "Q2", "THIS IS QUERY 2", "select 2",
// },
// },
// },
//},
//"single_mod_duplicate_query": {
// source: "test_data/mods/single_mod_duplicate_query",
// expected: "ERROR",
//},
//"single_mod_nested_sql_files": {
// source: "test_data/mods/single_mod_nested_sql_files",
// expected: &modconfig.Mod{
// Name: "m1",
// Title: "M1",
// Description: "THIS IS M1",
// Queries: []*modconfig.Query{
// {
// Name: "queries_a_aa_q1", SQL: "select 1",
// },
// {
// Name: "queries_a_q1", SQL: "select 1",
// },
// {
// Name: "queries_b_bb_q2", SQL: "select 2",
// },
// {
// Name: "queries_b_q2", SQL: "select 2",
// },
// },
// },
//},
//"single_mod_no_query": {
// source: "test_data/mods/single_mod_no_query",
// expected: &modconfig.Mod{
// Name: "m1",
// Title: "M1",
// Description: "THIS IS M1",
// ModDepends: []*modconfig.ModVersion{
// {"github.com/turbot/m2", "0.0.0", &alias},
// },
// },
//},
//"single_mod_one_query": {
// source: "test_data/mods/single_mod_one_query",
// expected: &modconfig.Mod{
// Name: "m1",
// Title: "M1",
// Description: "THIS IS M1",
// ModDepends: []*modconfig.ModVersion{
// {"github.com/turbot/m2", "0.0.0", &alias},
// },
// Queries: []*modconfig.Query{
// {
// "q1", "Q1", "THIS IS QUERY 1", "select 1",
// },
// },
// },
//},
//"single_mod_one_sql_file": {
// source: "test_data/mods/single_mod_one_sql_file",
// expected: &modconfig.Mod{
// Name: "m1",
// Title: "M1",
// Description: "THIS IS M1",
// Queries: []*modconfig.Query{{Name: "q1", SQL: "select 1"}},
// },
//},
//"single_mod_sql_file_and_hcl_query": {
// source: "test_data/mods/single_mod_sql_file_and_hcl_query",
// expected: &modconfig.Mod{
// Name: "m1",
// Title: "M1",
// Description: "THIS IS M1",
// Queries: []*modconfig.Query{
// {
// "q1", "Q1", "THIS IS QUERY 1", "select 1",
// },
// {
// Name: "q2", SQL: "select 2",
// },
// },
// },
//},
//"single_mod_two_queries_diff_files": {
// source: "test_data/mods/single_mod_two_queries_diff_files",
// expected: &modconfig.Mod{
// Name: "m1",
// Title: "M1",
// Description: "THIS IS M1",
// ModDepends: []*modconfig.ModVersion{
// {"github.com/turbot/m2", "0.0.0", &alias},
// },
// Queries: []*modconfig.Query{
// {
// "q1", "Q1", "THIS IS QUERY 1", "select 1",
// },
// {
// "q2", "Q2", "THIS IS QUERY 2", "select 2",
// },
// },
// },
//},
//"single_mod_two_queries_same_file": {
// source: "test_data/mods/single_mod_two_queries_same_file",
// expected: &modconfig.Mod{
// Name: "m1",
// Title: "M1",
// Description: "THIS IS M1",
// ModDepends: []*modconfig.ModVersion{
// {"github.com/turbot/m2", "0.0.0", &alias},
// },
// Queries: []*modconfig.Query{
// {
// "q1", "Q1", "THIS IS QUERY 1", "select 1",
// },
// {
// "q2", "Q2", "THIS IS QUERY 2", "select 2",
// },
// },
// },
//},
//"single_mod_two_sql_files": {
// source: "test_data/mods/single_mod_two_sql_files",
// expected: &modconfig.Mod{
// Name: "m1",
// Title: "M1",
// Description: "THIS IS M1",
// Queries: []*modconfig.Query{
// {
// Name: "q1", SQL: "select 1",
// },
// {
// Name: "q2", SQL: "select 2",
// },
// },
// },
//},
//"two_mods": {
// source: "test_data/mods/two_mods",
// expected: "ERROR",
//},
Name: toStringPointer("q1"), Title: toStringPointer("Q1"), Description: toStringPointer("THIS IS QUERY 1"), SQL: toStringPointer("select 1"),
},
},
Controls: map[string]*modconfig.Control{
"c1": {
Name: toStringPointer("c1"), Title: toStringPointer("C1"), Description: toStringPointer("THIS IS CONTROL 1"), SQL: toStringPointer("select 1"), Tags: map[string]string{
"Application": "demo",
"EnvironmentType": "prod",
"ProductType": "steampipe",
},
},
},
},
},
"single_mod_one_sql_file": {
source: "test_data/mods/single_mod_one_sql_file",
expected: &modconfig.Mod{
Name: toStringPointer("m1"),
Title: toStringPointer("M1"),
Description: toStringPointer("THIS IS M1"),
Queries: map[string]*modconfig.Query{"q1": {Name: toStringPointer("q1"), SQL: toStringPointer("select 1")}},
},
},
"single_mod_sql_file_and_hcl_query": {
source: "test_data/mods/single_mod_sql_file_and_hcl_query",
expected: &modconfig.Mod{
Name: toStringPointer("m1"),
Title: toStringPointer("M1"),
Description: toStringPointer("THIS IS M1"),
Queries: map[string]*modconfig.Query{
"q1": {
Name: toStringPointer("q1"), Title: toStringPointer("Q1"), Description: toStringPointer("THIS IS QUERY 1"), SQL: toStringPointer("select 1"),
},
"q2": {
Name: toStringPointer("q2"), SQL: toStringPointer("select 2"),
},
},
},
},
"single_mod_two_queries_diff_files": {
source: "test_data/mods/single_mod_two_queries_diff_files",
expected: &modconfig.Mod{
Name: toStringPointer("m1"),
Title: toStringPointer("M1"),
Description: toStringPointer("THIS IS M1"),
ModDepends: []*modconfig.ModVersion{
{"github.com/turbot/m2", "0.0.0", toStringPointer("_m1")},
},
Queries: map[string]*modconfig.Query{
"q1": {
Name: toStringPointer("q1"), Title: toStringPointer("Q1"), Description: toStringPointer("THIS IS QUERY 1"), SQL: toStringPointer("select 1"),
},
"q2": {
Name: toStringPointer("q2"), Title: toStringPointer("Q2"), Description: toStringPointer("THIS IS QUERY 2"), SQL: toStringPointer("select 2"),
},
},
},
},
"single_mod_two_queries_same_file": {
source: "test_data/mods/single_mod_two_queries_same_file",
expected: &modconfig.Mod{
Name: toStringPointer("m1"),
Title: toStringPointer("M1"),
Description: toStringPointer("THIS IS M1"),
ModDepends: []*modconfig.ModVersion{
{"github.com/turbot/m2", "0.0.0", toStringPointer("_m1")},
},
Queries: map[string]*modconfig.Query{
"q1": {
Name: toStringPointer("q1"), Title: toStringPointer("Q1"), Description: toStringPointer("THIS IS QUERY 1"), SQL: toStringPointer("select 1"),
},
"q2": {
Name: toStringPointer("q2"), Title: toStringPointer("Q2"), Description: toStringPointer("THIS IS QUERY 2"), SQL: toStringPointer("select 2"),
},
},
},
},
"single_mod_two_sql_files": {
source: "test_data/mods/single_mod_two_sql_files",
expected: &modconfig.Mod{
Name: toStringPointer("m1"),
Title: toStringPointer("M1"),
Description: toStringPointer("THIS IS M1"),
Queries: map[string]*modconfig.Query{
"q1": {
Name: toStringPointer("q1"), SQL: toStringPointer("select 1"),
},
"q2": {
Name: toStringPointer("q2"), SQL: toStringPointer("select 2"),
},
},
},
},
"two_mods": {
source: "test_data/mods/two_mods",
expected: "ERROR",
},
}
func TestLoadMod(t *testing.T) {
@@ -233,3 +233,8 @@ func executeLoadTest(t *testing.T, name string, test loadModTest, wd string) {
}
}
// return pointer to string
func toStringPointer(s string) *string {
return &s
}

36
steampipeconfig/modconfig/control.go Normal file
View File

@@ -0,0 +1,36 @@
package modconfig
import (
"fmt"
"reflect"
"github.com/turbot/go-kit/types"
)
type Control struct {
Name *string
Title *string `hcl:"title"`
Description *string `hcl:"description"`
Tags map[string]string `hcl:"tags"`
SQL *string `hcl:"sql"`
DocLink *string `hcl:"doc_link"`
}
func (c *Control) String() string {
return fmt.Sprintf(`
-----
Name: %s
Title: %s
Description: %s
SQL: %s
`, types.SafeString(c.Name), types.SafeString(c.Title), types.SafeString(c.Description), types.SafeString(c.SQL))
}
func (c *Control) Equals(other *Control) bool {
return types.SafeString(c.Name) == types.SafeString(other.Name) &&
types.SafeString(c.Title) == types.SafeString(other.Title) &&
types.SafeString(c.Description) == types.SafeString(other.Description) &&
types.SafeString(c.SQL) == types.SafeString(other.SQL) &&
types.SafeString(c.DocLink) == types.SafeString(other.DocLink) &&
reflect.DeepEqual(c.Tags, other.Tags)
}

65
steampipeconfig/modconfig/mod.go
View File

@@ -4,34 +4,26 @@ import (
"fmt"
"sort"
"strings"
"github.com/turbot/go-kit/types"
)
type Mod struct {
Name string
Title string `hcl:"title"`
Description string `hcl:"description"`
Version string
Name *string
Title *string `hcl:"title"`
Description *string `hcl:"description"`
Version *string
ModDepends []*ModVersion
PluginDepends []*PluginDependency
Queries []*Query
Queries map[string]*Query
Controls map[string]*Control
}
func (m *Mod) FullName() string {
if m.Version == "" {
return m.Name
if m.Version == nil {
return types.SafeString(m.Name)
}
return fmt.Sprintf("%s@%s", m.Name, m.Version)
}
// PopulateQueries :: convert a map of queries into a sorted array
// and set Queries property
func (mod *Mod) PopulateQueries(queries map[string]*Query) {
for _, q := range queries {
mod.Queries = append(mod.Queries, q)
}
sort.Slice(mod.Queries, func(i, j int) bool {
return mod.Queries[i].Name < mod.Queries[j].Name
})
return fmt.Sprintf("%s@%s", m.Name, types.SafeString(m.Version))
}
func (m *Mod) String() string {
@@ -46,14 +38,32 @@ func (m *Mod) String() string {
for _, d := range m.PluginDepends {
pluginDependStr = append(pluginDependStr, d.String())
}
// build ordered list of query names
var queryNames []string
for name := range m.Queries {
queryNames = append(queryNames, name)
}
sort.Strings(queryNames)
var queryStrings []string
for _, q := range m.Queries {
queryStrings = append(queryStrings, q.String())
for _, name := range queryNames {
queryStrings = append(queryStrings, m.Queries[name].String())
}
// build ordered list of control names
var controlNames []string
for name := range m.Controls {
controlNames = append(controlNames, name)
}
sort.Strings(controlNames)
var controlStrings []string
for _, name := range controlNames {
controlStrings = append(controlStrings, m.Controls[name].String())
}
versionString := ""
if m.Version != "" {
versionString = fmt.Sprintf("\nVersion: %s", m.Version)
if m.Version != nil {
versionString = fmt.Sprintf("\nVersion: %s", types.SafeString(m.Version))
}
return fmt.Sprintf(`Name: %s
Title: %s
@@ -61,13 +71,16 @@ Description: %s %s
Mod Dependencies: %s
Plugin Dependencies: %s
Queries:
%s
Controls:
%s`,
m.Name,
m.Title,
m.Description,
types.SafeString(m.Name),
types.SafeString(m.Title),
types.SafeString(m.Description),
versionString,
modDependStr,
pluginDependStr,
strings.Join(queryStrings, "\n"),
strings.Join(controlStrings, "\n"),
)
}

24
steampipeconfig/modconfig/query.go
View File

@@ -6,14 +6,16 @@ import (
"log"
"path/filepath"
"github.com/turbot/go-kit/types"
"github.com/turbot/steampipe/constants"
)
type Query struct {
Name string
Title string `hcl:"title"`
Description string `hcl:"description"`
SQL string `hcl:"sql"`
Name *string
Title *string `hcl:"title"`
Description *string `hcl:"description"`
SQL *string `hcl:"sql"`
}
func (q *Query) String() string {
@@ -23,14 +25,14 @@ func (q *Query) String() string {
Title: %s
Description: %s
SQL: %s
`, q.Name, q.Title, q.Description, q.SQL)
`, types.SafeString(q.Name), types.SafeString(q.Title), types.SafeString(q.Description), types.SafeString(q.SQL))
}
func (q *Query) Equals(other *Query) bool {
return q.Name == other.Name &&
q.Title == other.Title &&
q.Description == other.Description &&
q.SQL == other.SQL
return types.SafeString(q.Name) == types.SafeString(other.Name) &&
types.SafeString(q.Title) == types.SafeString(other.Title) &&
types.SafeString(q.Description) == types.SafeString(other.Description) &&
types.SafeString(q.SQL) == types.SafeString(other.SQL)
}
// QueryFromFile :: factory function
@@ -61,7 +63,7 @@ func (q *Query) InitialiseFromFile(modPath, filePath string) (MappableResource,
if err != nil {
return nil, err
}
q.Name = name
q.SQL = sql
q.Name = &name
q.SQL = &sql
return q, nil
}

17
steampipeconfig/parse_mod.go
View File

@@ -14,7 +14,7 @@ func parseMod(block *hcl.Block) (*modconfig.Mod, hcl.Diagnostics) {
return nil, diags
}
mod := &modconfig.Mod{
Name: block.Labels[0],
Name: &block.Labels[0],
}
moreDiags := parseModAttributes(content, mod)
if moreDiags.HasErrors() {
@@ -101,7 +101,20 @@ func parseQuery(block *hcl.Block) (*modconfig.Query, hcl.Diagnostics) {
return nil, diags
}
q.Name = block.Labels[0]
q.Name = &block.Labels[0]
return q, nil
}
func parseControl(block *hcl.Block) (*modconfig.Control, hcl.Diagnostics) {
var diags hcl.Diagnostics
var c = &modconfig.Control{}
diags = gohcl.DecodeBody(block.Body, nil, c)
if diags.HasErrors() {
return nil, diags
}
c.Name = &block.Labels[0]
return c, nil
}

4
steampipeconfig/schema.go
View File

@@ -46,6 +46,10 @@ var modFileSchema = &hcl.BodySchema{
Type: "query",
LabelNames: []string{"name"},
},
{
Type: "control",
LabelNames: []string{"name"},
},
},
}

0
steampipeconfig/test_data/mods/no_mod_nested_sql_files/queries/a/aa/q1.sql
View File

0
steampipeconfig/test_data/mods/no_mod_nested_sql_files/queries/a/q1.sql
View File

0
steampipeconfig/test_data/mods/no_mod_nested_sql_files/queries/b/bb/q2.sql
View File

0
steampipeconfig/test_data/mods/no_mod_nested_sql_files/queries/b/q2.sql
View File

4
steampipeconfig/test_data/mods/single_mod_nested_sql_files/mod.sp
View File

@@ -1,4 +0,0 @@
mod "m1"{
title = "M1"
description = "THIS IS M1"
}

1
steampipeconfig/test_data/mods/single_mod_nested_sql_files/queries/a/aa/q1.sql
View File

@@ -1 +0,0 @@
select 1

1
steampipeconfig/test_data/mods/single_mod_nested_sql_files/queries/a/q1.sql
View File

@@ -1 +0,0 @@
select 1

1
steampipeconfig/test_data/mods/single_mod_nested_sql_files/queries/b/bb/q2.sql
View File

@@ -1 +0,0 @@
select 2

1
steampipeconfig/test_data/mods/single_mod_nested_sql_files/queries/b/q2.sql
View File

@@ -1 +0,0 @@
select 2

10
steampipeconfig/test_data/mods/single_mod_one_query_one_control/control.sp Normal file
View File

@@ -0,0 +1,10 @@
control "c1"{
title ="C1"
description = "THIS IS CONTROL 1"
tags = {
"Application" = "demo"
"EnvironmentType" = "prod"
"ProductType" = "steampipe"
}
sql = "select 1"
}

10
steampipeconfig/test_data/mods/single_mod_one_query_one_control/mod.sp Normal file
View File

@@ -0,0 +1,10 @@
mod "m1"{
title = "M1"
description = "THIS IS M1"
mod_depends{
name = "github.com/turbot/m2"
version = "0.0.0"
alias = "_m2"
}
}

5
steampipeconfig/test_data/mods/single_mod_one_query_one_control/query.sp Normal file
View File

@@ -0,0 +1,5 @@
query "q1"{
title ="Q1"
description = "THIS IS QUERY 1"
sql = "select 1"
}

1
steampipeconfig/test_data/mods/single_mod_two_sql_files/q13.sql
View File

@@ -1 +0,0 @@
select 1

12
workspace/workspace.go
View File

@@ -30,10 +30,9 @@ func Load(workspacePath string) (*Workspace, error) {
workspace := &Workspace{Path: workspacePath}
// load the .steampipe ignore file
// NOTE DISABLED FOR NOW
//if err := workspace.LoadExclusions(); err != nil {
// return nil, err
//}
if err := workspace.LoadExclusions(); err != nil {
return nil, err
}
if err := workspace.loadMod(); err != nil {
return nil, err
@@ -115,12 +114,13 @@ func (w *Workspace) buildNamedQueryMap(modMap modconfig.ModMap) map[string]*modc
// build a list of long and short names for these queries
var res = make(map[string]*modconfig.Query)
// add local queries by short name: query.xxxx and long name: <workspace>.query.xxxx
// for LOCAL queries, add map entries keyed by both short name: query.xxxx and long name: <workspace>.query.xxxx
for _, q := range w.Mod.Queries {
shortName := fmt.Sprintf("query.%s", q.Name)
res[shortName] = q
}
// add queries from mod dependencies by FQN
// for mode dependencies, add queries keyed by long name only
for _, mod := range modMap {
for _, q := range mod.Queries {
longName := fmt.Sprintf("%s.query.%s", mod.Name, q.Name)