diff --git a/steampipeconfig/load_config_test.go b/steampipeconfig/load_config_test.go index 17f312877..288ce6104 100644 --- a/steampipeconfig/load_config_test.go +++ b/steampipeconfig/load_config_test.go @@ -66,7 +66,6 @@ secret_key = "aws_dmi_002_secret_key"`, steampipeDir: "test_data/connection_config/single_connection", expected: &SteampipeConfig{ Connections: map[string]*Connection{ - // todo normalise plugin names here? "a": { Name: "a", Plugin: "hub.steampipe.io/plugins/test_data/connection-test-1@latest", @@ -82,7 +81,6 @@ secret_key = "aws_dmi_002_secret_key"`, steampipeDir: "test_data/connection_config/single_connection_with_default_options", expected: &SteampipeConfig{ Connections: map[string]*Connection{ - // todo normalise plugin names here? "a": { Name: "a", Plugin: "hub.steampipe.io/plugins/test_data/connection-test-1@latest", @@ -120,7 +118,6 @@ secret_key = "aws_dmi_002_secret_key"`, workspaceDir: "test_data/workspaces/search_path_prefix", expected: &SteampipeConfig{ Connections: map[string]*Connection{ - // todo normalise plugin names here? "a": { Name: "a", Plugin: "hub.steampipe.io/plugins/test_data/connection-test-1@latest", @@ -154,7 +151,6 @@ secret_key = "aws_dmi_002_secret_key"`, workspaceDir: "test_data/workspaces/override_terminal_config", expected: &SteampipeConfig{ Connections: map[string]*Connection{ - // todo normalise plugin names here? "a": { Name: "a", Plugin: "hub.steampipe.io/plugins/test_data/connection-test-1@latest", diff --git a/steampipeconfig/load_mod.go b/steampipeconfig/load_mod.go index 437528e28..0d0a40abf 100644 --- a/steampipeconfig/load_mod.go +++ b/steampipeconfig/load_mod.go @@ -46,80 +46,48 @@ func LoadMod(modPath string, opts *LoadModOptions) (mod *modconfig.Mod, err erro } }() - var parseResult = newModParseResult() - - // NOTE: for now sp file loading is disabled - enableModLoading := false - if enableModLoading { - if opts == nil { - opts = &LoadModOptions{} - } - // verify the mod folder exists - if _, err := os.Stat(modPath); os.IsNotExist(err) { - return nil, fmt.Errorf("mod folder %s does not exist", modPath) - } - - // build list of all filepaths we need to parse/load - // NOTE: pseudo resource creation is handled separately below - var include = filehelpers.InclusionsFromExtensions([]string{constants.ModDataExtension}) - sourcePaths, err := getSourcePaths(modPath, include, opts.Exclude) - if err != nil { - log.Printf("[WARN] LoadMod: failed to get mod file paths: %v\n", err) - return nil, err - } - - // load the raw data - fileData, diags := loadFileData(sourcePaths) - if diags.HasErrors() { - log.Printf("[WARN] LoadMod: failed to load all mod files: %v\n", err) - return nil, plugin.DiagsToError("Failed to load all mod files", diags) - } - - // parse all hcl files - parseResult, err = parseModHcl(modPath, fileData) - if err != nil { - return nil, err - } + if opts == nil { + opts = &LoadModOptions{} + } + // verify the mod folder exists + if _, err := os.Stat(modPath); os.IsNotExist(err) { + return nil, fmt.Errorf("mod folder %s does not exist", modPath) } - // is there a mod resource definition? - if parseResult.mod != nil { - mod = parseResult.mod - } else { - // so there is no mod resource definition - check flag for our response - if !opts.CreateDefaultMod() { - // CreateDefaultMod flag NOT set - fail - return nil, fmt.Errorf("mod folder %s does not contain a mod resource definition", modPath) - } - // just create a default mod - mod = defaultWorkspaceMod() + // build list of all filepaths we need to parse/load + // NOTE: pseudo resource creation is handled separately below + var include = filehelpers.InclusionsFromExtensions([]string{constants.ModDataExtension}) + sourcePaths, err := getSourcePaths(modPath, include, opts.Exclude) + if err != nil { + log.Printf("[WARN] LoadMod: failed to get mod file paths: %v\n", err) + return nil, err + } + + // load the raw data + fileData, diags := loadFileData(sourcePaths) + if diags.HasErrors() { + log.Printf("[WARN] LoadMod: failed to load all mod files: %v\n", err) + return nil, plugin.DiagsToError("Failed to load all mod files", diags) + } + + // parse all hcl files. + mod, err = parseModHcl(modPath, fileData, opts) + if err != nil { + return nil, err } // if flag is set, create pseudo resources by mapping files if opts.CreatePseudoResources() { // now execute any pseudo-resource creations based on file mappings - err = createPseudoResources(modPath, parseResult, opts) + err = createPseudoResources(modPath, mod, opts) if err != nil { return nil, err } } - // now convert query map into an array and set on the mod object - mod.PopulateQueries(parseResult.queryMap) return } -type modParseResult struct { - queryMap map[string]*modconfig.Query - mod *modconfig.Mod -} - -func newModParseResult() *modParseResult { - return &modParseResult{ - queryMap: make(map[string]*modconfig.Query), - } -} - // GetModFileExtensions :: return list of all file extensions we care about // this will be the mod data extension, plus any registered extensions registered in fileToResourceMap func GetModFileExtensions() []string { @@ -149,7 +117,7 @@ func getSourcePaths(modPath string, include, exclude []string) ([]string, error) } // parse all source hcl files for the mod and associated resources -func parseModHcl(modPath string, fileData map[string][]byte) (*modParseResult, error) { +func parseModHcl(modPath string, fileData map[string][]byte, opts *LoadModOptions) (*modconfig.Mod, error) { var mod *modconfig.Mod body, diags := parseHclFiles(fileData) @@ -164,6 +132,7 @@ func parseModHcl(modPath string, fileData map[string][]byte) (*modParseResult, e } var queries = make(map[string]*modconfig.Query) + var controls = make(map[string]*modconfig.Control) for _, block := range content.Blocks { switch block.Type { case "variable": @@ -184,15 +153,31 @@ func parseModHcl(modPath string, fileData map[string][]byte) (*modParseResult, e diags = append(diags, moreDiags...) break } - if _, ok := queries[query.Name]; ok { + if _, ok := queries[*query.Name]; ok { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, - Summary: fmt.Sprintf("mod defines more that one query named %s", query.Name), + Summary: fmt.Sprintf("mod defines more that one query named %s", *query.Name), Subject: &block.DefRange, }) continue } - queries[query.Name] = query + queries[*query.Name] = query + + case "control": + control, moreDiags := parseControl(block) + if moreDiags.HasErrors() { + diags = append(diags, moreDiags...) + break + } + if _, ok := controls[*control.Name]; ok { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("mod defines more that one control named %s", *control.Name), + Subject: &block.DefRange, + }) + continue + } + controls[*control.Name] = control } } @@ -200,12 +185,26 @@ func parseModHcl(modPath string, fileData map[string][]byte) (*modParseResult, e return nil, plugin.DiagsToError("Failed to parse all mod hcl files", diags) } - return &modParseResult{queries, mod}, nil + // is there a mod resource definition? + if mod == nil { + + // should we creaste a default mod? + if !opts.CreateDefaultMod() { + // CreateDefaultMod flag NOT set - fail + return nil, fmt.Errorf("mod folder %s does not contain a mod resource definition", modPath) + } + // just create a default mod + mod = defaultWorkspaceMod() + } + mod.Queries = queries + mod.Controls = controls + + return mod, nil } // create pseudo-resources for any files whose extensions are registered // NOTE: this mutates parseResults -func createPseudoResources(modPath string, parseResults *modParseResult, opts *LoadModOptions) error { +func createPseudoResources(modPath string, mod *modconfig.Mod, opts *LoadModOptions) error { // list all registered files var include = filehelpers.InclusionsFromExtensions(modconfig.RegisteredFileExtensions()) sourcePaths, err := getSourcePaths(modPath, include, opts.Exclude) @@ -228,7 +227,7 @@ func createPseudoResources(modPath string, parseResults *modParseResult, opts *L continue } if resource != nil { - if err := addResourceIfUnique(resource, parseResults, path); err != nil { + if err := addResourceIfUnique(resource, mod, path); err != nil { errors = append(errors, err) } } @@ -245,19 +244,20 @@ func createPseudoResources(modPath string, parseResults *modParseResult, opts *L } // add resource to parse results, if there is no resource of same name -func addResourceIfUnique(resource modconfig.MappableResource, parseResults *modParseResult, path string) error { +func addResourceIfUnique(resource modconfig.MappableResource, mod *modconfig.Mod, path string) error { switch r := resource.(type) { case *modconfig.Query: // check there is not already a query with the same name - if _, ok := parseResults.queryMap[r.Name]; ok { + if _, ok := mod.Queries[*r.Name]; ok { // we have already created a query with this name - skip! - return fmt.Errorf("not creating resource for '%s' as there is already a query '%s' defined", path, r.Name) + return fmt.Errorf("not creating resource for '%s' as there is already a query '%s' defined", path, *r.Name) } - parseResults.queryMap[r.Name] = r + mod.Queries[*r.Name] = r } return nil } func defaultWorkspaceMod() *modconfig.Mod { - return &modconfig.Mod{Name: "local"} + defaultName := "local" + return &modconfig.Mod{Name: &defaultName} } diff --git a/steampipeconfig/load_mod_test.go b/steampipeconfig/load_mod_test.go index 20090e306..185f7920b 100644 --- a/steampipeconfig/load_mod_test.go +++ b/steampipeconfig/load_mod_test.go @@ -16,179 +16,179 @@ type loadModTest struct { expected interface{} } -var alias = "_m2" - var loadWorkspaceOptions = &LoadModOptions{ Exclude: []string{fmt.Sprintf("**/%s*", constants.WorkspaceDataDir)}, Flags: CreatePseudoResources | CreateDefaultMod, } -var testCasesLoadMod = map[string]loadModTest{ +var testCasesLoadMod map[string]loadModTest = map[string]loadModTest{ "no_mod_sql_files": { source: "test_data/mods/no_mod_sql_files", expected: &modconfig.Mod{ - Name: "local", - Queries: []*modconfig.Query{ - { - Name: "q1", SQL: "select 1", + Name: toStringPointer("local"), + Queries: map[string]*modconfig.Query{ + "q1": { + Name: toStringPointer("q1"), SQL: toStringPointer("select 1"), }, - { - Name: "q2", SQL: "select 2", + "q2": { + Name: toStringPointer("q2"), SQL: toStringPointer("select 2"), }, }}, }, - // commented out as we do not support SP file loading yet + "no_mod_hcl_queries": { + source: "test_data/mods/no_mod_hcl_queries", + expected: &modconfig.Mod{ + Name: toStringPointer("local"), + Queries: map[string]*modconfig.Query{ + "q1": { + toStringPointer("q1"), toStringPointer("Q1"), toStringPointer("THIS IS QUERY 1"), toStringPointer("select 1"), + }, + "q2": { + toStringPointer("q2"), toStringPointer("Q2"), toStringPointer("THIS IS QUERY 2"), toStringPointer("select 2"), + }, + }, + }, + }, + "single_mod_duplicate_query": { + source: "test_data/mods/single_mod_duplicate_query", + expected: "ERROR", + }, + "single_mod_no_query": { + source: "test_data/mods/single_mod_no_query", + expected: &modconfig.Mod{ + Name: toStringPointer("m1"), + Title: toStringPointer("M1"), + Description: toStringPointer("THIS IS M1"), + ModDepends: []*modconfig.ModVersion{ + {"github.com/turbot/m2", "0.0.0", toStringPointer("_m1")}, + }, + }, + }, + "single_mod_one_query": { + source: "test_data/mods/single_mod_one_query", + expected: &modconfig.Mod{ + Name: toStringPointer("m1"), + Title: toStringPointer("M1"), + Description: toStringPointer("THIS IS M1"), + ModDepends: []*modconfig.ModVersion{ + {"github.com/turbot/m2", "0.0.0", toStringPointer("_m1")}, + }, + Queries: map[string]*modconfig.Query{ + "q1": { + Name: toStringPointer("q1"), Title: toStringPointer("Q1"), Description: toStringPointer("THIS IS QUERY 1"), SQL: toStringPointer("select 1"), + }, + }, + }, + }, + "single_mod_one_query_one_control": { + source: "test_data/mods/single_mod_one_query_one_control", + expected: &modconfig.Mod{ + Name: toStringPointer("m1"), + Title: toStringPointer("M1"), + Description: toStringPointer("THIS IS M1"), + ModDepends: []*modconfig.ModVersion{ + {"github.com/turbot/m2", "0.0.0", toStringPointer("_m1")}, + }, + Queries: map[string]*modconfig.Query{ + "q1": { - //"no_mod_hcl_queries": { - // source: "test_data/mods/no_mod_hcl_queries", - // expected: &modconfig.Mod{ - // Name: "local", - // Queries: []*modconfig.Query{ - // { - // "q1", "Q1", "THIS IS QUERY 1", "select 1", - // }, - // { - // "q2", "Q2", "THIS IS QUERY 2", "select 2", - // }, - // }, - // }, - //}, - //"single_mod_duplicate_query": { - // source: "test_data/mods/single_mod_duplicate_query", - // expected: "ERROR", - //}, - //"single_mod_nested_sql_files": { - // source: "test_data/mods/single_mod_nested_sql_files", - // expected: &modconfig.Mod{ - // Name: "m1", - // Title: "M1", - // Description: "THIS IS M1", - // Queries: []*modconfig.Query{ - // { - // Name: "queries_a_aa_q1", SQL: "select 1", - // }, - // { - // Name: "queries_a_q1", SQL: "select 1", - // }, - // { - // Name: "queries_b_bb_q2", SQL: "select 2", - // }, - // { - // Name: "queries_b_q2", SQL: "select 2", - // }, - // }, - // }, - //}, - //"single_mod_no_query": { - // source: "test_data/mods/single_mod_no_query", - // expected: &modconfig.Mod{ - // Name: "m1", - // Title: "M1", - // Description: "THIS IS M1", - // ModDepends: []*modconfig.ModVersion{ - // {"github.com/turbot/m2", "0.0.0", &alias}, - // }, - // }, - //}, - //"single_mod_one_query": { - // source: "test_data/mods/single_mod_one_query", - // expected: &modconfig.Mod{ - // Name: "m1", - // Title: "M1", - // Description: "THIS IS M1", - // ModDepends: []*modconfig.ModVersion{ - // {"github.com/turbot/m2", "0.0.0", &alias}, - // }, - // Queries: []*modconfig.Query{ - // { - // "q1", "Q1", "THIS IS QUERY 1", "select 1", - // }, - // }, - // }, - //}, - //"single_mod_one_sql_file": { - // source: "test_data/mods/single_mod_one_sql_file", - // expected: &modconfig.Mod{ - // Name: "m1", - // Title: "M1", - // Description: "THIS IS M1", - // Queries: []*modconfig.Query{{Name: "q1", SQL: "select 1"}}, - // }, - //}, - //"single_mod_sql_file_and_hcl_query": { - // source: "test_data/mods/single_mod_sql_file_and_hcl_query", - // expected: &modconfig.Mod{ - // Name: "m1", - // Title: "M1", - // Description: "THIS IS M1", - // Queries: []*modconfig.Query{ - // { - // "q1", "Q1", "THIS IS QUERY 1", "select 1", - // }, - // { - // Name: "q2", SQL: "select 2", - // }, - // }, - // }, - //}, - //"single_mod_two_queries_diff_files": { - // source: "test_data/mods/single_mod_two_queries_diff_files", - // expected: &modconfig.Mod{ - // Name: "m1", - // Title: "M1", - // Description: "THIS IS M1", - // ModDepends: []*modconfig.ModVersion{ - // {"github.com/turbot/m2", "0.0.0", &alias}, - // }, - // Queries: []*modconfig.Query{ - // { - // "q1", "Q1", "THIS IS QUERY 1", "select 1", - // }, - // { - // "q2", "Q2", "THIS IS QUERY 2", "select 2", - // }, - // }, - // }, - //}, - //"single_mod_two_queries_same_file": { - // source: "test_data/mods/single_mod_two_queries_same_file", - // expected: &modconfig.Mod{ - // Name: "m1", - // Title: "M1", - // Description: "THIS IS M1", - // ModDepends: []*modconfig.ModVersion{ - // {"github.com/turbot/m2", "0.0.0", &alias}, - // }, - // Queries: []*modconfig.Query{ - // { - // "q1", "Q1", "THIS IS QUERY 1", "select 1", - // }, - // { - // "q2", "Q2", "THIS IS QUERY 2", "select 2", - // }, - // }, - // }, - //}, - //"single_mod_two_sql_files": { - // source: "test_data/mods/single_mod_two_sql_files", - // expected: &modconfig.Mod{ - // Name: "m1", - // Title: "M1", - // Description: "THIS IS M1", - // Queries: []*modconfig.Query{ - // { - // Name: "q1", SQL: "select 1", - // }, - // { - // Name: "q2", SQL: "select 2", - // }, - // }, - // }, - //}, - //"two_mods": { - // source: "test_data/mods/two_mods", - // expected: "ERROR", - //}, + Name: toStringPointer("q1"), Title: toStringPointer("Q1"), Description: toStringPointer("THIS IS QUERY 1"), SQL: toStringPointer("select 1"), + }, + }, + Controls: map[string]*modconfig.Control{ + "c1": { + Name: toStringPointer("c1"), Title: toStringPointer("C1"), Description: toStringPointer("THIS IS CONTROL 1"), SQL: toStringPointer("select 1"), Tags: map[string]string{ + "Application": "demo", + "EnvironmentType": "prod", + "ProductType": "steampipe", + }, + }, + }, + }, + }, + "single_mod_one_sql_file": { + source: "test_data/mods/single_mod_one_sql_file", + expected: &modconfig.Mod{ + Name: toStringPointer("m1"), + Title: toStringPointer("M1"), + Description: toStringPointer("THIS IS M1"), + Queries: map[string]*modconfig.Query{"q1": {Name: toStringPointer("q1"), SQL: toStringPointer("select 1")}}, + }, + }, + "single_mod_sql_file_and_hcl_query": { + source: "test_data/mods/single_mod_sql_file_and_hcl_query", + expected: &modconfig.Mod{ + Name: toStringPointer("m1"), + Title: toStringPointer("M1"), + Description: toStringPointer("THIS IS M1"), + Queries: map[string]*modconfig.Query{ + "q1": { + Name: toStringPointer("q1"), Title: toStringPointer("Q1"), Description: toStringPointer("THIS IS QUERY 1"), SQL: toStringPointer("select 1"), + }, + "q2": { + Name: toStringPointer("q2"), SQL: toStringPointer("select 2"), + }, + }, + }, + }, + "single_mod_two_queries_diff_files": { + source: "test_data/mods/single_mod_two_queries_diff_files", + expected: &modconfig.Mod{ + Name: toStringPointer("m1"), + Title: toStringPointer("M1"), + Description: toStringPointer("THIS IS M1"), + ModDepends: []*modconfig.ModVersion{ + {"github.com/turbot/m2", "0.0.0", toStringPointer("_m1")}, + }, + Queries: map[string]*modconfig.Query{ + "q1": { + Name: toStringPointer("q1"), Title: toStringPointer("Q1"), Description: toStringPointer("THIS IS QUERY 1"), SQL: toStringPointer("select 1"), + }, + "q2": { + Name: toStringPointer("q2"), Title: toStringPointer("Q2"), Description: toStringPointer("THIS IS QUERY 2"), SQL: toStringPointer("select 2"), + }, + }, + }, + }, + "single_mod_two_queries_same_file": { + source: "test_data/mods/single_mod_two_queries_same_file", + expected: &modconfig.Mod{ + Name: toStringPointer("m1"), + Title: toStringPointer("M1"), + Description: toStringPointer("THIS IS M1"), + ModDepends: []*modconfig.ModVersion{ + {"github.com/turbot/m2", "0.0.0", toStringPointer("_m1")}, + }, + Queries: map[string]*modconfig.Query{ + "q1": { + Name: toStringPointer("q1"), Title: toStringPointer("Q1"), Description: toStringPointer("THIS IS QUERY 1"), SQL: toStringPointer("select 1"), + }, + "q2": { + Name: toStringPointer("q2"), Title: toStringPointer("Q2"), Description: toStringPointer("THIS IS QUERY 2"), SQL: toStringPointer("select 2"), + }, + }, + }, + }, + "single_mod_two_sql_files": { + source: "test_data/mods/single_mod_two_sql_files", + expected: &modconfig.Mod{ + Name: toStringPointer("m1"), + Title: toStringPointer("M1"), + Description: toStringPointer("THIS IS M1"), + Queries: map[string]*modconfig.Query{ + "q1": { + Name: toStringPointer("q1"), SQL: toStringPointer("select 1"), + }, + "q2": { + Name: toStringPointer("q2"), SQL: toStringPointer("select 2"), + }, + }, + }, + }, + "two_mods": { + source: "test_data/mods/two_mods", + expected: "ERROR", + }, } func TestLoadMod(t *testing.T) { @@ -233,3 +233,8 @@ func executeLoadTest(t *testing.T, name string, test loadModTest, wd string) { } } + +// return pointer to string +func toStringPointer(s string) *string { + return &s +} diff --git a/steampipeconfig/modconfig/control.go b/steampipeconfig/modconfig/control.go new file mode 100644 index 000000000..8ccfdee94 --- /dev/null +++ b/steampipeconfig/modconfig/control.go @@ -0,0 +1,36 @@ +package modconfig + +import ( + "fmt" + "reflect" + + "github.com/turbot/go-kit/types" +) + +type Control struct { + Name *string + Title *string `hcl:"title"` + Description *string `hcl:"description"` + Tags map[string]string `hcl:"tags"` + SQL *string `hcl:"sql"` + DocLink *string `hcl:"doc_link"` +} + +func (c *Control) String() string { + return fmt.Sprintf(` + ----- + Name: %s + Title: %s + Description: %s + SQL: %s +`, types.SafeString(c.Name), types.SafeString(c.Title), types.SafeString(c.Description), types.SafeString(c.SQL)) +} + +func (c *Control) Equals(other *Control) bool { + return types.SafeString(c.Name) == types.SafeString(other.Name) && + types.SafeString(c.Title) == types.SafeString(other.Title) && + types.SafeString(c.Description) == types.SafeString(other.Description) && + types.SafeString(c.SQL) == types.SafeString(other.SQL) && + types.SafeString(c.DocLink) == types.SafeString(other.DocLink) && + reflect.DeepEqual(c.Tags, other.Tags) +} diff --git a/steampipeconfig/modconfig/mod.go b/steampipeconfig/modconfig/mod.go index 636aad567..0f21e2720 100644 --- a/steampipeconfig/modconfig/mod.go +++ b/steampipeconfig/modconfig/mod.go @@ -4,34 +4,26 @@ import ( "fmt" "sort" "strings" + + "github.com/turbot/go-kit/types" ) type Mod struct { - Name string - Title string `hcl:"title"` - Description string `hcl:"description"` - Version string + Name *string + Title *string `hcl:"title"` + Description *string `hcl:"description"` + Version *string ModDepends []*ModVersion PluginDepends []*PluginDependency - Queries []*Query + Queries map[string]*Query + Controls map[string]*Control } func (m *Mod) FullName() string { - if m.Version == "" { - return m.Name + if m.Version == nil { + return types.SafeString(m.Name) } - return fmt.Sprintf("%s@%s", m.Name, m.Version) -} - -// PopulateQueries :: convert a map of queries into a sorted array -// and set Queries property -func (mod *Mod) PopulateQueries(queries map[string]*Query) { - for _, q := range queries { - mod.Queries = append(mod.Queries, q) - } - sort.Slice(mod.Queries, func(i, j int) bool { - return mod.Queries[i].Name < mod.Queries[j].Name - }) + return fmt.Sprintf("%s@%s", m.Name, types.SafeString(m.Version)) } func (m *Mod) String() string { @@ -46,14 +38,32 @@ func (m *Mod) String() string { for _, d := range m.PluginDepends { pluginDependStr = append(pluginDependStr, d.String()) } + // build ordered list of query names + var queryNames []string + for name := range m.Queries { + queryNames = append(queryNames, name) + } + sort.Strings(queryNames) + var queryStrings []string - for _, q := range m.Queries { - queryStrings = append(queryStrings, q.String()) + for _, name := range queryNames { + queryStrings = append(queryStrings, m.Queries[name].String()) + } + // build ordered list of control names + var controlNames []string + for name := range m.Controls { + controlNames = append(controlNames, name) + } + sort.Strings(controlNames) + + var controlStrings []string + for _, name := range controlNames { + controlStrings = append(controlStrings, m.Controls[name].String()) } versionString := "" - if m.Version != "" { - versionString = fmt.Sprintf("\nVersion: %s", m.Version) + if m.Version != nil { + versionString = fmt.Sprintf("\nVersion: %s", types.SafeString(m.Version)) } return fmt.Sprintf(`Name: %s Title: %s @@ -61,13 +71,16 @@ Description: %s %s Mod Dependencies: %s Plugin Dependencies: %s Queries: +%s +Controls: %s`, - m.Name, - m.Title, - m.Description, + types.SafeString(m.Name), + types.SafeString(m.Title), + types.SafeString(m.Description), versionString, modDependStr, pluginDependStr, strings.Join(queryStrings, "\n"), + strings.Join(controlStrings, "\n"), ) } diff --git a/steampipeconfig/modconfig/query.go b/steampipeconfig/modconfig/query.go index a294d5c01..144091a22 100644 --- a/steampipeconfig/modconfig/query.go +++ b/steampipeconfig/modconfig/query.go @@ -6,14 +6,16 @@ import ( "log" "path/filepath" + "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe/constants" ) type Query struct { - Name string - Title string `hcl:"title"` - Description string `hcl:"description"` - SQL string `hcl:"sql"` + Name *string + Title *string `hcl:"title"` + Description *string `hcl:"description"` + SQL *string `hcl:"sql"` } func (q *Query) String() string { @@ -23,14 +25,14 @@ func (q *Query) String() string { Title: %s Description: %s SQL: %s -`, q.Name, q.Title, q.Description, q.SQL) +`, types.SafeString(q.Name), types.SafeString(q.Title), types.SafeString(q.Description), types.SafeString(q.SQL)) } func (q *Query) Equals(other *Query) bool { - return q.Name == other.Name && - q.Title == other.Title && - q.Description == other.Description && - q.SQL == other.SQL + return types.SafeString(q.Name) == types.SafeString(other.Name) && + types.SafeString(q.Title) == types.SafeString(other.Title) && + types.SafeString(q.Description) == types.SafeString(other.Description) && + types.SafeString(q.SQL) == types.SafeString(other.SQL) } // QueryFromFile :: factory function @@ -61,7 +63,7 @@ func (q *Query) InitialiseFromFile(modPath, filePath string) (MappableResource, if err != nil { return nil, err } - q.Name = name - q.SQL = sql + q.Name = &name + q.SQL = &sql return q, nil } diff --git a/steampipeconfig/parse_mod.go b/steampipeconfig/parse_mod.go index a7f260f8d..8ae0b95ee 100644 --- a/steampipeconfig/parse_mod.go +++ b/steampipeconfig/parse_mod.go @@ -14,7 +14,7 @@ func parseMod(block *hcl.Block) (*modconfig.Mod, hcl.Diagnostics) { return nil, diags } mod := &modconfig.Mod{ - Name: block.Labels[0], + Name: &block.Labels[0], } moreDiags := parseModAttributes(content, mod) if moreDiags.HasErrors() { @@ -101,7 +101,20 @@ func parseQuery(block *hcl.Block) (*modconfig.Query, hcl.Diagnostics) { return nil, diags } - q.Name = block.Labels[0] + q.Name = &block.Labels[0] return q, nil } +func parseControl(block *hcl.Block) (*modconfig.Control, hcl.Diagnostics) { + var diags hcl.Diagnostics + var c = &modconfig.Control{} + + diags = gohcl.DecodeBody(block.Body, nil, c) + if diags.HasErrors() { + return nil, diags + } + + c.Name = &block.Labels[0] + + return c, nil +} diff --git a/steampipeconfig/schema.go b/steampipeconfig/schema.go index e8c67b261..92b3a4600 100644 --- a/steampipeconfig/schema.go +++ b/steampipeconfig/schema.go @@ -46,6 +46,10 @@ var modFileSchema = &hcl.BodySchema{ Type: "query", LabelNames: []string{"name"}, }, + { + Type: "control", + LabelNames: []string{"name"}, + }, }, } diff --git a/steampipeconfig/test_data/mods/no_mod_nested_sql_files/queries/a/aa/q1.sql b/steampipeconfig/test_data/mods/no_mod_nested_sql_files/queries/a/aa/q1.sql deleted file mode 100644 index e69de29bb..000000000 diff --git a/steampipeconfig/test_data/mods/no_mod_nested_sql_files/queries/a/q1.sql b/steampipeconfig/test_data/mods/no_mod_nested_sql_files/queries/a/q1.sql deleted file mode 100644 index e69de29bb..000000000 diff --git a/steampipeconfig/test_data/mods/no_mod_nested_sql_files/queries/b/bb/q2.sql b/steampipeconfig/test_data/mods/no_mod_nested_sql_files/queries/b/bb/q2.sql deleted file mode 100644 index e69de29bb..000000000 diff --git a/steampipeconfig/test_data/mods/no_mod_nested_sql_files/queries/b/q2.sql b/steampipeconfig/test_data/mods/no_mod_nested_sql_files/queries/b/q2.sql deleted file mode 100644 index e69de29bb..000000000 diff --git a/steampipeconfig/test_data/mods/single_mod_nested_sql_files/mod.sp b/steampipeconfig/test_data/mods/single_mod_nested_sql_files/mod.sp deleted file mode 100644 index fd2175a73..000000000 --- a/steampipeconfig/test_data/mods/single_mod_nested_sql_files/mod.sp +++ /dev/null @@ -1,4 +0,0 @@ -mod "m1"{ - title = "M1" - description = "THIS IS M1" -} \ No newline at end of file diff --git a/steampipeconfig/test_data/mods/single_mod_nested_sql_files/queries/a/aa/q1.sql b/steampipeconfig/test_data/mods/single_mod_nested_sql_files/queries/a/aa/q1.sql deleted file mode 100644 index 6bb32b4a8..000000000 --- a/steampipeconfig/test_data/mods/single_mod_nested_sql_files/queries/a/aa/q1.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 \ No newline at end of file diff --git a/steampipeconfig/test_data/mods/single_mod_nested_sql_files/queries/a/q1.sql b/steampipeconfig/test_data/mods/single_mod_nested_sql_files/queries/a/q1.sql deleted file mode 100644 index 6bb32b4a8..000000000 --- a/steampipeconfig/test_data/mods/single_mod_nested_sql_files/queries/a/q1.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 \ No newline at end of file diff --git a/steampipeconfig/test_data/mods/single_mod_nested_sql_files/queries/b/bb/q2.sql b/steampipeconfig/test_data/mods/single_mod_nested_sql_files/queries/b/bb/q2.sql deleted file mode 100644 index 273c95989..000000000 --- a/steampipeconfig/test_data/mods/single_mod_nested_sql_files/queries/b/bb/q2.sql +++ /dev/null @@ -1 +0,0 @@ -select 2 \ No newline at end of file diff --git a/steampipeconfig/test_data/mods/single_mod_nested_sql_files/queries/b/q2.sql b/steampipeconfig/test_data/mods/single_mod_nested_sql_files/queries/b/q2.sql deleted file mode 100644 index 273c95989..000000000 --- a/steampipeconfig/test_data/mods/single_mod_nested_sql_files/queries/b/q2.sql +++ /dev/null @@ -1 +0,0 @@ -select 2 \ No newline at end of file diff --git a/steampipeconfig/test_data/mods/single_mod_one_query_one_control/control.sp b/steampipeconfig/test_data/mods/single_mod_one_query_one_control/control.sp new file mode 100644 index 000000000..ef1432906 --- /dev/null +++ b/steampipeconfig/test_data/mods/single_mod_one_query_one_control/control.sp @@ -0,0 +1,10 @@ +control "c1"{ + title ="C1" + description = "THIS IS CONTROL 1" + tags = { + "Application" = "demo" + "EnvironmentType" = "prod" + "ProductType" = "steampipe" +} + sql = "select 1" +} diff --git a/steampipeconfig/test_data/mods/single_mod_one_query_one_control/mod.sp b/steampipeconfig/test_data/mods/single_mod_one_query_one_control/mod.sp new file mode 100644 index 000000000..8ea4a382c --- /dev/null +++ b/steampipeconfig/test_data/mods/single_mod_one_query_one_control/mod.sp @@ -0,0 +1,10 @@ +mod "m1"{ + title = "M1" + description = "THIS IS M1" + mod_depends{ + name = "github.com/turbot/m2" + version = "0.0.0" + alias = "_m2" + } + +} \ No newline at end of file diff --git a/steampipeconfig/test_data/mods/single_mod_one_query_one_control/query.sp b/steampipeconfig/test_data/mods/single_mod_one_query_one_control/query.sp new file mode 100644 index 000000000..f93ddc094 --- /dev/null +++ b/steampipeconfig/test_data/mods/single_mod_one_query_one_control/query.sp @@ -0,0 +1,5 @@ +query "q1"{ + title ="Q1" + description = "THIS IS QUERY 1" + sql = "select 1" +} diff --git a/steampipeconfig/test_data/mods/single_mod_two_sql_files/q13.sql b/steampipeconfig/test_data/mods/single_mod_two_sql_files/q13.sql deleted file mode 100644 index 6bb32b4a8..000000000 --- a/steampipeconfig/test_data/mods/single_mod_two_sql_files/q13.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 \ No newline at end of file diff --git a/workspace/workspace.go b/workspace/workspace.go index 2fec1d1b8..a18f9f94e 100644 --- a/workspace/workspace.go +++ b/workspace/workspace.go @@ -30,10 +30,9 @@ func Load(workspacePath string) (*Workspace, error) { workspace := &Workspace{Path: workspacePath} // load the .steampipe ignore file - // NOTE DISABLED FOR NOW - //if err := workspace.LoadExclusions(); err != nil { - // return nil, err - //} + if err := workspace.LoadExclusions(); err != nil { + return nil, err + } if err := workspace.loadMod(); err != nil { return nil, err @@ -115,12 +114,13 @@ func (w *Workspace) buildNamedQueryMap(modMap modconfig.ModMap) map[string]*modc // build a list of long and short names for these queries var res = make(map[string]*modconfig.Query) - // add local queries by short name: query.xxxx and long name: .query.xxxx + // for LOCAL queries, add map entries keyed by both short name: query.xxxx and long name: .query.xxxx for _, q := range w.Mod.Queries { shortName := fmt.Sprintf("query.%s", q.Name) res[shortName] = q } - // add queries from mod dependencies by FQN + + // for mode dependencies, add queries keyed by long name only for _, mod := range modMap { for _, q := range mod.Queries { longName := fmt.Sprintf("%s.query.%s", mod.Name, q.Name)