jprdonnelly/steampipe
1
0
Fork 0
mirror of https://github.com/turbot/steampipe.git synced 2025-12-19 18:12:43 -05:00
Code Issues Packages Projects Releases Wiki Activity

Remove deprecated options. Closes #4131, Closes #4132, Closes #3751

Browse Source
This commit is contained in:
Puskar Basu
2024-02-22 20:28:34 +05:30
committed by GitHub
parent 7ff6c8692b
commit f63ff54741
10 changed files with 66 additions and 384 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
pkg/steampipeconfig/options/general.go
View File

@@ -9,7 +9,6 @@ import (
type General struct {
UpdateCheck *string `hcl:"update_check"`
MaxParallel *int `hcl:"max_parallel"`
Telemetry *string `hcl:"telemetry"`
LogLevel *string `hcl:"log_level"`
MemoryMaxMb *int `hcl:"memory_max_mb"`
@@ -25,9 +24,6 @@ func (g *General) ConfigMap() map[string]interface{} {
if g.Telemetry != nil {
res[constants.ArgTelemetry] = g.Telemetry
}
if g.MaxParallel != nil {
res[constants.ArgMaxParallel] = g.MaxParallel
}
if g.LogLevel != nil {
res[constants.ArgLogLevel] = g.LogLevel
}
@@ -59,13 +55,6 @@ func (g *General) String() string {
} else {
str = append(str, fmt.Sprintf(" UpdateCheck: %s", *g.UpdateCheck))
}
if g.MaxParallel == nil {
str = append(str, " MaxParallel: nil")
} else {
str = append(str, fmt.Sprintf(" MaxParallel: %d", *g.MaxParallel))
}
if g.Telemetry == nil {
str = append(str, " Telemetry: nil")
} else {

10
pkg/steampipeconfig/parse/connection.go
View File

@@ -9,7 +9,6 @@ import (
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/turbot/pipe-fittings/hclhelpers"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
"github.com/zclconf/go-cty/cty"
"golang.org/x/exp/maps"
@@ -70,15 +69,6 @@ func DecodeConnection(block *hcl.Block) (*modconfig.Connection, hcl.Diagnostics)
diags = append(diags, moreDiags...)
}
// TODO: remove in 0.22 [https://github.com/turbot/steampipe/issues/3251]
if connection.Options != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagWarning,
Summary: fmt.Sprintf("%s in %s have been deprecated and will be removed in subsequent versions of steampipe", constants.Bold("'connection' options"), constants.Bold("'connection' blocks")),
Subject: hclhelpers.BlockRangePointer(connectionBlock),
})
}
default:
// this can never happen
diags = append(diags, &hcl.Diagnostic{

14
pkg/steampipeconfig/parse/decode_options.go
View File

@@ -39,14 +39,12 @@ type OptionsBlockMapping = map[string]options.Options
func defaultOptionsBlockMapping() OptionsBlockMapping {
mapping := OptionsBlockMapping{
options.ConnectionBlock: &options.Connection{},
options.DatabaseBlock: &options.Database{},
options.TerminalBlock: &options.Terminal{},
options.GeneralBlock: &options.General{},
options.QueryBlock: &options.Query{},
options.CheckBlock: &options.Check{},
options.DashboardBlock: &options.GlobalDashboard{},
options.PluginBlock: &options.Plugin{},
options.DatabaseBlock: &options.Database{},
options.GeneralBlock: &options.General{},
options.QueryBlock: &options.Query{},
options.CheckBlock: &options.Check{},
options.DashboardBlock: &options.GlobalDashboard{},
options.PluginBlock: &options.Plugin{},
}
return mapping
}

49
pkg/steampipeconfig/steampipeconfig.go
View File

@@ -38,11 +38,6 @@ type SteampipeConfig struct {
PluginOptions *options.Plugin
// map of installed plugin versions, keyed by plugin image ref
PluginVersions map[string]*versionfile.InstalledVersion
// TODO remove this in 0.22
// it is only needed due to conflicts with output name in terminal options
// https://github.com/turbot/steampipe/issues/2534
commandName string
}
func NewSteampipeConfig(commandName string) *SteampipeConfig {
@@ -50,7 +45,6 @@ func NewSteampipeConfig(commandName string) *SteampipeConfig {
Connections: make(map[string]*modconfig.Connection),
Plugins: make(map[string][]*modconfig.Plugin),
PluginsInstances: make(map[string]*modconfig.Plugin),
commandName: commandName,
}
}
@@ -94,9 +88,6 @@ func (c *SteampipeConfig) ConfigMap() map[string]interface{} {
if c.DashboardOptions != nil {
res.PopulateConfigMapForOptions(c.DashboardOptions)
}
if c.TerminalOptions != nil {
res.PopulateConfigMapForOptions(c.TerminalOptions)
}
if c.PluginOptions != nil {
res.PopulateConfigMapForOptions(c.PluginOptions)
}
@@ -108,14 +99,6 @@ func (c *SteampipeConfig) SetOptions(opts options.Options) (errorsAndWarnings *e
errorsAndWarnings = error_helpers.NewErrorsAndWarning(nil)
switch o := opts.(type) {
case *options.Connection:
// TODO: remove in 0.21 [https://github.com/turbot/steampipe/issues/3251]
errorsAndWarnings.AddWarning(deprecationWarning("connection options"))
if c.DefaultConnectionOptions == nil {
c.DefaultConnectionOptions = o
} else {
c.DefaultConnectionOptions.Merge(o)
}
case *options.Database:
if c.DatabaseOptions == nil {
c.DatabaseOptions = o
@@ -128,54 +111,22 @@ func (c *SteampipeConfig) SetOptions(opts options.Options) (errorsAndWarnings *e
} else {
c.DashboardOptions.Merge(o)
}
case *options.Terminal:
// TODO: remove in 0.21 [https://github.com/turbot/steampipe/issues/3251]
errorsAndWarnings.AddWarning(deprecationWarning("terminal options"))
// NOTE: ignore terminal options if current command is not query
// this is a short term workaround to handle the clashing 'output' argument
// this will be refactored
// TODO: remove in 0.21 [https://github.com/turbot/steampipe/issues/3251]
if c.commandName != constants.CmdNameQuery {
break
}
if c.TerminalOptions == nil {
c.TerminalOptions = o
} else {
c.TerminalOptions.Merge(o)
}
case *options.General:
if c.GeneralOptions == nil {
c.GeneralOptions = o
} else {
c.GeneralOptions.Merge(o)
}
// TODO: remove in 0.22 [https://github.com/turbot/steampipe/issues/3251]
if c.GeneralOptions.MaxParallel != nil {
errorsAndWarnings.AddWarning(deprecationWarning(fmt.Sprintf("'%s' in %s", constants.Bold("max_parallel"), constants.Bold("general options"))))
}
case *options.Plugin:
if c.PluginOptions == nil {
c.PluginOptions = o
} else {
c.PluginOptions.Merge(o)
}
// TODO: remove in 0.21 [https://github.com/turbot/steampipe/issues/3251]
if c.GeneralOptions.MaxParallel != nil {
errorsAndWarnings.AddWarning(deprecationWarning(fmt.Sprintf("'%s' in %s", constants.Bold("max_parallel"), constants.Bold("general options"))))
}
}
return errorsAndWarnings
}
func deprecationWarning(subject string) string {
if subject == "terminal options" {
return fmt.Sprintf("%s has been deprecated and will be removed in a future version of Steampipe.\nThese can now be set in a steampipe %s.", constants.Bold(subject), constants.Bold("workspace"))
}
return fmt.Sprintf("%s has been deprecated and will be removed in a future version of Steampipe.\nThis can now be set in a steampipe %s.", subject, constants.Bold("workspace"))
}
var defaultCacheEnabled = true
var defaultTTL = 300

8
tests/acceptance/test_data/source_files/chaos_options.json
View File

@@ -5,13 +5,7 @@
"regions": [
"us-east-1",
"us-west-2"
],
"options": {
"connection": {
"cache": true,
"cache_ttl": 300
}
}
]
}
}
}

4
tests/acceptance/test_data/source_files/chaos_options.spc
View File

@@ -1,8 +1,4 @@
connection "chaos6" {
plugin = "chaos"
regions = ["us-east-1", "us-west-2"]
options "connection" {
cache = true
cache_ttl = 300
}
}

4
tests/acceptance/test_data/source_files/chaos_options.yml
View File

@@ -4,7 +4,3 @@ connection:
regions:
- us-east-1
- us-west-2
options:
connection:
cache: true
cache_ttl: 300

3
tests/acceptance/test_data/source_files/workspace_cache_disabled.spc Normal file
View File

@@ -0,0 +1,3 @@
workspace "default" {
cache = false
}

4
tests/acceptance/test_data/source_files/workspace_cache_enabled.spc Normal file
View File

@@ -0,0 +1,4 @@
workspace "default" {
cache = true
cache_ttl = 10
}

343
tests/acceptance/test_files/cache.bats
View File

@@ -1,297 +1,6 @@
load "$LIB_BATS_ASSERT/load.bash"
load "$LIB_BATS_SUPPORT/load.bash"
@test "steampipe check options config is being parsed and used(cache=true; hcl)" {
cp $SRC_DATA_DIR/chaos_options.spc $STEAMPIPE_INSTALL_DIR/config/chaos_options.spc
# cache functionality check since cache=true in options
cd $CONFIG_PARSING_TEST_MOD
run steampipe check benchmark.config_parsing_benchmark --export test.json --max-parallel 1
# store the unique number from 1st control in `content`
content=$(cat test.json | jq '.groups[].controls[0].results[0].resource')
# store the unique number from 2nd control in `new_content`
new_content=$(cat test.json | jq '.groups[].controls[1].results[0].resource')
echo $content
echo $new_content
# remove the output and the config files
rm -f test.json
rm -f $STEAMPIPE_INSTALL_DIR/config/chaos_options.spc
# verify that `content` and `new_content` are the same
assert_equal "$new_content" "$content"
}
@test "steampipe check options config is being parsed and used(cache=true; yml)" {
cp $SRC_DATA_DIR/chaos_options.yml $STEAMPIPE_INSTALL_DIR/config/chaos_options.yml
# cache functionality check since cache=true in options
cd $CONFIG_PARSING_TEST_MOD
run steampipe check benchmark.config_parsing_benchmark --export test.json --max-parallel 1
# store the unique number from 1st control in `content`
content=$(cat test.json | jq '.groups[].controls[0].results[0].resource')
# store the unique number from 2nd control in `new_content`
new_content=$(cat test.json | jq '.groups[].controls[1].results[0].resource')
echo $content
echo $new_content
# remove the output and the config files
rm -f test.json
rm -f $STEAMPIPE_INSTALL_DIR/config/chaos_options.yml
# verify that `content` and `new_content` are the same
assert_equal "$new_content" "$content"
}
@test "steampipe check options config is being parsed and used(cache=true; json)" {
cp $SRC_DATA_DIR/chaos_options.json $STEAMPIPE_INSTALL_DIR/config/chaos_options.json
# cache functionality check since cache=true in options
cd $CONFIG_PARSING_TEST_MOD
run steampipe check benchmark.config_parsing_benchmark --export test.json --max-parallel 1
# store the unique number from 1st control in `content`
content=$(cat test.json | jq '.groups[].controls[0].results[0].resource')
# store the unique number from 2nd control in `new_content`
new_content=$(cat test.json | jq '.groups[].controls[1].results[0].resource')
echo $content
echo $new_content
# remove the output and the config files
rm -f test.json
rm -f $STEAMPIPE_INSTALL_DIR/config/chaos_options.json
# verify that `content` and `new_content` are the same
assert_equal "$new_content" "$content"
}
@test "steampipe check options config is being parsed and used(cache=false; hcl)" {
cp $SRC_DATA_DIR/chaos_options_2.spc $STEAMPIPE_INSTALL_DIR/config/chaos_options_2.spc
# cache functionality check since cache=false in options
cd $CONFIG_PARSING_TEST_MOD
run steampipe check benchmark.config_parsing_benchmark --export test.json --max-parallel 1
# store the unique number from 1st control in `content`
content=$(cat test.json | jq '.groups[].controls[0].results[0].resource')
# store the unique number from 2nd control in `new_content`
new_content=$(cat test.json | jq '.groups[].controls[1].results[0].resource')
echo $content
echo $new_content
# verify that `content` and `new_content` are not the same
if [[ "$content" == "$new_content" ]]; then
flag=1
else
flag=0
fi
# remove the output and the config files
rm -f test.json
rm -f $STEAMPIPE_INSTALL_DIR/config/chaos_options_2.spc
assert_equal "$flag" "0"
}
@test "steampipe check options config is being parsed and used(cache=false; yml)" {
cp $SRC_DATA_DIR/chaos_options_2.yml $STEAMPIPE_INSTALL_DIR/config/chaos_options_2.yml
# cache functionality check since cache=false in options
cd $CONFIG_PARSING_TEST_MOD
run steampipe check benchmark.config_parsing_benchmark --export test.json --max-parallel 1
# store the unique number from 1st control in `content`
content=$(cat test.json | jq '.groups[].controls[0].results[0].resource')
# store the unique number from 2nd control in `new_content`
new_content=$(cat test.json | jq '.groups[].controls[1].results[0].resource')
echo $content
echo $new_content
# verify that `content` and `new_content` are not the same
if [[ "$content" == "$new_content" ]]; then
flag=1
else
flag=0
fi
# remove the output and the config files
rm -f test.json
rm -f $STEAMPIPE_INSTALL_DIR/config/chaos_options_2.yml
assert_equal "$flag" "0"
}
# This test checks whether the options in the hcl connection config(chaos_options.spc) is parsed and used correctly.
# To test this behaviour:
# 1. We create a config with options passed(cache=true; cache_ttl=300)
# 2. Start the service and run the query which selects an unique value(unique_col) from the chaos_cache_check table.
# 3. Run the query twice and store the values to compare, before stopping the service.
# 4. Compare the values, both the unique values should be equal since we had cache enabled in config.
@test "steampipe query options config is being parsed and used(cache=true; hcl)" {
cp $SRC_DATA_DIR/chaos_options.spc $STEAMPIPE_INSTALL_DIR/config/chaos_options.spc
# start the service
steampipe service start
# cache functionality check since cache=true in options
steampipe query "select unique_col from chaos6.chaos_cache_check where id=2" --output json > out1.json
steampipe query "select unique_col from chaos6.chaos_cache_check where id=2" --output json > out2.json
# stop the service
steampipe service stop
# store the unique number from 1st query in `content`
content=$(cat out1.json | jq '.[].unique_col')
# store the unique number from 2nd query in `new_content`
new_content=$(cat out2.json | jq '.[].unique_col')
echo $content
echo $new_content
# remove the output and the config files
rm -f out*.json
rm -f $STEAMPIPE_INSTALL_DIR/config/chaos_options.spc
# verify that `content` and `new_content` are the same
assert_equal "$new_content" "$content"
}
# This test checks whether the options in the yml connection config(chaos_options.yml) is parsed and used correctly.
# To test this behaviour:
# 1. We create a config with options passed(cache=true; cache_ttl=300)
# 2. Start the service and run the query which selects an unique value(unique_col) from the chaos_cache_check table.
# 3. Run the query twice and store the values to compare, before stopping the service.
# 4. Compare the values, both the unique values should be equal since we had cache enabled in config.
@test "steampipe query options config is being parsed and used(cache=true; yml)" {
cp $SRC_DATA_DIR/chaos_options.yml $STEAMPIPE_INSTALL_DIR/config/chaos_options.yml
# start the service
steampipe service start
# cache functionality check since cache=true in options
steampipe query "select unique_col from chaos6.chaos_cache_check where id=2" --output json > out1.json
steampipe query "select unique_col from chaos6.chaos_cache_check where id=2" --output json > out2.json
# stop the service
steampipe service stop
# store the unique number from 1st query in `content`
content=$(cat out1.json | jq '.[].unique_col')
# store the unique number from 2nd query in `new_content`
new_content=$(cat out2.json | jq '.[].unique_col')
echo $content
echo $new_content
# remove the output and the config files
rm -f out*.json
rm -f $STEAMPIPE_INSTALL_DIR/config/chaos_options.yml
# verify that `content` and `new_content` are the same
assert_equal "$new_content" "$content"
}
# This test checks whether the options in the json connection config(chaos_options.spc) is parsed and used correctly.
# To test this behaviour:
# 1. We create a config with options passed(cache=true; cache_ttl=300)
# 2. Start the service and run the query which selects an unique value(unique_col) from the chaos_cache_check table.
# 3. Run the query twice and store the values to compare, before stopping the service.
# 4. Compare the values, both the unique values should be equal since we had cache enabled in config.
@test "steampipe query options config is being parsed and used(cache=true; json)" {
cp $SRC_DATA_DIR/chaos_options.json $STEAMPIPE_INSTALL_DIR/config/chaos_options.json
# start the service
steampipe service start
# cache functionality check since cache=true in options
steampipe query "select unique_col from chaos6.chaos_cache_check where id=2" --output json > out1.json
steampipe query "select unique_col from chaos6.chaos_cache_check where id=2" --output json > out2.json
# stop the service
steampipe service stop
# store the unique number from 1st query in `content`
content=$(cat out1.json | jq '.[].unique_col')
# store the unique number from 2nd query in `new_content`
new_content=$(cat out2.json | jq '.[].unique_col')
echo $content
echo $new_content
# remove the output and the config files
rm -f out*.json
rm -f $STEAMPIPE_INSTALL_DIR/config/chaos_options.json
# verify that `content` and `new_content` are the same
assert_equal "$new_content" "$content"
}
# This test checks whether the options in the hcl connection config(chaos_options.spc) is parsed and used correctly.
# To test this behaviour:
# 1. We create a config with options passed(cache=false; cache_ttl=300)
# 2. Start the service and run the query which selects an unique value(unique_col) from the chaos_cache_check table.
# 3. Run the query twice and store the values to compare, before stopping the service.
# 4. Compare the values, both the unique values should different since we had cache disabled in config.
@test "steampipe query options config is being parsed and used(cache=false; hcl)" {
cp $SRC_DATA_DIR/chaos_options_2.spc $STEAMPIPE_INSTALL_DIR/config/chaos_options_2.spc
# start the service
steampipe service start
# cache functionality check since cache=false in options
steampipe query "select unique_col from chaos6.chaos_cache_check where id=2" --output json > out1.json
steampipe query "select unique_col from chaos6.chaos_cache_check where id=2" --output json > out2.json
# stop the service
steampipe service stop
# store the unique number from 1st query in `content`
content=$(cat out1.json | jq '.[].unique_col')
# store the unique number from 2nd query in `new_content`
new_content=$(cat out2.json | jq '.[].unique_col')
# verify that `content` and `new_content` are not the same
if [[ "$content" == "$new_content" ]]; then
flag=1
else
flag=0
fi
# remove the output and the config files
rm -f out*.json
rm -f $STEAMPIPE_INSTALL_DIR/config/chaos_options_2.spc
assert_equal "$flag" "0"
}
# This test checks whether the options in the yml connection config(chaos_options.spc) is parsed and used correctly.
# To test this behaviour:
# 1. We create a config with options passed(cache=false; cache_ttl=300)
# 2. Start the service and run the query which selects an unique value(unique_col) from the chaos_cache_check table.
# 3. Run the query twice and store the values to compare, before stopping the service.
# 4. Compare the values, both the unique values should different since we had cache disabled in config.
@test "steampipe query options config is being parsed and used(cache=false; yml)" {
cp $SRC_DATA_DIR/chaos_options_2.yml $STEAMPIPE_INSTALL_DIR/config/chaos_options_2.yml
# start the service
steampipe service start
# cache functionality check since cache=false in options
steampipe query "select unique_col from chaos6.chaos_cache_check where id=2" --output json > out1.json
steampipe query "select unique_col from chaos6.chaos_cache_check where id=2" --output json > out2.json
# stop the service
steampipe service stop
# store the unique number from 1st query in `content`
content=$(cat out1.json | jq '.[].unique_col')
# store the unique number from 2nd query in `new_content`
new_content=$(cat out2.json | jq '.[].unique_col')
# verify that `content` and `new_content` are not the same
if [[ "$content" == "$new_content" ]]; then
flag=1
else
flag=0
fi
# remove the output and the config files
rm -f out*.json
rm -f $STEAMPIPE_INSTALL_DIR/config/chaos_options_2.yml
assert_equal "$flag" "0"
}
@test "steampipe cache functionality check ON" {
run steampipe plugin install chaos
cd $FUNCTIONALITY_TEST_MOD
@@ -469,6 +178,58 @@ load "$LIB_BATS_SUPPORT/load.bash"
assert_not_equal "$unique1" "$unique3"
}
@test "test caching with cache=true in workspace profile" {
cp $SRC_DATA_DIR/chaos_options.spc $STEAMPIPE_INSTALL_DIR/config/chaos_options.spc
cp $SRC_DATA_DIR/workspace_cache_enabled.spc $STEAMPIPE_INSTALL_DIR/config/workspace_cache_enabled.spc
# cache functionality check since cache=true in workspace profile
cd $CONFIG_PARSING_TEST_MOD
run steampipe check benchmark.config_parsing_benchmark --export test.json --max-parallel 1
# store the unique number from 1st control in `content`
content=$(cat test.json | jq '.groups[].controls[0].results[0].resource')
# store the unique number from 2nd control in `new_content`
new_content=$(cat test.json | jq '.groups[].controls[1].results[0].resource')
echo $content
echo $new_content
# remove the output and the config files
rm -f test.json
rm -f $STEAMPIPE_INSTALL_DIR/config/chaos_options.spc
rm -f $STEAMPIPE_INSTALL_DIR/config/workspace_cache_enabled.spc
# verify that `content` and `new_content` are the same
assert_equal "$new_content" "$content"
}
@test "test caching with cache=false in workspace profile" {
cp $SRC_DATA_DIR/chaos_options.spc $STEAMPIPE_INSTALL_DIR/config/chaos_options.spc
cp $SRC_DATA_DIR/workspace_cache_disabled.spc $STEAMPIPE_INSTALL_DIR/config/workspace_cache_disabled.spc
# cache functionality check since cache=false in workspace profile
cd $CONFIG_PARSING_TEST_MOD
run steampipe check benchmark.config_parsing_benchmark --export test.json --max-parallel 1
# store the unique number from 1st control in `content`
content=$(cat test.json | jq '.groups[].controls[0].results[0].resource')
# store the unique number from 2nd control in `new_content`
new_content=$(cat test.json | jq '.groups[].controls[1].results[0].resource')
echo $content
echo $new_content
# verify that `content` and `new_content` are not the same
if [[ "$content" == "$new_content" ]]; then
flag=1
else
flag=0
fi
# remove the output and the config files
rm -f test.json
rm -f $STEAMPIPE_INSTALL_DIR/config/chaos_options.spc
rm -f $STEAMPIPE_INSTALL_DIR/config/workspace_cache_disabled.spc
assert_equal "$flag" "0"
}
@test "verify cache ttl works when set in workspace profile" {
cp $FILE_PATH/test_data/source_files/workspace_cache_ttl.spc $STEAMPIPE_INSTALL_DIR/config/workspace.spc
cp $SRC_DATA_DIR/chaos_no_options.spc $STEAMPIPE_INSTALL_DIR/config/chaos_no_options.spc