From 4fa11a424eefd37ec340f95dd63ceb71d7462ff1 Mon Sep 17 00:00:00 2001 From: kaidaguerre Date: Fri, 29 Sep 2023 19:48:17 +0100 Subject: [PATCH] Add function to clear connection cache. Closes #3937 * Add function to clear connection cache. * Update FDW to v1.9.0 - Add ability to clear connection cache by inserting into settings table. * Update plugin sdk to v5.6.1 * fix listen unit tests --- design/internal_introspection_tables.md | 8 ++++---- design/search_path.md | 2 +- go.mod | 2 +- go.sum | 4 ++-- pkg/constants/db.go | 7 ++++--- pkg/db/db_common/functions.go | 11 +++++++++++ pkg/interactive/interactive_client.go | 2 +- pkg/interactive/interactive_client_init.go | 2 +- pkg/steampipeconfig/connection_state_map.go | 4 ++-- pkg/steampipeconfig/parse/connection.go | 2 +- tests/acceptance/test_files/service.bats | 14 +++++++------- 11 files changed, 35 insertions(+), 23 deletions(-) diff --git a/design/internal_introspection_tables.md b/design/internal_introspection_tables.md index d6c56a024..77b481501 100644 --- a/design/internal_introspection_tables.md +++ b/design/internal_introspection_tables.md @@ -8,7 +8,7 @@ Lists all connections as defined in the connection config. - `` Lists all plugin instances as defined in the connection config. - `steampipe_plugin_limiter` -Lists all plugin Limiters as defined either in the plugin binary or the plugin conneciton block +Lists all plugin Limiters as defined either in the plugin binary or the plugin connection block ## Lifecycle @@ -20,7 +20,7 @@ Lists all plugin Limiters as defined either in the plugin binary or the plugin c - The connection states are set to either `pending` (if currently `ready`) or `incomplete` (if not). (These states will be updated by RefreshConnections.) - The connections are written back to the table -- RefreshConnections is triggered - this will apply any necessary conneciton updates and set the states of the connections +- RefreshConnections is triggered - this will apply any necessary connection updates and set the states of the connections to either `ready` or `error` #### steampipe_plugin @@ -86,6 +86,6 @@ It then builds a set of SetConnectionConfigRequest, one for each plugin instance ### Usage `steampipe_connection` table is used to determine whether a connection has been loaded yet. -This is used to allow us to execute queries without wasiting for all connecitons to load. Instead, we execute the query, -and if it fails with a relation not found error, we poll the coneciton state table until the conneciton is ready. +This is used to allow us to execute queries without wasiting for all connections to load. Instead, we execute the query, +and if it fails with a relation not found error, we poll the coneciton state table until the connection is ready. Then we retry the query. diff --git a/design/search_path.md b/design/search_path.md index ea9befefc..0497378c9 100644 --- a/design/search_path.md +++ b/design/search_path.md @@ -91,7 +91,7 @@ It is started by the steampipe service startup code. In the plugin manager process, a connection config file-watcher runs. If the connection config or options have changed, `RefreshConnectionsAndSearchPaths` is called. As discussed above this has the affect of: -- setting the user search path on the DB (this search path will be used for any subsequent connecitons from external clients) +- setting the user search path on the DB (this search path will be used for any subsequent connections from external clients) - setting the `requiredSessionSearchPath` on the (local) DbClient. HOWEVER - this just sets the required search path on the DbClient in the plugin manager process, NOT any DbClient used by Steampipe Query or Dashboard processes. ####Dashboard service search path implementation diff --git a/go.mod b/go.mod index 216c7b81e..ad32a14c5 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/stevenle/topsort v0.2.0 github.com/turbot/go-kit v0.9.0-rc.0 github.com/turbot/steampipe-cloud-sdk-go v0.6.0 - github.com/turbot/steampipe-plugin-sdk/v5 v5.6.0 + github.com/turbot/steampipe-plugin-sdk/v5 v5.6.1 github.com/xlab/treeprint v1.2.0 github.com/zclconf/go-cty v1.14.0 github.com/zclconf/go-cty-yaml v1.0.3 diff --git a/go.sum b/go.sum index db24b7fd0..90362a674 100644 --- a/go.sum +++ b/go.sum @@ -1098,8 +1098,8 @@ github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 h1: github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= github.com/turbot/steampipe-cloud-sdk-go v0.6.0 h1:ufAxOpKS1uq7eejuE5sfEu1+d7QAd0RBjl8Bn6+mIs8= github.com/turbot/steampipe-cloud-sdk-go v0.6.0/go.mod h1:M42TMBdMim4bV1YTMxhKyzfSGSMo4CXUkm3wt9w7t1Y= -github.com/turbot/steampipe-plugin-sdk/v5 v5.6.0 h1:jC7APLD/VBOHQ/NY/aAJVN9v3fhow0ges4YP0x+hOSk= -github.com/turbot/steampipe-plugin-sdk/v5 v5.6.0/go.mod h1:u2ubq9W5/5y6wG481LyulS7vuMOTRPmXAUfGLoVmwnA= +github.com/turbot/steampipe-plugin-sdk/v5 v5.6.1 h1:PnguL3AjhPwRtzS2KLEnyTPStTYLRSwQWz13GK5QWSA= +github.com/turbot/steampipe-plugin-sdk/v5 v5.6.1/go.mod h1:u2ubq9W5/5y6wG481LyulS7vuMOTRPmXAUfGLoVmwnA= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go v0.0.0-20180813092308-00b869d2f4a5/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= diff --git a/pkg/constants/db.go b/pkg/constants/db.go index ef1ebe483..63a04d573 100644 --- a/pkg/constants/db.go +++ b/pkg/constants/db.go @@ -28,7 +28,7 @@ const ( // constants for installing db and fdw images const ( DatabaseVersion = "14.2.0" - FdwVersion = "1.8.0" + FdwVersion = "1.9.0" // PostgresImageRef is the OCI Image ref for the database binaries PostgresImageRef = "us-docker.pkg.dev/steampipe/steampipe/db:14.2.0" @@ -78,8 +78,9 @@ const ( ForeignTableSettingsCacheTtlKey = "cache_ttl" ForeignTableSettingsCacheClearTimeKey = "cache_clear_time" - FunctionCacheSet = "meta_cache" - FunctionCacheSetTtl = "meta_cache_ttl" + FunctionCacheSet = "meta_cache" + FunctionConnectionCacheClear = "meta_connection_cache_clear" + FunctionCacheSetTtl = "meta_cache_ttl" // legacy LegacyCommandSchema = "steampipe_command" diff --git a/pkg/db/db_common/functions.go b/pkg/db/db_common/functions.go index 8d105621c..212bc5468 100644 --- a/pkg/db/db_common/functions.go +++ b/pkg/db/db_common/functions.go @@ -36,6 +36,17 @@ begin RAISE EXCEPTION 'Unknown value % for set_cache - valid values are on, off and clear.', $1; END IF; end; +`, + }, + { + Name: constants.FunctionConnectionCacheClear, + Params: map[string]string{"connection": "text"}, + Returns: "void", + Language: "plpgsql", + Body: ` +begin + INSERT INTO steampipe_internal.steampipe_settings("name","value") VALUES ('connection_cache_clear',connection); +end; `, }, { diff --git a/pkg/interactive/interactive_client.go b/pkg/interactive/interactive_client.go index 3b13b8af7..6047f0fcf 100644 --- a/pkg/interactive/interactive_client.go +++ b/pkg/interactive/interactive_client.go @@ -745,7 +745,7 @@ func (c *InteractiveClient) handleErrorsAndWarningsNotification(ctx context.Cont } func (c *InteractiveClient) handleConnectionUpdateNotification(ctx context.Context) { // ignore schema update notifications until initialisation is complete - // (we may receive schema update messages from the initial refresh connecitons, but we do not need to reload + // (we may receive schema update messages from the initial refresh connections, but we do not need to reload // the schema as we will have already loaded the correct schema) if !c.initialisationComplete { log.Printf("[INFO] received schema update notification but ignoring it as we are initializing") diff --git a/pkg/interactive/interactive_client_init.go b/pkg/interactive/interactive_client_init.go index b1f775469..9a9edd609 100644 --- a/pkg/interactive/interactive_client_init.go +++ b/pkg/interactive/interactive_client_init.go @@ -111,7 +111,7 @@ func (c *InteractiveClient) readInitDataStream(ctx context.Context) { statushooks.SetStatus(ctx, "Load plugin schemas…") // fetch the schema // TODO make this async https://github.com/turbot/steampipe/issues/3400 - // NOTE: we would like to do this asyncronously, but we are currently limited to a single Db conneciton in our + // NOTE: we would like to do this asyncronously, but we are currently limited to a single Db connection in our // as the client cache settings are set per connection so we rely on only having a single connection // This means that the schema load would block other queries anyway so there is no benefit right not in making asyncronous diff --git a/pkg/steampipeconfig/connection_state_map.go b/pkg/steampipeconfig/connection_state_map.go index 45ba96aab..719a898c9 100644 --- a/pkg/steampipeconfig/connection_state_map.go +++ b/pkg/steampipeconfig/connection_state_map.go @@ -47,7 +47,7 @@ func GetRequiredConnectionStateMap(connectionMap map[string]*modconfig.Connectio // if the connection is in error, create an error connection state // this may have been set by the loading code if connection.Error != nil { - // add error conneciton state + // add error connection state requiredState[connection.Name] = newErrorConnectionState(connection) // if error is a missing plugin, add to missingPluginMap // this will be used to build missing plugin warnings @@ -123,7 +123,7 @@ func (m ConnectionStateMap) Loaded(connections ...string) bool { for _, connectionName := range connections { connectionState, ok := m[connectionName] if !ok { - // ignore if we have no state loaded for this conneciton name + // ignore if we have no state loaded for this connection name continue } log.Println("[TRACE] Checking state for", connectionName) diff --git a/pkg/steampipeconfig/parse/connection.go b/pkg/steampipeconfig/parse/connection.go index 4c29429b4..f27262122 100644 --- a/pkg/steampipeconfig/parse/connection.go +++ b/pkg/steampipeconfig/parse/connection.go @@ -167,7 +167,7 @@ func getPluginInstanceFromDependency(dependencies []*modconfig.ResourceDependenc return split[1], true } -// build a hcl string with all attributes in the conneciton config which are NOT specified in the coneciton block schema +// build a hcl string with all attributes in the connection config which are NOT specified in the coneciton block schema // this is passed to the plugin who will validate and parse it func pluginConnectionConfigToHclString(body hcl.Body, connectionContent *hcl.BodyContent) (string, hcl.Diagnostics) { var diags hcl.Diagnostics diff --git a/tests/acceptance/test_files/service.bats b/tests/acceptance/test_files/service.bats index 474316353..9c919dda9 100644 --- a/tests/acceptance/test_files/service.bats +++ b/tests/acceptance/test_files/service.bats @@ -284,7 +284,7 @@ load "$LIB_BATS_SUPPORT/load.bash" assert_success # Extract listen from the state file - listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c '.resolved_listen | index("'$IPV4_ADDR'")') + listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c '.listen | index("'$IPV4_ADDR'")') echo $listen assert_not_equal "$listen" "null" @@ -303,7 +303,7 @@ load "$LIB_BATS_SUPPORT/load.bash" assert_success # Extract listen from the state file - listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c .resolved_listen) + listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c .listen) echo $listen assert_equal "$listen" '["127.0.0.1","::1","localhost"]' @@ -325,7 +325,7 @@ load "$LIB_BATS_SUPPORT/load.bash" assert_success # Extract listen from the state file - listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c '.resolved_listen | index("'$IPV4_ADDR'")') + listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c '.listen | index("'$IPV4_ADDR'")') echo $listen assert_not_equal "$listen" "null" @@ -347,7 +347,7 @@ load "$LIB_BATS_SUPPORT/load.bash" assert_success # Extract listen from the state file - listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c .resolved_listen) + listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c .listen) echo $listen assert_equal "$listen" '["127.0.0.1"]' @@ -369,7 +369,7 @@ load "$LIB_BATS_SUPPORT/load.bash" assert_success # Extract listen from the state file - listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c .resolved_listen) + listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c .listen) echo $listen assert_equal "$listen" '["127.0.0.1","::1"]' @@ -392,7 +392,7 @@ load "$LIB_BATS_SUPPORT/load.bash" assert_success # Extract listen from the state file - listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c '.resolved_listen | index("'$IPV4_ADDR'")') + listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c '.listen | index("'$IPV4_ADDR'")') echo $listen assert_not_equal "$listen" "null" @@ -418,7 +418,7 @@ load "$LIB_BATS_SUPPORT/load.bash" assert_success # Extract listen from the state file - listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c .resolved_listen) + listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c .listen) echo $listen assert_equal "$listen" '["127.0.0.1","'$IPV6_ADDR'"]'