Add function to clear connection cache. Closes #3937

* Add function to clear connection cache. * Update FDW to v1.9.0 - Add ability to clear connection cache by inserting into settings table. * Update plugin sdk to v5.6.1 * fix listen unit tests
2025-12-19 09:58:53 -05:00 · 2023-09-29 19:48:17 +01:00
parent e9b0a0a2b8
commit 4fa11a424e
11 changed files with 35 additions and 23 deletions
--- a/design/internal_introspection_tables.md
+++ b/design/internal_introspection_tables.md
@@ -8,7 +8,7 @@ Lists all connections as defined in the connection config.
 - ``
 Lists all plugin instances as defined in the connection config.
 - `steampipe_plugin_limiter`
-Lists all plugin Limiters as defined either in the plugin binary or the plugin conneciton block
+Lists all plugin Limiters as defined either in the plugin binary or the plugin connection block


 ## Lifecycle
@@ -20,7 +20,7 @@ Lists all plugin Limiters as defined either in the plugin binary or the plugin c
 - The connection states are set to either `pending` (if currently `ready`) or `incomplete` (if not).
  (These states will be updated by RefreshConnections.)
 - The connections are written back to the table
- RefreshConnections is triggered - this will apply any necessary conneciton updates and set the states of the connections 
+- RefreshConnections is triggered - this will apply any necessary connection updates and set the states of the connections 
 to either `ready` or `error`

 #### steampipe_plugin
@@ -86,6 +86,6 @@ It then builds a set of SetConnectionConfigRequest, one for each plugin instance
 ### Usage

 `steampipe_connection` table is used to determine whether a connection has been loaded yet.
-This is used to allow us to execute queries without wasiting for all connecitons to load. Instead, we execute the query,
-and if it fails with a relation not found error, we poll the coneciton state table until the conneciton is ready.
+This is used to allow us to execute queries without wasiting for all connections to load. Instead, we execute the query,
+and if it fails with a relation not found error, we poll the coneciton state table until the connection is ready.
 Then we retry the query. 
--- a/design/search_path.md
+++ b/design/search_path.md
@@ -91,7 +91,7 @@ It is started by the steampipe service startup code.

 In the plugin manager process, a connection config file-watcher runs. If the connection config or options have changed,
 `RefreshConnectionsAndSearchPaths` is called. As discussed above this has the affect of:
- setting the user search path on the DB (this search path will be used for any subsequent connecitons from external clients)
+- setting the user search path on the DB (this search path will be used for any subsequent connections from external clients)
 - setting the `requiredSessionSearchPath` on the (local) DbClient. HOWEVER - this just sets the required search path on the DbClient in the plugin manager process, NOT any DbClient used by Steampipe Query or Dashboard processes.

 ####Dashboard service search path implementation
--- a/go.mod
+++ b/go.mod
@@ -45,7 +45,7 @@ require (
 	github.com/stevenle/topsort v0.2.0
 	github.com/turbot/go-kit v0.9.0-rc.0
 	github.com/turbot/steampipe-cloud-sdk-go v0.6.0
-	github.com/turbot/steampipe-plugin-sdk/v5 v5.6.0
+	github.com/turbot/steampipe-plugin-sdk/v5 v5.6.1
 	github.com/xlab/treeprint v1.2.0
 	github.com/zclconf/go-cty v1.14.0
 	github.com/zclconf/go-cty-yaml v1.0.3
--- a/go.sum
+++ b/go.sum
@@ -1098,8 +1098,8 @@ github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 h1:
 github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw=
 github.com/turbot/steampipe-cloud-sdk-go v0.6.0 h1:ufAxOpKS1uq7eejuE5sfEu1+d7QAd0RBjl8Bn6+mIs8=
 github.com/turbot/steampipe-cloud-sdk-go v0.6.0/go.mod h1:M42TMBdMim4bV1YTMxhKyzfSGSMo4CXUkm3wt9w7t1Y=
-github.com/turbot/steampipe-plugin-sdk/v5 v5.6.0 h1:jC7APLD/VBOHQ/NY/aAJVN9v3fhow0ges4YP0x+hOSk=
-github.com/turbot/steampipe-plugin-sdk/v5 v5.6.0/go.mod h1:u2ubq9W5/5y6wG481LyulS7vuMOTRPmXAUfGLoVmwnA=
+github.com/turbot/steampipe-plugin-sdk/v5 v5.6.1 h1:PnguL3AjhPwRtzS2KLEnyTPStTYLRSwQWz13GK5QWSA=
+github.com/turbot/steampipe-plugin-sdk/v5 v5.6.1/go.mod h1:u2ubq9W5/5y6wG481LyulS7vuMOTRPmXAUfGLoVmwnA=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go v0.0.0-20180813092308-00b869d2f4a5/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
--- a/pkg/constants/db.go
+++ b/pkg/constants/db.go
@@ -28,7 +28,7 @@ const (
 // constants for installing db and fdw images
 const (
 	DatabaseVersion = "14.2.0"
-	FdwVersion      = "1.8.0"
+	FdwVersion      = "1.9.0"

 	// PostgresImageRef is the OCI Image ref for the database binaries
 	PostgresImageRef    = "us-docker.pkg.dev/steampipe/steampipe/db:14.2.0"
@@ -78,8 +78,9 @@ const (
 	ForeignTableSettingsCacheTtlKey       = "cache_ttl"
 	ForeignTableSettingsCacheClearTimeKey = "cache_clear_time"

-	FunctionCacheSet    = "meta_cache"
-	FunctionCacheSetTtl = "meta_cache_ttl"
+	FunctionCacheSet             = "meta_cache"
+	FunctionConnectionCacheClear = "meta_connection_cache_clear"
+	FunctionCacheSetTtl          = "meta_cache_ttl"

 	// legacy
 	LegacyCommandSchema = "steampipe_command"
--- a/pkg/db/db_common/functions.go
+++ b/pkg/db/db_common/functions.go
@@ -36,6 +36,17 @@ begin
 		RAISE EXCEPTION 'Unknown value % for set_cache - valid values are on, off and clear.', $1;
 	END IF;
 end;
+`,
+	},
+	{
+		Name:     constants.FunctionConnectionCacheClear,
+		Params:   map[string]string{"connection": "text"},
+		Returns:  "void",
+		Language: "plpgsql",
+		Body: `
+begin
+		INSERT INTO steampipe_internal.steampipe_settings("name","value") VALUES ('connection_cache_clear',connection);
+end;
 `,
 	},
 	{
--- a/pkg/interactive/interactive_client.go
+++ b/pkg/interactive/interactive_client.go
@@ -745,7 +745,7 @@ func (c *InteractiveClient) handleErrorsAndWarningsNotification(ctx context.Cont
 }
 func (c *InteractiveClient) handleConnectionUpdateNotification(ctx context.Context) {
 	// ignore schema update notifications until initialisation is complete
-	// (we may receive schema update messages from the initial refresh connecitons, but we do not need to reload
+	// (we may receive schema update messages from the initial refresh connections, but we do not need to reload
 	// the schema as we will have already loaded the correct schema)
 	if !c.initialisationComplete {
 		log.Printf("[INFO] received schema update notification but ignoring it as we are initializing")
--- a/pkg/interactive/interactive_client_init.go
+++ b/pkg/interactive/interactive_client_init.go
@@ -111,7 +111,7 @@ func (c *InteractiveClient) readInitDataStream(ctx context.Context) {
 	statushooks.SetStatus(ctx, "Load plugin schemas…")
 	//  fetch the schema
 	// TODO make this async https://github.com/turbot/steampipe/issues/3400
-	// NOTE: we would like to do this asyncronously, but we are currently limited to a single Db conneciton in our
+	// NOTE: we would like to do this asyncronously, but we are currently limited to a single Db connection in our
 	// as the client cache settings are set per connection so we rely on only having a single connection
 	// This means that the schema load would block other queries anyway so there is no benefit right not in making asyncronous

--- a/pkg/steampipeconfig/connection_state_map.go
+++ b/pkg/steampipeconfig/connection_state_map.go
@@ -47,7 +47,7 @@ func GetRequiredConnectionStateMap(connectionMap map[string]*modconfig.Connectio
 		// if the connection is in error, create an error connection state
 		// this may have been set by the loading code
 		if connection.Error != nil {
-			// add error conneciton state
+			// add error connection state
 			requiredState[connection.Name] = newErrorConnectionState(connection)
 			// if error is a missing plugin, add to missingPluginMap
 			// this will be used to build missing plugin warnings
@@ -123,7 +123,7 @@ func (m ConnectionStateMap) Loaded(connections ...string) bool {
 	for _, connectionName := range connections {
 		connectionState, ok := m[connectionName]
 		if !ok {
-			// ignore if we have no state loaded for this conneciton name
+			// ignore if we have no state loaded for this connection name
 			continue
 		}
 		log.Println("[TRACE] Checking state for", connectionName)
--- a/pkg/steampipeconfig/parse/connection.go
+++ b/pkg/steampipeconfig/parse/connection.go
@@ -167,7 +167,7 @@ func getPluginInstanceFromDependency(dependencies []*modconfig.ResourceDependenc
 	return split[1], true
 }

-// build a hcl string with all attributes in the conneciton config which are NOT specified in the coneciton block schema
+// build a hcl string with all attributes in the connection config which are NOT specified in the coneciton block schema
 // this is passed to the plugin who will validate and parse it
 func pluginConnectionConfigToHclString(body hcl.Body, connectionContent *hcl.BodyContent) (string, hcl.Diagnostics) {
 	var diags hcl.Diagnostics
--- a/tests/acceptance/test_files/service.bats
+++ b/tests/acceptance/test_files/service.bats
@@ -284,7 +284,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
  assert_success

  # Extract listen from the state file
-  listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c '.resolved_listen | index("'$IPV4_ADDR'")')
+  listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c '.listen | index("'$IPV4_ADDR'")')
  echo $listen

  assert_not_equal "$listen" "null"
@@ -303,7 +303,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
  assert_success

  # Extract listen from the state file
-  listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c .resolved_listen)
+  listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c .listen)
  echo $listen

  assert_equal "$listen" '["127.0.0.1","::1","localhost"]'
@@ -325,7 +325,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
  assert_success

  # Extract listen from the state file
-  listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c '.resolved_listen | index("'$IPV4_ADDR'")')
+  listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c '.listen | index("'$IPV4_ADDR'")')
  echo $listen

  assert_not_equal "$listen" "null"
@@ -347,7 +347,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
  assert_success

  # Extract listen from the state file
-  listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c .resolved_listen)
+  listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c .listen)
  echo $listen

  assert_equal "$listen" '["127.0.0.1"]'
@@ -369,7 +369,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
  assert_success

  # Extract listen from the state file
-  listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c .resolved_listen)
+  listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c .listen)
  echo $listen

  assert_equal "$listen" '["127.0.0.1","::1"]'
@@ -392,7 +392,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
  assert_success

  # Extract listen from the state file
-  listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c '.resolved_listen | index("'$IPV4_ADDR'")')
+  listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c '.listen | index("'$IPV4_ADDR'")')
  echo $listen

  assert_not_equal "$listen" "null"
@@ -418,7 +418,7 @@ load "$LIB_BATS_SUPPORT/load.bash"
  assert_success

  # Extract listen from the state file
-  listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c .resolved_listen)
+  listen=$(cat $STEAMPIPE_INSTALL_DIR/internal/steampipe.json | jq -c .listen)
  echo $listen

  assert_equal "$listen" '["127.0.0.1","'$IPV6_ADDR'"]'