Split versions.json into 2 files, one in the plugins dir, one in the database dir. Closes #576

This commit is contained in:
tyagiparth
2021-07-13 17:38:16 +05:30
committed by GitHub
parent c2b1af79af
commit 3fbf2b542e
16 changed files with 394 additions and 459 deletions

View File

@@ -324,7 +324,7 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) {
}
// load up the version file data
versionData, err := versionfile.Load()
versionData, err := versionfile.LoadPluginVersionFile()
if err != nil {
utils.ShowError(fmt.Errorf("error loading current plugin data"))
return

View File

@@ -12,11 +12,15 @@ import (
const (
DefaultInstallDir = "~/.steampipe"
ConnectionsStateFileName = "connection.json"
versionFileName = "versions.json"
)
var SteampipeDir string
func steampipeSubDir(dirName string) string {
if SteampipeDir == "" {
panic(fmt.Errorf("cannot call any Steampipe directory functions before SteampipeDir is set"))
}
subDir := filepath.Join(SteampipeDir, dirName)
if _, err := os.Stat(subDir); os.IsNotExist(err) {
@@ -27,42 +31,60 @@ func steampipeSubDir(dirName string) string {
return subDir
}
// PluginDir :: returns the path to the plugins directory (creates if missing)
// PluginDir returns the path to the plugins directory (creates if missing)
func PluginDir() string {
return steampipeSubDir("plugins")
}
// ConnectionStatePath :: returns the path of the connections state file
// ConnectionStatePath returns the path of the connections state file
func ConnectionStatePath() string {
return filepath.Join(InternalDir(), ConnectionsStateFileName)
}
// ModsDir :: returns the path to the mods directory (creates if missing)
// ModsDir returns the path to the mods directory (creates if missing)
func ModsDir() string {
return steampipeSubDir("mods")
}
// ConfigDir :: returns the path to the config directory (creates if missing)
// ConfigDir returns the path to the config directory (creates if missing)
func ConfigDir() string {
return steampipeSubDir("config")
}
// InternalDir :: returns the path to the internal directory (creates if missing)
// InternalDir returns the path to the internal directory (creates if missing)
func InternalDir() string {
return steampipeSubDir("internal")
}
// DatabaseDir :: returns the path to the db directory (creates if missing)
// DatabaseDir returns the path to the db directory (creates if missing)
func DatabaseDir() string {
return steampipeSubDir("db")
}
// LogDir :: returns the path to the db log directory (creates if missing)
// LogDir returns the path to the db log directory (creates if missing)
func LogDir() string {
return steampipeSubDir("logs")
}
// TempDir :: returns the path to the steampipe tmp directory (creates if missing)
// TempDir returns the path to the steampipe tmp directory (creates if missing)
func TempDir() string {
return steampipeSubDir("tmp")
}
// LegacyVersionFilePath returns the legacy version file path
func LegacyVersionFilePath() string {
path := filepath.Join(InternalDir(), versionFileName)
return path
}
// PluginVersionFilePath returns the plugin version file path
func PluginVersionFilePath() string {
path := filepath.Join(PluginDir(), versionFileName)
return path
}
// DatabaseVersionFilePath returns the plugin version file path
func DatabaseVersionFilePath() string {
path := filepath.Join(DatabaseDir(), versionFileName)
return path
}

View File

@@ -218,7 +218,7 @@ func installSteampipeHub() error {
func fdwNeedsUpdate() bool {
// check FDW version
versionInfo, err := versionfile.Load()
versionInfo, err := versionfile.LoadDatabaseVersionFile()
if err != nil {
utils.FailOnError(fmt.Errorf("could not verify required FDW version"))
}
@@ -299,7 +299,7 @@ func IsInstalled() bool {
}
func updateDownloadedBinarySignature() error {
versionInfo, err := versionfile.Load()
versionInfo, err := versionfile.LoadDatabaseVersionFile()
if err != nil {
return err
}

1
go.mod
View File

@@ -6,7 +6,6 @@ require (
github.com/Machiel/slugify v1.0.1
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
github.com/ahmetb/go-linq v3.0.0+incompatible
github.com/blang/semver v3.5.1+incompatible
github.com/bmatcuk/doublestar v1.3.4 // indirect
github.com/briandowns/spinner v1.11.1
github.com/c-bata/go-prompt v0.2.5

302
go.sum

File diff suppressed because it is too large Load Diff

View File

@@ -34,7 +34,7 @@ func InstallDB(imageRef string, dest string) (string, error) {
func updateVersionFileDB(image *SteampipeImage) error {
timeNow := versionfile.FormatTime(time.Now())
v, err := versionfile.Load()
v, err := versionfile.LoadDatabaseVersionFile()
if err != nil {
return err
}

View File

@@ -36,7 +36,7 @@ func InstallFdw(imageRef string, dbLocation string) (string, error) {
func updateVersionFileFdw(image *SteampipeImage) error {
timeNow := versionfile.FormatTime(time.Now())
v, err := versionfile.Load()
v, err := versionfile.LoadDatabaseVersionFile()
if err != nil {
return err
}

View File

@@ -43,7 +43,7 @@ func InstallPlugin(imageRef string) (*SteampipeImage, error) {
func updateVersionFilePlugin(image *SteampipeImage) error {
timeNow := versionfile.FormatTime(time.Now())
v, err := versionfile.Load()
v, err := versionfile.LoadPluginVersionFile()
if err != nil {
return err
}

View File

@@ -0,0 +1,111 @@
package versionfile
import (
"encoding/json"
"io/ioutil"
"log"
"os"
"time"
"github.com/turbot/go-kit/helpers"
"github.com/turbot/steampipe/constants"
)
const (
dbVersionFileName = "versions.json"
)
type DatabaseVersionFile struct {
FdwExtension InstalledVersion `json:"fdwExtension"`
EmbeddedDB InstalledVersion `json:"embeddedDB"`
}
func NewDBVersionFile() *DatabaseVersionFile {
return &DatabaseVersionFile{
FdwExtension: InstalledVersion{},
EmbeddedDB: InstalledVersion{},
}
}
type InstalledVersion struct {
Name string `json:"name"`
Version string `json:"version"`
ImageDigest string `json:"imageDigest"`
InstalledFrom string `json:"installedFrom"`
LastCheckedDate string `json:"lastCheckedDate"`
InstallDate string `json:"installDate"`
}
func databaseVersionFileFromLegacy(legacyFile *LegacyVersionFile) *DatabaseVersionFile {
return &DatabaseVersionFile{
FdwExtension: legacyFile.FdwExtension,
EmbeddedDB: legacyFile.EmbeddedDB,
}
}
// LoadDatabaseVersionFile migrates from the old version file format if necessary and loads the database version data
func LoadDatabaseVersionFile() (*DatabaseVersionFile, error) {
// first, see if a migration is necessary - if so, it will return the version data to us
migratedVersionFile, err := MigrateDatabaseVersionFile()
if err != nil {
return nil, err
}
if migratedVersionFile != nil {
log.Println("[TRACE] using migrated database version file")
return migratedVersionFile, nil
}
versionFilePath := constants.DatabaseVersionFilePath()
if helpers.FileExists(versionFilePath) {
return readDatabaseVersionFile(versionFilePath)
}
return NewDBVersionFile(), nil
}
func readDatabaseVersionFile(path string) (*DatabaseVersionFile, error) {
file, _ := ioutil.ReadFile(path)
var data DatabaseVersionFile
if err := json.Unmarshal([]byte(file), &data); err != nil {
log.Println("[ERROR]", "Error while reading DB version file", err)
return nil, err
}
if data.FdwExtension == (InstalledVersion{}) {
data.FdwExtension = InstalledVersion{}
}
if data.EmbeddedDB == (InstalledVersion{}) {
data.EmbeddedDB = InstalledVersion{}
}
return &data, nil
}
// Save writes the config
func (f *DatabaseVersionFile) Save() error {
versionFilePath := constants.DatabaseVersionFilePath()
return f.write(versionFilePath)
}
func (f *DatabaseVersionFile) write(path string) error {
versionFileJSON, err := json.MarshalIndent(f, "", " ")
if err != nil {
log.Println("[ERROR]", "Error while writing version file", err)
return err
}
return os.WriteFile(path, versionFileJSON, 0644)
}
// delete the file on disk if it exists
func (f *DatabaseVersionFile) delete() {
versionFilePath := constants.DatabaseVersionFilePath()
if helpers.FileExists(versionFilePath) {
os.Remove(versionFilePath)
}
}
// FormatTime :: format time as RFC3339 in UTC
func FormatTime(localTime time.Time) string {
loc, _ := time.LoadLocation("UTC")
utcTime := localTime.In(loc)
return (utcTime.Format(time.RFC3339))
}

View File

@@ -6,9 +6,9 @@ import (
"time"
)
func TestWrite(t *testing.T) {
func TestWriteDatabaseVersionFile(t *testing.T) {
var v VersionFile
var v DatabaseVersionFile
fileName := "test.json"
timeNow := time.Now()
@@ -29,36 +29,11 @@ func TestWrite(t *testing.T) {
v.FdwExtension.LastCheckedDate = timeNow2.Format(time.UnixDate)
v.FdwExtension.InstallDate = timeNow2.Format(time.UnixDate)
v.Plugins = make(map[string]*(InstalledVersion))
awsPlugin := InstalledVersion{
Name: "hub.steampipe.io/steampipe/plugin/turbot/aws@latest",
Version: "0.0.3",
ImageDigest: "88995cc15963225884b825b12409f798b24aa7364bbf35a83d3a5fb5db85f346",
InstalledFrom: "hub.steampipe.io/steampipe/plugin/turbot/aws:latest",
LastCheckedDate: timeNow2.Format(time.UnixDate),
InstallDate: timeNow2.Format(time.UnixDate),
}
v.Plugins[awsPlugin.Name] = &awsPlugin
googlePlugin := InstalledVersion{
Name: "hub.steampipe.io/steampipe/plugin/turbot/google@1",
Version: "1.0.7",
ImageDigest: "3211232123654987313216549876516351",
InstalledFrom: "hub.steampipe.io/steampipe/plugin/turbot/google:1",
LastCheckedDate: timeNow2.Format(time.UnixDate),
InstallDate: timeNow2.Format(time.UnixDate),
}
v.Plugins[googlePlugin.Name] = &googlePlugin
//v.Plugins = append(v.Plugins, googlePlugin)
if err := v.write(fileName); err != nil {
t.Errorf("\nError writing file: %s", err.Error())
}
v2, err := read(fileName)
v2, err := readDatabaseVersionFile(fileName)
if err != nil {
t.Errorf("\nError reading file: %s", err.Error())
}
@@ -82,10 +57,5 @@ func TestWrite(t *testing.T) {
t.Errorf("\nError EmbeddedDB.InstallDate is: %s, expected %s", v2.EmbeddedDB.InstallDate, v.EmbeddedDB.InstallDate)
}
if len(v2.Plugins) != 2 {
t.Errorf("\nexpected 2 plugins, found %d", len(v2.Plugins))
}
os.Remove(fileName)
}

View File

@@ -0,0 +1,96 @@
package versionfile
import (
"encoding/json"
"io/ioutil"
"log"
"os"
"github.com/turbot/go-kit/helpers"
"github.com/turbot/steampipe/constants"
)
type LegacyVersionFile struct {
Plugins map[string]*InstalledVersion `json:"plugins"`
FdwExtension InstalledVersion `json:"fdwExtension"`
EmbeddedDB InstalledVersion `json:"embeddedDB"`
}
// LoadLegacyVersionFile loads the legacy version file, or returns nil if it does not exist
func LoadLegacyVersionFile() (*LegacyVersionFile, error) {
versionFilePath := constants.LegacyVersionFilePath()
if helpers.FileExists(versionFilePath) {
return readLegacyVersionFile(versionFilePath)
}
return nil, nil
}
func readLegacyVersionFile(path string) (*LegacyVersionFile, error) {
file, _ := ioutil.ReadFile(path)
var data LegacyVersionFile
if err := json.Unmarshal([]byte(file), &data); err != nil {
log.Println("[ERROR]", "Error while reading version file", err)
return nil, err
}
if data.Plugins == nil {
data.Plugins = map[string]*InstalledVersion{}
}
for key := range data.Plugins {
// hard code the name to the key
data.Plugins[key].Name = key
}
return &data, nil
}
func migrateVersionFiles() (*PluginVersionFile, *DatabaseVersionFile, error) {
legacyVersionFile, err := LoadLegacyVersionFile()
if err != nil {
return nil, nil, err
}
if legacyVersionFile == nil {
return nil, nil, nil
}
log.Printf("[TRACE] migrating version file from '%s' to '%s' and '%s'\n",
constants.LegacyVersionFilePath(),
constants.DatabaseVersionFilePath(),
constants.PluginVersionFilePath())
pluginVersionFile := pluginVersionFileFromLegacy(legacyVersionFile)
databaseVersionFile := databaseVersionFileFromLegacy(legacyVersionFile)
// save the new files and remove the old one
if err := pluginVersionFile.Save(); err != nil {
return nil, nil, err
}
if err := databaseVersionFile.Save(); err != nil {
// delete the plugin version file which we have already saved
pluginVersionFile.delete()
return nil, nil, err
}
legacyVersionFile.delete()
return pluginVersionFile, databaseVersionFile, nil
}
// delete the file on disk if it exists
func (f *LegacyVersionFile) delete() {
versionFilePath := constants.LegacyVersionFilePath()
if helpers.FileExists(versionFilePath) {
os.Remove(versionFilePath)
}
}
func MigrateDatabaseVersionFile() (*DatabaseVersionFile, error) {
_, databaseVersionFile, err := migrateVersionFiles()
return databaseVersionFile, err
}
func MigratePluginVersionFile() (*PluginVersionFile, error) {
pluginVersionFile, _, err := migrateVersionFiles()
return pluginVersionFile, err
}

View File

@@ -0,0 +1,91 @@
package versionfile
import (
"encoding/json"
"io/ioutil"
"log"
"os"
"github.com/turbot/go-kit/helpers"
"github.com/turbot/steampipe/constants"
)
type PluginVersionFile struct {
Plugins map[string]*InstalledVersion `json:"plugins"`
}
func NewPluginVersionFile() *PluginVersionFile {
return &PluginVersionFile{
Plugins: map[string]*InstalledVersion{},
}
}
func pluginVersionFileFromLegacy(legacyFile *LegacyVersionFile) *PluginVersionFile {
return &PluginVersionFile{
Plugins: legacyFile.Plugins,
}
}
// LoadPluginVersionFile migrates from the old version file format if necessary and loads the plugin version data
func LoadPluginVersionFile() (*PluginVersionFile, error) {
// first, see if a migration is necessary - if so, it will return the version data to us
migratedVersionFile, err := MigratePluginVersionFile()
if err != nil {
return nil, err
}
if migratedVersionFile != nil {
log.Println("[TRACE] using migrated plugin version file")
return migratedVersionFile, nil
}
versionFilePath := constants.PluginVersionFilePath()
if helpers.FileExists(versionFilePath) {
return readPluginVersionFile(versionFilePath)
}
return NewPluginVersionFile(), nil
}
func readPluginVersionFile(path string) (*PluginVersionFile, error) {
file, _ := ioutil.ReadFile(path)
var data PluginVersionFile
if err := json.Unmarshal([]byte(file), &data); err != nil {
log.Println("[ERROR]", "Error while reading plugin version file", err)
return nil, err
}
if data.Plugins == nil {
data.Plugins = map[string]*InstalledVersion{}
}
for key := range data.Plugins {
// hard code the name to the key
data.Plugins[key].Name = key
}
return &data, nil
}
// Save writes the config file to disk
func (f *PluginVersionFile) Save() error {
versionFilePath := constants.PluginVersionFilePath()
return f.write(versionFilePath)
}
func (f *PluginVersionFile) write(path string) error {
versionFileJSON, err := json.MarshalIndent(f, "", " ")
if err != nil {
log.Println("[ERROR]", "Error while writing version file", err)
return err
}
return os.WriteFile(path, versionFileJSON, 0644)
}
// delete the file on disk if it exists
func (f *PluginVersionFile) delete() {
versionFilePath := constants.PluginVersionFilePath()
if helpers.FileExists(versionFilePath) {
os.Remove(versionFilePath)
}
}

View File

@@ -0,0 +1,52 @@
package versionfile
import (
"os"
"testing"
"time"
)
func TestWrite(t *testing.T) {
var v PluginVersionFile
fileName := "test.json"
timeNow := time.Now()
timeNow2 := timeNow.Add(time.Minute * 10)
v.Plugins = make(map[string]*(InstalledVersion))
awsPlugin := InstalledVersion{
Name: "hub.steampipe.io/steampipe/plugin/turbot/aws@latest",
Version: "0.0.3",
ImageDigest: "88995cc15963225884b825b12409f798b24aa7364bbf35a83d3a5fb5db85f346",
InstalledFrom: "hub.steampipe.io/steampipe/plugin/turbot/aws:latest",
LastCheckedDate: timeNow2.Format(time.UnixDate),
InstallDate: timeNow2.Format(time.UnixDate),
}
v.Plugins[awsPlugin.Name] = &awsPlugin
googlePlugin := InstalledVersion{
Name: "hub.steampipe.io/steampipe/plugin/turbot/google@1",
Version: "1.0.7",
ImageDigest: "3211232123654987313216549876516351",
InstalledFrom: "hub.steampipe.io/steampipe/plugin/turbot/google:1",
LastCheckedDate: timeNow2.Format(time.UnixDate),
InstallDate: timeNow2.Format(time.UnixDate),
}
v.Plugins[googlePlugin.Name] = &googlePlugin
if err := v.write(fileName); err != nil {
t.Errorf("\nError writing file: %s", err.Error())
}
v2, err := readPluginVersionFile(fileName)
if err != nil {
t.Errorf("\nError reading file: %s", err.Error())
}
if len(v2.Plugins) != 2 {
t.Errorf("\nexpected 2 plugins, found %d", len(v2.Plugins))
}
os.Remove(fileName)
}

View File

@@ -1,104 +0,0 @@
package versionfile
import (
"encoding/json"
"io/ioutil"
"log"
"os"
"path/filepath"
"time"
"github.com/turbot/steampipe/constants"
)
const (
versionFileName = "versions.json"
)
type VersionFile struct {
Plugins map[string]*InstalledVersion `json:"plugins"`
FdwExtension InstalledVersion `json:"fdwExtension"`
EmbeddedDB InstalledVersion `json:"embeddedDB"`
}
func NewVersionFile() *VersionFile {
return &VersionFile{
Plugins: map[string]*InstalledVersion{},
}
}
type InstalledVersion struct {
Name string `json:"name"`
Version string `json:"version"`
ImageDigest string `json:"imageDigest"`
InstalledFrom string `json:"installedFrom"`
LastCheckedDate string `json:"lastCheckedDate"`
InstallDate string `json:"installDate"`
}
// Load :: load the config file from the standard path, or return an empty one
func Load() (*VersionFile, error) {
versionFilePath := versionFileLocation()
if fileExists(versionFilePath) {
return read(versionFilePath)
}
return NewVersionFile(), nil
}
// Save :: Save the config file from the standard path, or return an empty one
func (f *VersionFile) Save() error {
versionFilePath := versionFileLocation()
return f.write(versionFilePath)
}
func fileExists(filename string) bool {
info, err := os.Stat(filename)
if os.IsNotExist(err) {
return false
}
return !info.IsDir()
}
func (f *VersionFile) write(path string) error {
versionFileJSON, err := json.MarshalIndent(f, "", " ")
if err != nil {
log.Println("[ERROR]", "Error while writing version file", err)
return err
}
return ioutil.WriteFile(path, versionFileJSON, 0644)
}
func read(path string) (*VersionFile, error) {
file, _ := ioutil.ReadFile(path)
var data VersionFile
if err := json.Unmarshal([]byte(file), &data); err != nil {
log.Println("[ERROR]", "Error while reading version file", err)
return nil, err
}
if data.Plugins == nil {
data.Plugins = map[string]*InstalledVersion{}
}
for key := range data.Plugins {
// hard code the name to the key
data.Plugins[key].Name = key
}
return &data, nil
}
// ex: $CONFIG_DIR/plugins/hub.steampipe.io/turbot/aws/1.1.2/steampipe-plugin-aws
func versionFileLocation() string {
path := filepath.Join(constants.InternalDir(), versionFileName)
return path
}
// FormatTime :: format time as RFC3339 in UTC
func FormatTime(localTime time.Time) string {
loc, _ := time.LoadLocation("UTC")
utcTime := localTime.In(loc)
return (utcTime.Format(time.RFC3339))
}

View File

@@ -43,7 +43,7 @@ func Remove(image string, pluginConnections map[string][]string) error {
}
// update the version file
v, err := versionfile.Load()
v, err := versionfile.LoadPluginVersionFile()
if err != nil {
return err
}
@@ -53,7 +53,7 @@ func Remove(image string, pluginConnections map[string][]string) error {
// Exists looks up the version file and reports whether a plugin is already installed
func Exists(plugin string) (bool, error) {
versionData, err := versionfile.Load()
versionData, err := versionfile.LoadPluginVersionFile()
if err != nil {
return false, err
}
@@ -95,7 +95,7 @@ func List(pluginConnectionMap map[string][]string) ([]PluginListItem, error) {
return nil
})
v, err := versionfile.Load()
v, err := versionfile.LoadPluginVersionFile()
if err != nil {
return nil, err
}

View File

@@ -60,7 +60,7 @@ func GetAllUpdateReport(installationID string) map[string]VersionCheckReport {
versionChecker.signature = installationID
versionChecker.pluginsToCheck = []*versionfile.InstalledVersion{}
versionFileData, err := versionfile.Load()
versionFileData, err := versionfile.LoadPluginVersionFile()
if err != nil {
log.Println("[TRACE]", "CheckAndReportPluginUpdates", "could not load versionfile")
return nil
@@ -76,7 +76,7 @@ func GetAllUpdateReport(installationID string) map[string]VersionCheckReport {
}
func (v *VersionChecker) reportPluginUpdates() map[string]VersionCheckReport {
versionFileData, err := versionfile.Load()
versionFileData, err := versionfile.LoadPluginVersionFile()
if err != nil {
log.Println("[TRACE]", "CheckAndReportPluginUpdates", "could not load versionfile")
return nil