Install from file (#238)

cmd/qliksense/apply.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/qlik-oss/sense-installer/pkg/qliksense"
+	"github.com/spf13/cobra"
+)
+
+func applyCmd(q *qliksense.Qliksense) *cobra.Command {
+
+	filePath := ""
+	c := &cobra.Command{
+		Use:     "apply",
+		Short:   "install qliksense based on provided cr file",
+		Long:    `install qliksense based on provided cr file`,
+		Example: `qliksense apply -f file_name or cat cr_file | qliksense apply -f -`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if filePath == "-" {
+				if !isInputFromPipe() {
+					return errors.New("No input pipe present")
+				}
+				return q.ApplyCRFromReader(os.Stdin)
+			}
+			file, e := os.Open(filePath)
+			if e != nil {
+				return errors.Wrapf(e,
+					"unable to read the file %s", filePath)
+			}
+			return q.ApplyCRFromReader(file)
+		},
+	}
+
+	f := c.Flags()
+	f.StringVarP(&filePath, "file", "f", "", "Install from a CR file")
+	c.MarkFlagRequired("file")
+	return c
+}

cmd/qliksense/load.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/qlik-oss/sense-installer/pkg/qliksense"
+	"github.com/spf13/cobra"
+)
+
+func loadCrFile(q *qliksense.Qliksense) *cobra.Command {
+	filePath := ""
+	c := &cobra.Command{
+		Use:   "load",
+		Short: "load a CR a file and create necessary structure for future use",
+		Long:  `load a CR a file and create necessary structure for future use`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if filePath == "-" {
+				if !isInputFromPipe() {
+					return errors.New("No input pipe present")
+				}
+				return q.LoadCr(os.Stdin)
+			}
+			file, e := os.Open(filePath)
+			if e != nil {
+				return errors.Wrapf(e,
+					"unable to read the file %s", filePath)
+			}
+			return q.LoadCr(file)
+		},
+	}
+	f := c.Flags()
+	f.StringVarP(&filePath, "file", "f", "", "File to laod CR from")
+	c.MarkFlagRequired("file")
+	return c
+}
+
+func isInputFromPipe() bool {
+	fileInfo, _ := os.Stdin.Stat()
+	return fileInfo.Mode()&os.ModeCharDevice == 0
+}

cmd/qliksense/root.go

@@ -192,7 +192,8 @@ func rootCmd(p *qliksense.Qliksense) *cobra.Command {
 	//preflightCmd.AddCommand(preflightCheckAllCmd(p))
 
 	cmd.AddCommand(preflightCmd)
-
+	cmd.AddCommand(loadCrFile(p))
+	cmd.AddCommand((applyCmd(p)))
 	return cmd
 }
 

docs/command_reference.md

@@ -0,0 +1,41 @@
+# qliksense command reference
+
+## qliksense apply
+
+`qliksense apply` command takes input a cr file or input from pipe
+
+- `qliksense apply -f cr-file.yaml`
+- `cat cr-file.yaml | qliksense apply -f -`
+
+the content of `cr-file.yaml` should be something similar
+
+```yaml
+apiVersion: qlik.com/v1
+kind: Qliksense
+metadata:
+  name: qlik-test
+  labels:
+    version: v0.0.2
+spec:
+  configs:
+    qliksense:
+    - name: acceptEULA
+      value: "yes"
+  secrets:
+    qliksense:
+    - name: mongoDbUri
+      value: mongodb://qlik-test-mongodb:27017/qliksense?ssl=false
+  profile: docker-desktop
+  rotateKeys: "yes"
+  ```
+
+after doing one of the above commands, cli will set the current context to the cr name and install the qliksense into the cluster. so make sure you dont have a context (cr name = context name) with the same name already. It will though error if it same context name already exist. It will encrypt the secrets from CR while writing CR into the disk.
+
+## qliksense load
+
+`qliksense load` command takes input a cr file or input from pipe.
+
+- `qliksense load -f cr-file.yaml`
+- `cat cr-file.yaml | qliksense load -f -`
+
+It will load the cr into `~/.qliksense` folder and create context strcture with for the CR and set the current context with the cr name. In case of multiple documents in a single stream/file it will set the current context to the last one. It will encrypt the secrets from CR while writing the CR into the disk.

mkdocs.yml

@@ -17,6 +17,7 @@ markdown_extensions:
 nav:
   - Overview: index.md
   - getting_started.md
+  - command_reference.md
   - concepts.md
   - air_gap.md
   - Releases ⧉: https://github.com/qlik-oss/sense-installer/releases

pkg/api/apis.go

@@ -54,11 +54,7 @@ func (qc *QliksenseConfig) GetCR(contextName string) (*QliksenseCR, error) {
 	if crFilePath == "" {
 		return nil, errors.New("context name " + contextName + " not found")
 	}
-	return getCRObject(crFilePath)
-}
-
-func getUnencryptedCR() {
-
+	return GetCRObject(crFilePath)
 }
 
 // GetCurrentCR create a QliksenseCR object for current context
@@ -85,7 +81,8 @@ func (qc *QliksenseConfig) SetCrLocation(contextName, filepath string) (*Qliksen
 	return nil, errors.New("cannot find the context")
 }
 
-func getCRObject(crfile string) (*QliksenseCR, error) {
+// GetCRObject create a qliksense CR object from file
+func GetCRObject(crfile string) (*QliksenseCR, error) {
 	cr := &QliksenseCR{}
 	err := ReadFromFile(cr, crfile)
 	if err != nil {
@@ -96,6 +93,20 @@ func getCRObject(crfile string) (*QliksenseCR, error) {
 	return cr, nil
 }
 
+//CreateCRObjectFromString create a QliksenseCR from string content
+func CreateCRObjectFromString(crContent string) (*QliksenseCR, error) {
+	if crContent == "" {
+		return nil, errors.New("empty string cannot qliksensecr")
+	}
+	cr := &QliksenseCR{}
+	err := ReadFromStream(cr, strings.NewReader(crContent))
+	if err != nil {
+		fmt.Println("cannot unmarshal cr ", err)
+		return nil, err
+	}
+	return cr, nil
+}
+
 func (qc *QliksenseConfig) getCRFilePath(contextName string) string {
 	crFilePath := ""
 	for _, ctx := range qc.Spec.Contexts {
@@ -230,6 +241,15 @@ func (qc *QliksenseConfig) getDockerConfigJsonSecret(name string) (*DockerConfig
 }
 
 func (qc *QliksenseConfig) getCurrentContextEncryptionKeyPairLocation() (string, error) {
+
+	if qcr, err := qc.GetCurrentCR(); err != nil {
+		return "", err
+	} else {
+		return qc.getContextEncryptionKeyPairLocation(qcr.GetName())
+	}
+}
+
+func (qc *QliksenseConfig) getContextEncryptionKeyPairLocation(contextName string) (string, error) {
 	// Check env var: QLIKSENSE_KEY_LOCATION to determine location to store keypair
 	var secretKeyPairLocation string
 	if os.Getenv("QLIKSENSE_KEY_LOCATION") != "" {
@@ -238,13 +258,9 @@ func (qc *QliksenseConfig) getCurrentContextEncryptionKeyPairLocation() (string,
 	} else {
 		// QLIKSENSE_KEY_LOCATION has not been set, hence storing key pair in default location:
 		// /.qliksense/secrets/contexts/<current-context>/secrets/
-		if qcr, err := qc.GetCurrentCR(); err != nil {
-			return "", err
-		} else {
-			secretKeyPairLocation = filepath.Join(qc.QliksenseHomePath, qliksenseSecretsDirName, qliksenseContextsDirName, qcr.GetObjectMeta().GetName(), qliksenseSecretsDirName)
-		}
+		secretKeyPairLocation = filepath.Join(qc.QliksenseHomePath, qliksenseSecretsDirName, qliksenseContextsDirName, contextName, qliksenseSecretsDirName)
+
 	}
-	LogDebugMessage("SecretKeyLocation to store key pair: %s", secretKeyPairLocation)
 	return secretKeyPairLocation, nil
 }
 
@@ -261,7 +277,15 @@ func (qc *QliksenseConfig) GetCurrentContextEjsonKeyDir() (string, error) {
 }
 
 func (qc *QliksenseConfig) GetCurrentContextEncryptionKeyPair() (*rsa.PublicKey, *rsa.PrivateKey, error) {
-	secretKeyPairLocation, err := qc.getCurrentContextEncryptionKeyPairLocation()
+	if qcr, err := qc.GetCurrentCR(); err != nil {
+		return nil, nil, err
+	} else {
+		return qc.GetContextEncryptionKeyPair(qcr.GetName())
+	}
+}
+
+func (qc *QliksenseConfig) GetContextEncryptionKeyPair(contextName string) (*rsa.PublicKey, *rsa.PrivateKey, error) {
+	secretKeyPairLocation, err := qc.getContextEncryptionKeyPairLocation(contextName)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -386,3 +410,36 @@ func (qc *QliksenseConfig) GetDecryptedCr(cr *QliksenseCR) (*QliksenseCR, error)
 	newCr.Spec.Secrets = finalSecrets
 	return newCr, nil
 }
+
+//Validate validate CR
+func (cr *QliksenseCR) Validate() bool {
+	return true
+}
+
+//CreateContextDirs create context dir structure ~/.qliksense/contexts/contextName
+func (qc *QliksenseConfig) CreateContextDirs(contextName string) {
+	contexPath := filepath.Join(qc.QliksenseHomePath, qliksenseContextsDirName, contextName)
+	os.MkdirAll(contexPath, os.ModePerm)
+}
+
+func (qc *QliksenseConfig) BuildCrFilePath(contextName string) string {
+	return filepath.Join(qc.QliksenseHomePath, qliksenseContextsDirName, contextName, contextName+".yaml")
+}
+
+//AddToContexts add the context into qc.Spec.Contexts
+func (qc *QliksenseConfig) AddToContexts(crName, crFile string) {
+	qc.Spec.Contexts = append(qc.Spec.Contexts, []Context{
+		{CrFile: crFile,
+			Name: crName},
+	}...)
+}
+
+//SetCurrentContextName set the qc.Spec.CurrentContext
+func (qc *QliksenseConfig) SetCurrentContextName(name string) {
+	qc.Spec.CurrentContext = name
+}
+
+//Write write QliksenseConfig into config.yaml
+func (qc *QliksenseConfig) Write() error {
+	return WriteToFile(qc, filepath.Join(qc.QliksenseHomePath, "config.yaml"))
+}

pkg/api/context_apis.go

@@ -3,8 +3,10 @@ package api
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
+	"os"
 
 	"github.com/qlik-oss/k-apis/pkg/config"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -103,14 +105,22 @@ func ReadFromFile(content interface{}, sourceFile string) error {
 	if content == nil || sourceFile == "" {
 		return nil
 	}
-	contents, err := ioutil.ReadFile(sourceFile)
+	file, e := os.Open(sourceFile)
+	if e != nil {
+		return e
+	}
+	return ReadFromStream(content, file)
+}
+
+// ReadFromStream reads from input stream and creat yaml struct of type content
+func ReadFromStream(content interface{}, reader io.Reader) error {
+	contents, err := ioutil.ReadAll(reader)
 	if err != nil {
-		err = fmt.Errorf("There was an error reading from file: %s, %v", sourceFile, err)
+		err = fmt.Errorf("There was an error reading from reader: %v", err)
 		return err
 	}
 	// reading k8s style object
 	// https://stackoverflow.com/questions/44306554/how-to-deserialize-kubernetes-yaml-file
 	dec := machine_yaml.NewYAMLOrJSONDecoder(bytes.NewReader(contents), 10000)
-	dec.Decode(content)
-	return nil
+	return dec.Decode(content)
 }

pkg/qliksense/apply.go

@@ -0,0 +1,23 @@
+package qliksense
+
+import (
+	"io"
+
+	qapi "github.com/qlik-oss/sense-installer/pkg/api"
+)
+
+func (q *Qliksense) ApplyCRFromReader(r io.Reader) error {
+	if err := q.LoadCr(r); err != nil {
+		return err
+	}
+	qConfig := qapi.NewQConfig(q.QliksenseHome)
+	cr, err := qConfig.GetCurrentCR()
+	if err != nil {
+		return err
+	}
+	opts := &InstallCommandOptions{}
+	if err := q.InstallQK8s(cr.GetLabelFromCr("version"), opts, true); err != nil {
+		return err
+	}
+	return nil
+}

pkg/qliksense/context_configs_test.go

@@ -152,6 +152,7 @@ func removePrivateKey() {
 
 func setup() func() {
 	// create tests dir
+	os.RemoveAll(testDir)
 	if err := os.Mkdir(testDir, 0777); err != nil {
 		log.Printf("\nError occurred: %v", err)
 	}
@@ -164,7 +165,7 @@ metadata:
 spec:
   contexts:
   - name: qlik-default
-    crFile: /root/.qliksense/contexts/qlik-default.yaml
+    crFile: ./tests/contexts/qlik-default/qlik-default.yaml
   currentContext: qlik-default
 `
 	configFile := filepath.Join(testDir, "config.yaml")

pkg/qliksense/install_test.go

@@ -0,0 +1,65 @@
+package qliksense
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	qapi "github.com/qlik-oss/sense-installer/pkg/api"
+)
+
+func TestCreateK8sResoruceBeforePatch(t *testing.T) {
+	td := setup()
+	sampleCr := `
+apiVersion: qlik.com/v1
+kind: Qliksense
+metadata:
+  name: qlik-test3
+  labels:
+    version: v0.0.2
+spec:
+  git:
+    repository: https://github.com/ffoysal/qliksense-k8s
+    accessToken: abababababababaab
+    userName: "blblbl"
+  gitOps:
+    enabled: "no"
+    schedule: "*/1 * * * *"
+    watchBranch: pr-branch-db1d26d6
+    image: qlik-docker-oss.bintray.io/qliksense-repo-watcher
+  configs:
+    qliksense:
+    - name: acceptEULA
+      value: "yes"
+  secrets:
+    qliksense:
+    - name: mongoDbUri
+      value: mongodb://qlik-default-mongodb:27017/qliksense?ssl=false
+  profile: docker-desktop
+  rotateKeys: "yes"`
+
+	crFile := filepath.Join(testDir, "install_test.yaml")
+	ioutil.WriteFile(crFile, []byte(sampleCr), 0644)
+	q := New(testDir)
+	file, e := os.Open(crFile)
+	if e != nil {
+		t.Log(e)
+		t.FailNow()
+	}
+	if err := q.LoadCr(file); err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	qConfig := qapi.NewQConfig(testDir)
+	cr, err := qConfig.GetCR("qlik-test3")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	if err = q.createK8sResoruceBeforePatch(cr); err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	td()
+}

pkg/qliksense/load_cr.go

@@ -0,0 +1,83 @@
+package qliksense
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+
+	qapi "github.com/qlik-oss/sense-installer/pkg/api"
+)
+
+//
+func (q *Qliksense) LoadCr(reader io.Reader) error {
+	for _, doc := range readMultipleYamlFromReader(reader) {
+		if crName, err := q.loadCrStringIntoFileSystem(doc); err != nil {
+			return err
+		} else {
+			fmt.Println("cr name: [ " + crName + " ] has been loaded")
+		}
+	}
+	return nil
+}
+
+func (q *Qliksense) loadCrStringIntoFileSystem(crstr string) (string, error) {
+	cr, err := qapi.CreateCRObjectFromString(crstr)
+	if err != nil {
+		return "", err
+	}
+	qConfig := qapi.NewQConfig(q.QliksenseHome)
+	if qConfig.IsContextExist(cr.GetName()) {
+		return "", errors.New("Context Name: " + cr.GetName() + " already exist. please delete the existing context first using delete-context command")
+	}
+	qConfig.CreateContextDirs(cr.GetName())
+
+	// encrypt the secrets and do base64 then update the CR
+	rsaPublicKey, _, err := qConfig.GetContextEncryptionKeyPair(cr.GetName())
+	if err != nil {
+		return "", err
+	}
+	for svc, nvs := range cr.Spec.Secrets {
+		for _, nv := range nvs {
+			if nv.ValueFrom == nil {
+				skv := &qapi.ServiceKeyValue{
+					Key:     nv.Name,
+					Value:   nv.Value,
+					SvcName: svc,
+				}
+				if err := q.processSecret(skv, rsaPublicKey, cr, false); err != nil {
+					return cr.GetName(), err
+				}
+			}
+		}
+	}
+
+	// write to disk
+	if err = qapi.WriteToFile(cr, qConfig.BuildCrFilePath(cr.GetName())); err != nil {
+		return "", err
+	}
+	qConfig.AddToContexts(cr.GetName(), qConfig.BuildCrFilePath(cr.GetName()))
+	qConfig.SetCurrentContextName(cr.GetName())
+	qConfig.Write()
+
+	return cr.GetName(), nil
+}
+
+func readMultipleYamlFromReader(reader io.Reader) []string {
+	docs := make([]string, 0)
+	scanner := bufio.NewScanner(bufio.NewReader(reader))
+	adoc := ""
+	for scanner.Scan() {
+		s := scanner.Text()
+		if s == "---" {
+			docs = append(docs, adoc)
+			adoc = ""
+			s = ""
+		}
+		adoc = adoc + "\n" + s
+	}
+	if adoc != "" {
+		docs = append(docs, adoc)
+	}
+	return docs
+}

pkg/qliksense/load_cr_test.go

@@ -0,0 +1,129 @@
+package qliksense
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	qapi "github.com/qlik-oss/sense-installer/pkg/api"
+)
+
+func TestLoadCrFile(t *testing.T) {
+	td := setup()
+	setup()
+	sampleCr := `
+apiVersion: qlik.com/v1
+kind: Qliksense
+metadata:
+  name: qlik-test
+  labels:
+    version: v0.0.2
+spec:
+  git:
+    repository: https://github.com/ffoysal/qliksense-k8s
+    accessToken: abababababababaab
+    userName: "blblbl"
+  gitOps:
+    enabled: "no"
+    schedule: "*/1 * * * *"
+    watchBranch: pr-branch-db1d26d6
+    image: qlik-docker-oss.bintray.io/qliksense-repo-watcher
+  configs:
+    qliksense:
+    - name: acceptEULA
+      value: "yes"
+  secrets:
+    qliksense:
+    - name: mongoDbUri
+      value: mongodb://qlik-default-mongodb:27017/qliksense?ssl=false
+  profile: docker-desktop
+  rotateKeys: "yes"
+---
+apiVersion: qlik.com/v1
+kind: Qliksense
+metadata:
+  name: qlik-test3
+  labels:
+    version: v0.0.2
+spec:
+  git:
+    repository: https://github.com/ffoysal/qliksense-k8s
+    accessToken: abababababababaab
+    userName: "blblbl"
+  gitOps:
+    enabled: "no"
+    schedule: "*/1 * * * *"
+    watchBranch: pr-branch-db1d26d6
+    image: qlik-docker-oss.bintray.io/qliksense-repo-watcher
+  configs:
+    qliksense:
+    - name: acceptEULA
+      value: "yes"
+  secrets:
+    qliksense:
+    - name: mongoDbUri
+      value: mongodb://qlik-default-mongodb:27017/qliksense?ssl=false
+  profile: docker-desktop
+  rotateKeys: "yes"`
+
+	duplicateCr := `
+apiVersion: qlik.com/v1
+kind: Qliksense
+metadata:
+  name: qlik-default
+  labels:
+    version: v0.0.2
+spec:
+  git:
+    repository: https://github.com/ffoysal/qliksense-k8s
+    accessToken: abababababababaab
+    userName: "blblbl"`
+	crFile := filepath.Join(testDir, "testcr.yaml")
+	ioutil.WriteFile(crFile, []byte(sampleCr), 0644)
+
+	dupCrFile := filepath.Join(testDir, "dupcr.yaml")
+	ioutil.WriteFile(dupCrFile, []byte(duplicateCr), 0644)
+
+	q := New(testDir)
+	file, e := os.Open(crFile)
+	if e != nil {
+		t.Log(e)
+		t.FailNow()
+	}
+	if err := q.LoadCr(file); err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	qConfig := qapi.NewQConfig(testDir)
+	cr, err := qConfig.GetCR("qlik-test")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	if cr.GetName() != "qlik-test" {
+		t.FailNow()
+	}
+
+	cr, err = qConfig.GetCR("qlik-test3")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	if cr.GetName() != "qlik-test3" {
+		t.FailNow()
+	}
+
+	if qConfig.Spec.CurrentContext != "qlik-test3" {
+		t.FailNow()
+	}
+	file, e = os.Open(dupCrFile)
+	if e != nil {
+		t.Log(e)
+		t.FailNow()
+	}
+	if err := q.LoadCr(file); err == nil {
+		t.FailNow()
+	}
+	td()
+}