Merge pull request #82 from qlik-oss/fix_Makefile

Make makefile portable

.gitignore

@@ -1,2 +1,6 @@
 bin
 .vscode
+cmd/qliksense/__debug_bin
+pkg/qliksense/crds
+pkg/qliksense/packrd
+pkg/qliksense/qliksense-packr.go

Makefile

@@ -56,7 +56,7 @@ $(BINDIR)/$(VERSION)/$(MIXIN)-$(CLIENT_PLATFORM)-$(CLIENT_ARCH)$(FILE_EXT):
 generate: get-crds packr2
 	go generate ./...
 
-HAS_PACKR2 := $(shell command -v packr2)
+HAS_PACKR2 := $(shell packr2)
 packr2:
 ifndef HAS_PACKR2
 	go get -u github.com/gobuffalo/packr/v2/packr2
@@ -69,11 +69,12 @@ clean: clean-packr
 clean-packr: packr2
 	cd pkg/qliksense && packr2 clean
 
-get-crds: 
-	git clone --depth=1 git@github.com:qlik-oss/qliksense-operator.git -b ms-3 /tmp/operator
+get-crds:
+	$(eval TMP := $(shell mktemp -d))
+	git clone git@github.com:qlik-oss/qliksense-operator.git -b ms-3 $(TMP)/operator
 	mkdir -p pkg/qliksense/crds/cr
 	mkdir -p pkg/qliksense/crds/crd
 	mkdir -p pkg/qliksense/crds/crd-deploy
-	cp /tmp/operator/deploy/*.yaml pkg/qliksense/crds/crd-deploy
-	cp /tmp/operator/deploy/crds/*_crd.yaml pkg/qliksense/crds/crd
-	cp /tmp/operator/deploy/crds/*_cr.yaml pkg/qliksense/crds/cr
+	cp $(TMP)/operator/deploy/*.yaml pkg/qliksense/crds/crd-deploy
+	cp $(TMP)/operator/deploy/crds/*_crd.yaml pkg/qliksense/crds/crd
+	cp $(TMP)/operator/deploy/crds/*_cr.yaml pkg/qliksense/crds/cr

cmd/qliksense/fetch.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"errors"
-	"github.com/Masterminds/semver/v3"
 	"github.com/qlik-oss/sense-installer/pkg/qliksense"
 	"github.com/spf13/cobra"
 )
@@ -17,9 +16,6 @@ func fetchCmd(q *qliksense.Qliksense) *cobra.Command {
 			if len(args) != 1 {
 				return errors.New("requires a version (i.e. v1.0.0)")
 			}
-			if _, err := semver.NewVersion(args[0]); err != nil {
-				return errors.New("is it not a valid version. should be something like this v1.0.0")
-			}
 			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {

cmd/qliksense/install.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"errors"
-	"github.com/Masterminds/semver/v3"
 	"github.com/qlik-oss/sense-installer/pkg/qliksense"
 	"github.com/spf13/cobra"
 )
@@ -18,9 +17,6 @@ func installCmd(q *qliksense.Qliksense) *cobra.Command {
 			if len(args) != 1 {
 				return errors.New("requires a version (i.e. v1.0.0)")
 			}
-			if _, err := semver.NewVersion(args[0]); err != nil {
-				return errors.New("is it not a valid version. should be something like this v1.0.0")
-			}
 			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -32,6 +28,7 @@ func installCmd(q *qliksense.Qliksense) *cobra.Command {
 	f.StringVarP(&opts.AcceptEULA, "acceptEULA", "a", "", "AcceptEULA for qliksense")
 	f.StringVarP(&opts.Namespace, "namespace", "n", "", "Namespace where to install the qliksense")
 	f.StringVarP(&opts.StorageClass, "storageClass", "s", "", "Storage class for qliksense")
-
+	f.StringVarP(&opts.MongoDbUri, "mongoDbUri", "m", "", "mongoDbUri for qliksense (i.e. mongodb://qlik-default-mongodb:27017/qliksense?ssl=false)")
+	f.StringVarP(&opts.RotateKeys, "rotateKeys", "r", "", "Rotate JWT keys for qliksense (yes:rotate keys/ no:use exising keys from cluster/ None: use default EJSON_KEY from env")
 	return c
 }

cmd/qliksense/root.go

@@ -41,7 +41,9 @@ func initAndExecute() error {
 	qliksense.LogDebugMessage("QliksenseHomeDir: %s", qlikSenseHome)
 	qliksense.SetUpQliksenseDefaultContext(qlikSenseHome)
 
-	if err = rootCmd(qliksense.New(qlikSenseHome)).Execute(); err != nil {
+	if qliksenseClient, err := qliksense.New(qlikSenseHome); err != nil {
+		return err
+	} else if err := rootCmd(qliksenseClient).Execute(); err != nil {
 		return err
 	}
 
@@ -63,7 +65,11 @@ func setUpPaths() (string, error) {
 		}
 		qlikSenseHome = filepath.Join(homeDir, qlikSenseDirVar)
 	}
-	os.Mkdir(qlikSenseHome, os.ModePerm)
+
+	if err := os.MkdirAll(qlikSenseHome, os.ModePerm); err != nil {
+		return "", err
+	}
+
 	return qlikSenseHome, nil
 }
 

go.mod

@@ -16,13 +16,14 @@ replace (
 	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20191004115801-a2eda9f80ab8
 	k8s.io/client-go => k8s.io/client-go v0.0.0-20191016111102-bec269661e48
 	k8s.io/kubectl => k8s.io/kubectl v0.0.0-20191016120415-2ed914427d51
-	sigs.k8s.io/kustomize/api => github.com/qlik-oss/kustomize/api v0.3.3-0.20200129153315-09eb26c762c8
+	sigs.k8s.io/kustomize/api => github.com/qlik-oss/kustomize/api v0.3.3-0.20200206224201-2e697eccbad9
 )
 
 require (
 	cloud.google.com/go v0.52.0 // indirect
 	cloud.google.com/go/storage v1.5.0 // indirect
 	github.com/Masterminds/semver/v3 v3.0.3
+	github.com/Shopify/ejson v1.2.1
 	github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
 	github.com/aws/aws-sdk-go v1.28.9 // indirect
 	github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 // indirect
@@ -52,7 +53,7 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/pkg/errors v0.8.1
-	github.com/qlik-oss/k-apis v0.0.4
+	github.com/qlik-oss/k-apis v0.0.5
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/viper v1.6.1
@@ -67,6 +68,7 @@ require (
 	gopkg.in/fatih/pool.v2 v2.0.0 // indirect
 	gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect
 	gopkg.in/yaml.v2 v2.2.8
+	gopkg.in/yaml.v3 v3.0.0-20190924164351-c8b7dadae555
 	sigs.k8s.io/kustomize/api v0.3.2
 	vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 // indirect
 )

go.sum

@@ -909,8 +909,12 @@ github.com/qlik-oss/k-apis v0.0.3 h1:LJTQik87Rcsl+rphXfyPtxQc9UgQy+XGdT+epQqi+YY
 github.com/qlik-oss/k-apis v0.0.3/go.mod h1:KOFzKVIdRqp47ytnHg3+9zb8fTlnrQjO6aKiwcrCJUE=
 github.com/qlik-oss/k-apis v0.0.4 h1:fNX1LsGbNZtR7X/0o/2HAnQkuEJs6JcnedHzacBcbfM=
 github.com/qlik-oss/k-apis v0.0.4/go.mod h1:KOFzKVIdRqp47ytnHg3+9zb8fTlnrQjO6aKiwcrCJUE=
+github.com/qlik-oss/k-apis v0.0.5 h1:CpiujicAo+clZqy7Pe3CDAqNhTx8cCC3qmzz3ovG7OU=
+github.com/qlik-oss/k-apis v0.0.5/go.mod h1:KOFzKVIdRqp47ytnHg3+9zb8fTlnrQjO6aKiwcrCJUE=
 github.com/qlik-oss/kustomize/api v0.3.3-0.20200129153315-09eb26c762c8 h1:WLVkArXf58T+3SHDvSddE8P6OjkeeUtGEgHk8LDdfeo=
 github.com/qlik-oss/kustomize/api v0.3.3-0.20200129153315-09eb26c762c8/go.mod h1:OCt7FTrRVHj4kmR2xLJJUIqu00BTr6GeF09hSmM17Kw=
+github.com/qlik-oss/kustomize/api v0.3.3-0.20200206224201-2e697eccbad9 h1:iqeqTS4zjp6rPEaxmFB7pemA2CMjOEN5dYSXZaQ82uw=
+github.com/qlik-oss/kustomize/api v0.3.3-0.20200206224201-2e697eccbad9/go.mod h1:OCt7FTrRVHj4kmR2xLJJUIqu00BTr6GeF09hSmM17Kw=
 github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
 github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be h1:ta7tUOvsPHVHGom5hKW5VXNc2xZIkfCKP8iaqOyYtUQ=
 github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be/go.mod h1:MIDFMn7db1kT65GmV94GzpX9Qdi7N/pQlwb+AN8wh+Q=

pkg/qliksense/config.go

@@ -2,10 +2,12 @@ package qliksense
 
 import (
 	"fmt"
-	"github.com/mitchellh/go-homedir"
+	"os"
 	"path"
 	"path/filepath"
 
+	"github.com/mitchellh/go-homedir"
+
 	"github.com/qlik-oss/k-apis/pkg/cr"
 	qapi "github.com/qlik-oss/sense-installer/pkg/api"
 	"gopkg.in/yaml.v2"
@@ -24,14 +26,24 @@ func (q *Qliksense) ConfigApplyQK8s() error {
 		fmt.Println("cannot get the current-context cr", err)
 		return err
 	}
-	return applyConfigToK8s(qcr)
+	return q.applyConfigToK8s(qcr)
 }
 
-func applyConfigToK8s(qcr *qapi.QliksenseCR) error {
+func (q *Qliksense) applyConfigToK8s(qcr *qapi.QliksenseCR) error {
 	// apply qliksense-init crd first
 	mroot := qcr.Spec.GetManifestsRoot()
 	qInitMsPath := filepath.Join(mroot, Q_INIT_CRD_PATH)
 
+	if err := os.Unsetenv("EJSON_KEY"); err != nil {
+		fmt.Printf("error unsetting EJSON_KEY environment variable: %v\n", err)
+		return err
+	}
+
+	if err := os.Setenv("EJSON_KEYDIR", q.QliksenseEjsonKeyDir); err != nil {
+		fmt.Printf("error setting EJSON_KEYDIR environment variable: %v\n", err)
+		return err
+	}
+
 	qInitByte, err := executeKustomizeBuild(qInitMsPath)
 	if err != nil {
 		fmt.Println("cannot generate crds for qliksense-init", err)

pkg/qliksense/context_configs.go

@@ -25,7 +25,9 @@ const (
 	QliksenseDefaultProfile    = "docker-desktop"
 	QliksenseConfigFile        = "config.yaml"
 	QliksenseContextsDir       = "contexts"
-	DefaultQliksenseContext    = "qliksense-default"
+	DefaultQliksenseContext    = "qlik-default"
+	DefaultRotateKeys          = "yes"
+	MaxContextNameLength       = 17
 )
 
 // WriteToFile writes content into specified file
@@ -82,6 +84,8 @@ func AddCommonConfig(qliksenseCR api.QliksenseCR, contextName string) api.Qlikse
 	}
 	qliksenseCR.Spec = &config.CRSpec{}
 	qliksenseCR.Spec.Profile = QliksenseDefaultProfile
+	qliksenseCR.Spec.ReleaseName = contextName
+	qliksenseCR.Spec.RotateKeys = DefaultRotateKeys
 	return qliksenseCR
 }
 
@@ -243,8 +247,13 @@ func SetOtherConfigs(q *Qliksense, args []string) error {
 			LogDebugMessage("Current StorageClassName: %s, Incoming StorageClassName: %s", qliksenseCR.Spec.StorageClassName, argsString[1])
 			qliksenseCR.Spec.StorageClassName = argsString[1]
 			LogDebugMessage("Current StorageClassName after modification: %s ", qliksenseCR.Spec.StorageClassName)
+		case "rotateKeys":
+			LogDebugMessage("Current rotateKeys: %s, Incoming rotateKeys: %s", qliksenseCR.Spec.RotateKeys, argsString[1])
+			rotateKeys := validateInput(argsString[1])
+			qliksenseCR.Spec.RotateKeys = rotateKeys
+			LogDebugMessage("Current rotateKeys after modification: %s ", qliksenseCR.Spec.RotateKeys)
 		default:
-			log.Println("As part of the `qliksense config set` command, please enter one of: profile, namespace, storageClassName or git.repository arguments")
+			log.Println("As part of the `qliksense config set` command, please enter one of: profile, namespace, storageClassName,rotateKeys or git.repository arguments")
 		}
 	} else {
 		log.Fatalf("No args were provided. Please provide args to configure the current context")
@@ -273,9 +282,15 @@ func SetUpQliksenseDefaultContext(qlikSenseHome string) {
 
 // SetUpQliksenseContext - to setup qliksense context
 func SetUpQliksenseContext(qlikSenseHome, contextName string, isDefaultContext bool) {
+	// check the length of the context name entered by the user, it should not exceed 17 chars
+	if len(contextName) > MaxContextNameLength {
+		log.Fatal("Please enter a context-name with utmost 17 characters")
+	}
+
 	qliksenseConfigFile := filepath.Join(qlikSenseHome, QliksenseConfigFile)
 	var qliksenseConfig api.QliksenseConfig
 	configFileTrack := false
+
 	if !FileExists(qliksenseConfigFile) {
 		qliksenseConfig = AddBaseQliksenseConfigs(qliksenseConfig, contextName)
 	} else {
@@ -293,7 +308,7 @@ func SetUpQliksenseContext(qlikSenseHome, contextName string, isDefaultContext b
 		}
 	}
 	LogDebugMessage("%s exists", qliksenseContextsDir1)
-	// creating contexts/qliksense-default.yaml file
+	// creating contexts/qlik-default.yaml file
 
 	qliksenseContextFile := filepath.Join(qliksenseContextsDir1, contextName, contextName+".yaml")
 	var qliksenseCR api.QliksenseCR
@@ -334,3 +349,18 @@ func SetUpQliksenseContext(qlikSenseHome, contextName string, isDefaultContext b
 		WriteToFile(&qliksenseConfig, qliksenseConfigFile)
 	}
 }
+
+func validateInput(input string) string {
+	validInputs := []string{"yes", "no", "None"}
+	isValid := false
+	for _, elem := range validInputs {
+		if input == elem {
+			isValid = true
+			break
+		}
+	}
+	if !isValid {
+		log.Fatal("Please enter one of: yes, no or None")
+	}
+	return input
+}

pkg/qliksense/fetch.go

@@ -29,7 +29,7 @@ func fetchAndUpdateCR(qConfig *qapi.QliksenseConfig, version string) error {
 
 	if repo, err := kapis_git.CloneRepository(destDir, QLIK_GIT_REPO, nil); err != nil {
 		return err
-	} else if err = kapis_git.Checkout(repo, version, "", nil); err != nil {
+	} else if err = kapis_git.Checkout(repo, version, version, nil); err != nil {
 		return err
 	}
 	qcr.Spec.ManifestsRoot = qConfig.BuildCurrentManifestsRoot(version)

pkg/qliksense/install.go

@@ -2,6 +2,7 @@ package qliksense
 
 import (
 	"fmt"
+
 	qapi "github.com/qlik-oss/sense-installer/pkg/api"
 )
 
@@ -9,6 +10,8 @@ type InstallCommandOptions struct {
 	AcceptEULA   string
 	Namespace    string
 	StorageClass string
+	MongoDbUri   string
+	RotateKeys   string
 }
 
 func (q *Qliksense) InstallQK8s(version string, opts *InstallCommandOptions) error {
@@ -39,15 +42,20 @@ func (q *Qliksense) InstallQK8s(version string, opts *InstallCommandOptions) err
 	if opts.AcceptEULA != "" {
 		qcr.Spec.AddToConfigs("qliksense", "acceptEULA", opts.AcceptEULA)
 	}
+	if opts.MongoDbUri != "" {
+		qcr.Spec.AddToSecrets("qliksense", "mongoDbUri", opts.MongoDbUri)
+	}
 	if opts.StorageClass != "" {
 		qcr.Spec.StorageClassName = opts.StorageClass
 	}
 	if opts.Namespace != "" {
 		qcr.Spec.NameSpace = opts.Namespace
 	}
-
+	if opts.RotateKeys != "" {
+		qcr.Spec.RotateKeys = opts.RotateKeys
+	}
 	qConfig.WriteCurrentContextCR(qcr)
-	if err := applyConfigToK8s(qcr); err != nil {
+	if err := q.applyConfigToK8s(qcr); err != nil {
 		fmt.Println("cannot do kubectl apply on manifests")
 		return err
 	}

pkg/qliksense/kuz_test.go

@@ -1,11 +1,22 @@
 package qliksense
 
 import (
+	"bytes"
+	"encoding/base64"
+	"io"
 	"io/ioutil"
+	"log"
 	"os"
 	"path"
+	"strings"
 	"testing"
 
+	"gopkg.in/yaml.v3"
+
+	"github.com/Shopify/ejson"
+	"github.com/qlik-oss/k-apis/pkg/config"
+	"github.com/qlik-oss/k-apis/pkg/qust"
+
 	kapis_git "github.com/qlik-oss/k-apis/pkg/git"
 )
 
@@ -47,7 +58,7 @@ metadata:
 	}
 }
 
-func Test_executeKustomizeBuild_onQlikConfig_DISABLED(t *testing.T) {
+func Test_executeKustomizeBuild_onQlikConfig_regenerateKeys(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping in short mode")
 	}
@@ -65,7 +76,70 @@ func Test_executeKustomizeBuild_onQlikConfig_DISABLED(t *testing.T) {
 		t.Fatalf("unexpected error: %v\n", err)
 	}
 
-	if _, err := executeKustomizeBuild(path.Join(configPath, "manifests", "base")); err != nil {
+	cr := &config.CRSpec{
+		ManifestsRoot: configPath,
+	}
+
+	if err := os.Setenv("EJSON_KEYDIR", tmpDir); err != nil {
+		t.Fatalf("unexpected error setting EJSON_KEYDIR environment variable: %v\n", err)
+	}
+
+	if err := os.Unsetenv("EJSON_KEY"); err != nil {
+		t.Fatalf("unexpected error unsetting EJSON_KEY: %v\n", err)
+	}
+
+	generateKeys(cr, "won't-use")
+
+	yamlResources, err := executeKustomizeBuild(path.Join(configPath, "manifests", "base"))
+	if err != nil {
 		t.Fatalf("unexpected kustomize error: %v\n", err)
 	}
+
+	decoder := yaml.NewDecoder(bytes.NewReader(yamlResources))
+	var resource map[string]interface{}
+	keyIdBase64 := ""
+	for {
+		err := decoder.Decode(&resource)
+		if err != nil {
+			if err != io.EOF {
+				t.Fatalf("unexpected yaml decode error: %v\n", err)
+			}
+			break
+		}
+		if resource["kind"].(string) == "Secret" && strings.Contains(resource["metadata"].(map[string]interface{})["name"].(string), "-users-secrets-") {
+			keyIdBase64 = resource["data"].(map[string]interface{})["tokenAuthPrivateKeyId"].(string)
+			break
+		}
+	}
+
+	untransformedKeyId := `(( (ds "data").kid ))`
+	if keyIdBase64 == "" {
+		t.Fatalf("expected keyIdBase64 for users secret to be non empty:\n")
+	} else if keyId, err := base64.StdEncoding.DecodeString(keyIdBase64); err != nil {
+		t.Fatalf("unexpected base64 decode error: %v\n", err)
+	} else if string(keyId) == untransformedKeyId {
+		t.Fatalf("unexpected users keyId: %v\n", untransformedKeyId)
+	}
+}
+
+func generateKeys(cr *config.CRSpec, defaultKeyDir string) {
+	log.Println("rotating all keys")
+	keyDir := getEjsonKeyDir(defaultKeyDir)
+	if ejsonPublicKey, ejsonPrivateKey, err := ejson.GenerateKeypair(); err != nil {
+		log.Printf("error generating an ejson key pair: %v\n", err)
+	} else if err := qust.GenerateKeys(cr, ejsonPublicKey); err != nil {
+		log.Printf("error generating application keys: %v\n", err)
+	} else if err := os.MkdirAll(keyDir, os.ModePerm); err != nil {
+		log.Printf("error makeing sure private key storage directory: %v exists, error: %v\n", keyDir, err)
+	} else if err := ioutil.WriteFile(path.Join(keyDir, ejsonPublicKey), []byte(ejsonPrivateKey), os.ModePerm); err != nil {
+		log.Printf("error storing ejson private key: %v\n", err)
+	}
+}
+
+func getEjsonKeyDir(defaultKeyDir string) string {
+	ejsonKeyDir := os.Getenv("EJSON_KEYDIR")
+	if ejsonKeyDir == "" {
+		ejsonKeyDir = defaultKeyDir
+	}
+	return ejsonKeyDir
 }

pkg/qliksense/operator.go

@@ -2,6 +2,7 @@ package qliksense
 
 import (
 	"fmt"
+	"path/filepath"
 	"io"
 	"os"
 	"strings"
@@ -32,7 +33,7 @@ func (q *Qliksense) getYamlFromPackrFile(packrFile string) string {
 func (q *Qliksense) getFileList(resourceType string) []string {
 	var resList []string
 	for _, v := range q.CrdBox.List() {
-		if strings.Contains(v, resourceType+"/") {
+		if strings.Contains(v, filepath.Join(resourceType, "")) {
 			resList = append(resList, []string{v}...)
 		}
 	}

pkg/qliksense/qliksense.go

@@ -2,19 +2,29 @@
 package qliksense
 
 import (
+	"os"
+	"path"
+
 	"github.com/gobuffalo/packr/v2"
 )
 
 // Qliksense is the logic behind the qliksense client
 type Qliksense struct {
-	QliksenseHome string
-	CrdBox        *packr.Box
+	QliksenseHome        string
+	QliksenseEjsonKeyDir string
+	CrdBox               *packr.Box ``
 }
 
 // New qliksense client, initialized with useful defaults.
-func New(qliksenseHome string) *Qliksense {
-	return &Qliksense{
+func New(qliksenseHome string) (*Qliksense, error) {
+	qliksenseClient := &Qliksense{
 		QliksenseHome: qliksenseHome,
 		CrdBox:        packr.New("crds", "./crds"),
 	}
+
+	qliksenseClient.QliksenseEjsonKeyDir = path.Join(qliksenseHome, "ejson", "keys")
+	if err := os.MkdirAll(qliksenseClient.QliksenseEjsonKeyDir, os.ModePerm); err != nil {
+		return nil, err
+	}
+	return qliksenseClient, nil
 }