Lagoon Sync configuration generation (#119)

* Supports custom names for syncers

* First pass at generator

* Makes docker-compose read looser

* Refactors to remove fallthrough

* Refactors to remove fallthrough

* Changes load order for config files

* Adds ambient ssh info

* updates template

* First wizard checkin

* Adds wizard command

* Adds wizard long description

* fixes some output formatting

* Adds cluster details to wizard

* More documentation

* fixes typo

* Splits out documentation

---------

Co-authored-by: Blaize Kaye <[email protected]>

cmd/generate.go

@@ -0,0 +1,87 @@
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/uselagoon/lagoon-sync/generator"
+	"log"
+	"os"
+	"text/template"
+)
+
+var generateCmd = &cobra.Command{
+	Use:   "generate path/to/docker-compose.yml",
+	Short: "Generate a lagoon-sync configuration stanza from a docker-compose file",
+	Long: `Attempts to generate a lagoon-sync configuration from a docker-compose file.
+Currently supports filesystem definitions, mariadb/mysql services, and postgres.
+`,
+
+	Args: cobra.MinimumNArgs(1),
+	Run:  genCommandRun,
+}
+
+var outputfile string
+
+func genCommandRun(cmd *cobra.Command, args []string) {
+
+	_, err := os.Stat(args[0])
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	project, err := generator.LoadComposeFile(args[0])
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	services := generator.ProcessServicesFromCompose(project)
+
+	stanza, err := generator.BuildConfigStanzaFromServices(services)
+
+	const yamlTemplate = `
+{{ .Sync }}
+`
+
+	tmpl, err := template.New("yaml").Parse(yamlTemplate)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var output bytes.Buffer
+	err = tmpl.Execute(&output, struct {
+		Sync string
+	}{
+		Sync: stanza,
+	})
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if outputfile != "" {
+		// Create or open the file
+		file, err := os.Create(outputfile)
+		if err != nil {
+			fmt.Println("Error creating file:", err)
+			return
+		}
+		defer file.Close()
+
+		// Write the string to the file
+		_, err = file.WriteString(output.String())
+		if err != nil {
+			fmt.Println("Error writing to file:", err)
+			return
+		}
+		fmt.Println("Successfully wrote output to : " + outputfile)
+	} else {
+		fmt.Println(output.String())
+	}
+
+}
+
+func init() {
+	rootCmd.AddCommand(generateCmd)
+	generateCmd.PersistentFlags().StringVarP(&outputfile, "outputfile", "o", "", "Write output to file - outputs to STDOUT if unset")
+}

cmd/root.go

@@ -122,9 +122,9 @@ func processConfig(cfgFile string) error {
 	// Next, check for common lagoon config files and override defaults.
 	for _, path := range paths {
 		cfgFiles := []string{
-			filepath.Join(path, ".lagoon.yml"),
 			filepath.Join(path, ".lagoon-sync.yml"),
 			filepath.Join(path, ".lagoon-sync"),
+			filepath.Join(path, ".lagoon.yml"),
 		}
 		for _, filePath := range cfgFiles {
 			if utils.FileExists(filePath) {

cmd/sync.go

@@ -52,9 +52,17 @@ var syncCmd = &cobra.Command{
 }
 
 func syncCommandRun(cmd *cobra.Command, args []string) {
+
+	// SyncerType can be one of two things
+	// 1. a direct reference to a syncer - i.e. mariadb, postgres, files
+	// 2. a reference to an alias in the configuration file (.lagoon.yml/.lagoon-sync.yml)
+	// 3. a reference to a custom syncer, also defined in the config file.
 	SyncerType := args[0]
+
 	viper.Set("syncer-type", args[0])
 
+	// configRoot will be filled with the configuration information that is passed
+	// to the syncers when they're run.
 	var configRoot synchers.SyncherConfigRoot
 
 	if viper.ConfigFileUsed() == "" {
@@ -112,7 +120,7 @@ func syncCommandRun(cmd *cobra.Command, args []string) {
 	// Syncers are registered in their init() functions - so here we attempt to match
 	// the syncer type with the argument passed through to this command
 	// (e.g. if we're running `lagoon-sync sync mariadb --...options follow` the function
-	// GetSyncersForTypeFromConfigRoot will return a prepared mariadb syncher object)
+	// GetSyncersForTypeFromConfigRoot will return a prepared mariadb syncer object)
 	lagoonSyncer, err := synchers.GetSyncerForTypeFromConfigRoot(SyncerType, configRoot)
 	if err != nil {
 		// Let's ask the custom syncer if this will work, if so, we fall back on it ...
@@ -142,13 +150,29 @@ func syncCommandRun(cmd *cobra.Command, args []string) {
 	if configRoot.LagoonSync["ssh"] != nil {
 		mapstructure.Decode(configRoot.LagoonSync["ssh"], &sshConfig)
 	}
+
 	sshHost := SSHHost
-	if sshConfig.Host != "" && SSHHost == "ssh.lagoon.amazeeio.cloud" {
-		sshHost = sshConfig.Host
+	if SSHHost == "ssh.lagoon.amazeeio.cloud" { // we're using the default - lets see if there are other options
+		envSshHost, exists := os.LookupEnv("LAGOON_CONFIG_SSH_HOST")
+		if exists { // we prioritize env data
+			sshHost = envSshHost
+		} else {
+			if sshConfig.Host != "" {
+				sshHost = sshConfig.Host
+			}
+		}
 	}
+
 	sshPort := SSHPort
-	if sshConfig.Port != "" && SSHPort == "32222" {
-		sshPort = sshConfig.Port
+	if SSHPort == "32222" { // we're using the default - lets see if there are other options
+		envSshPort, exists := os.LookupEnv("LAGOON_CONFIG_SSH_PORT")
+		if exists { // we prioritize env data
+			sshPort = envSshPort
+		} else {
+			if sshConfig.Port != "" {
+				sshPort = sshConfig.Port
+			}
+		}
 	}
 
 	sshKey := SSHKey
@@ -238,6 +262,9 @@ func syncCommandRun(cmd *cobra.Command, args []string) {
 	}
 }
 
+// getServiceName will return the name of the service in which we run the commands themselves. This is typically
+// the cli pod in a project
+// TODO: this needs to be expanded to be dynamic in the future.
 func getServiceName(SyncerType string) string {
 	if SyncerType == "mongodb" {
 		return SyncerType

cmd/wizard.go

@@ -0,0 +1,51 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/uselagoon/lagoon-sync/generator"
+	"log"
+	"os"
+)
+
+var wizardCmd = &cobra.Command{
+	Use:   "interactive-config",
+	Short: "Generate a lagoon-sync configuration stanza interactively",
+	Long:  `This command shows a wizard that will help with generating a .lagoon-sync.yml style yaml stanza`,
+	Run:   genwizCommandRun,
+}
+
+func genwizCommandRun(cmd *cobra.Command, args []string) {
+
+	str, gerr := generator.RunWizard()
+
+	if gerr != nil {
+		log.Fatal(gerr)
+	}
+
+	if outputfile != "" {
+		// Create or open the file
+		file, err := os.Create(outputfile)
+		if err != nil {
+			fmt.Println("Error creating file:", err)
+			return
+		}
+		defer file.Close()
+
+		// Write the string to the file
+		_, err = file.WriteString(str)
+		if err != nil {
+			fmt.Println("Error writing to file:", err)
+			return
+		}
+		fmt.Println("Successfully wrote output to : " + outputfile)
+	} else {
+		fmt.Println(str)
+	}
+
+}
+
+func init() {
+	rootCmd.AddCommand(wizardCmd)
+	wizardCmd.PersistentFlags().StringVarP(&outputfile, "outputfile", "o", "", "Write output to file - outputs to STDOUT if unset")
+}

documentation/CONFIG.md

@@ -0,0 +1,82 @@
+
+# config
+
+The `config` command will output all current configuration information it can find on the environment. This is used, for example, to gather prerequisite data which can be used to determine how `lagoon-sync` should proceed with a transfer. For example, when running the tool on a environment that doesn't have rsync, then the syncer will know to install a static copy of rsync on that machine for us. This is because rsync requires that you need to have it available on both environments in order to transfer.
+
+This can be run with:
+
+`$ lagoon-sync config`
+
+# Configuring lagoon-sync
+
+Lagoon-sync configuration can be managed via yaml-formatted configuration files. The paths to these config files can be defined either by the `--config` argument, or by environment variables (`LAGOON_SYNC_PATH` or `LAGOON_SYNC_DEFAULTS_PATH`).
+
+The order of configuration precedence is as follows:
+
+1. `--config` argument (e.g. `lagoon-sync [command] --config ./.custom-lagoon-sync-config.yaml`).
+2. `.lagoon-sync.yml` typically contains a separate lagoon-sync configuration. Although this can, if required, be merged into the `.lagoon.yml` file
+3. `.lagoon.yaml` files (i.e. in project root, or `lagoon` directory). If an `.lagoon.yml` is available within the project, then this file will be used as the active configuration file by default.
+4. `LAGOON_SYNC_PATH` or `LAGOON_SYNC_DEFAULTS_PATH` environment variables.
+5. Finally, if no config file can be found the default configuration will be used a safely written to a new '.lagoon.yml`
+
+There are some configuration examples in the `examples` directory of this repo.
+
+2021/01/22 11:34:10 (DEBUG) Using config file: /lagoon/.lagoon-sync
+2021/01/22 11:34:10 (DEBUG) Config that will be used for sync:
+{
+"Config": {
+"DbHostname": "$MARIADB_HOST",
+"DbUsername": "$MARIADB_USERNAME",
+"DbPassword": "$MARIADB_PASSWORD",
+"DbPort": "$MARIADB_PORT",
+"DbDatabase": "$MARIADB_DATABASE",
+...
+
+To recap, the configuration files that can be used by default, in order of priority when available are:
+* /lagoon/.lagoon-sync-defaults
+* /lagoon/.lagoon-sync
+* .lagoon.yml
+
+
+## Custom configuration files
+If you don't want your configuration file inside `/lagoon` and want to give it another name then you can define a custom file and tell sync to use that by providing the file path. This can be done with `--config` flag such as:Config files that can be used in order of priority:
+- .lagoon-sync-defaults _(no yaml ext neeeded)_
+- .lagoon-sync _(no yaml ext neeeded)_
+- .lagoon.yml _Main config file - path can be given as an argument with `--config`, default is `.lagoon.yml`_
+  å
+```
+$ lagoon-sync sync mariadb -p mysite-com -e dev --config=/app/.lagoon-sync --show-debug
+
+2021/01/22 11:43:50 (DEBUG) Using config file: /app/.lagoon-sync
+```
+
+You can also use an environment variable to set the config sync path with either `LAGOON_SYNC_PATH` or `LAGOON_SYNC_DEFAULTS_PATH`.
+
+```
+$ LAGOON_SYNC_PATH=/app/.lagoon-sync lagoon-sync sync mariadb -p mysite-com -e dev --show-debug
+
+2021/01/22 11:46:42 (DEBUG) LAGOON_SYNC_PATH env var found: /app/.lagoon-sync
+2021/01/22 11:46:42 (DEBUG) Using config file: /app/.lagoon-sync
+```
+
+
+To double check which config file is loaded you can also run the `lagoon-sync config` command.
+
+
+### Example sync config overrides
+```
+lagoon-sync:
+  mariadb:
+    config:
+      hostname: "${MARIADB_HOST:-mariadb}"
+      username: "${MARIADB_USERNAME:-drupal}"
+      password: "${MARIADB_PASSWORD:-drupal}"
+      port: "${MARIADB_PORT:-3306}"
+      database: "${MARIADB_DATABASE:-drupal}"
+  files:
+    config:
+      sync-directory: "/app/web/sites/default/files"
+  drupalconfig:
+    config:
+      syncpath: "./config/sync"
+```

documentation/CONTRIBUTING.md

@@ -0,0 +1,18 @@
+# Contributing
+
+Setting up locally:
+
+* `make all`                          Installs missing dependencies, runs tests and build locally.
+* `make build`                        Compiles binary based on current go env.
+* `make local-build-linux`            Compile linix binary.
+* `make local-build-darwin`           Compile macOS (darwin) binary.
+* `make check-current-tag-version`    Check the current version.
+* `make clean`                        Remove all build files and assets.
+
+## Releases
+
+We are using [goreleaser](https://github.com/goreleaser/goreleaser) for the official build, release and publish steps that will be run from a GitHub Action on a pushed tag event.
+
+Locally, we can run `make release-test` to check if our changes will build. If compiling was successful we can commit our changes and then run `make release-[patch|minor|major]` to tag with next release number and it will push up to GitHub. A GitHub action will then be triggered which will publish the official release using goreleaser.
+
+Prior to that, we can locally test our release to ensure that it will successfully build with `make release-test`. If compiling was successful we can commit our changes and then run `make release-[patch|minor|major]` to tag with next release number and it will push up to GitHub. A GitHub action will then be triggered which will publish the official release using goreleaser.