From 76c107cb1d6e0509d2f4eede0a671c16facb128d Mon Sep 17 00:00:00 2001 From: Blaize Kaye Date: Fri, 20 Dec 2024 10:23:35 +1300 Subject: [PATCH] More documentation --- README.md | 144 ++++++++++++++++++++++++--- cmd/generate.go | 2 - cmd/wizard.go | 2 +- examples/tutorial/.lagoon-sync.yml | 28 ++++++ examples/tutorial/docker-compose.yml | 95 ++++++++++++++++++ 5 files changed, 255 insertions(+), 16 deletions(-) create mode 100644 examples/tutorial/.lagoon-sync.yml create mode 100644 examples/tutorial/docker-compose.yml diff --git a/README.md b/README.md index 44b3b27..fb9a4b4 100644 --- a/README.md +++ b/README.md @@ -4,18 +4,134 @@ Lagoon-sync is cli tool written in Go that fundamentally provides the functional Lagoon-sync offers: * Sync commands for databases such as `mariadb`, `postgres` and `mongodb` -* Php/node-based framework support such as Drupal, Laravel or Node.js * Standard file transfer support with `files` syncer * Has built-in default configuration values for syncing out-the-box -* Provides an easy way to override sync configuration via `.lagoon.yml` or `.lagoon-sync.yml` files +* Provides an easy way to override sync configuration via `.lagoon-sync.yml` files * Offers `--dry-run` flag to see what commands would be executed before running a transfer * `--no-interaction` can be used to auto-run all processes without prompt - useful for CI/builds * `config` command shows the configuration of the current environment * There is a `--show-debug` flag to output more verbose logging for debugging -* Lagoon-sync uses `rsync` for the transfer of data, and will automatically detect and install `rsync` if it is not available on target environments * Secure cross-platform self-updating with `selfUpdate` command +# Getting started with lagoon-sync + +Here we'll describe the typical use case for lagoon-sync. While it's able to do quite a number of things, we're going to +speak about the standard use case - that is, how do we use lagoon-sync to sync our databases and our files between environments. + +We'll focus on a very simple example, setting up `lagoon-sync` for a Laravel project. + +This tutorial assumes that you've already [lagoonized](https://docs.lagoon.sh/lagoonizing/) your project, and that you have a `docker-compose.yml` file that +describes the services that you're going to need. + +## Where does `lagoon-sync` actually run? + +This is a common question we get, because it can be kind of confusing. Where exactly are you supposed to run `lagoon-sync`? + +Well, it will always run _inside a container_ - it's not a tool like the lagoon cli that just runs from anywhere. +`lagoon-sync` is essentially a wrapper around commands like `mysqldump`, `rsync`, `mongodump`, etc. +In fact, everything that `lagoon-sync` does, you could do manually if you `ssh`ed into your running containers and typed out the various commands. + +There is no special, secret sauce. It's more like a collection of neat `bash` scripts than anything else. And so, like with `bash` scripts, +it needs to run in the actual containers. + +This means that `lagoon-sync` needs to be available insider your container - typically, we find the easiest way of doing this is +including it in your `cli` dockerfile, if you have one. + + +## .lagoon-sync.yml + +You can run `lagoon-sync` in a myriad ways. But here is the simplest, most straight forward, that should work in most cases. +We encourage this pattern. + +You can add a `.lagoon-sync.yml` file to the root of your application's source code, alongside your `.lagoon.yml` file. + +This `.lagoon-sync.yml` will describe all of the syncs that you might want `lagoon-sync` to do. Each option for syncing will +appear as a separate item under the `lagoon-sync:` key. + +Let's look at an [example](./examples/tutorial/.lagoon-sync.yml): + +``` +lagoon-sync: + mariadb: + type: mariadb + config: + hostname: "${MARIADB_HOST:-mariadb}" + username: "${MARIADB_USERNAME:-lagoon}" + password: "${MARIADB_PASSWORD:-lagoon}" + port: "${MARIADB_PORT:-3306}" + database: "${MARIADB_DATABASE:-lagoon}" + cli: + type: files + config: + sync-directory: "/app/storage/" +``` + +This lagoon sync config above describes two synchers - `mariadb` for the database, and `cli` for the files. +With this in my `.lagoon-sync.yml`, inside my `cli` container, I can run the command +`lagoon-sync sync cli -p myprojectname -e sourceenvironment` and `lagoon-sync` will rsync all the files in `sourceenvironment`'s +`/app/storage/` directory into my local environment. + +The same will be the case, except it will sync the database, if I ran `lagoon-sync sync mariadb -p myprojectname -e sourceenvironment`. + +Note, that in the example above, `mariadb` and `cli` are simply aliases, we could have renamed them to `mydatabase` and `filestorage` like so: +``` +lagoon-sync: + mydatabase: + type: mariadb + config: +<..snip../> + filestorage: + type: files + config: + sync-directory: "/app/storage/" +``` + +and run the syncs with `lagoon-sync sync mydatabase -p mypr...` and `lagoon-sync sync filestorage -p mypr...` +The nested keys `cli` and `mariadb` are simply names - it's the `type:` key that tells `lagoon-sync` what it's actually syncing. + +You can define as many of these syncers as you need - if you have multiple databases, for instance, or, more likely, if you +have multiple files/directories you'd like to sync separately. + +## How should I be generating a `.lagoon-sync.yml`? + +This is the part of the process that seems to trip most people up, so we've made it fairly simple. + +If you'd like to generate a `.lagoon-sync.yml`, you can use `lagoon-sync`'s built in functions `generate` and `interactive-config`. + +The `generate` command tries to take your lagoonized `docker-compose.yml` file and generate a `.lagoon-sync.yml` file based +off what it finds in the service definition. + +The [example](./examples/tutorial/.lagoon-sync.yml) above was actually generated by [this docker-compose file](./examples/tutorial/docker-compose.yml). + +In order to generate a file, simply run `lagoon-sync generate ./docker-compose.yml -o .lagoon-sync.yml` and it should, +hopefully, generate a reasonable lagoon sync config. + +If you'd like to be more hands-on you can run `lagoon-sync interactive-config -o .lagoon-sync.yml` and you'll be presented with a +menu that you can use to generate a sync config. + +## What's all this about clusters? + +If your project is on anything except the amazeeio cluster, which are the defaults +and you're running lagoon-sync from a local container, you may have to set these variables +you can grab this information from running the lagoon cli's `lagoon config list` +this will output the ssh endpoints and ports you need. + +Typically, though, this information is also available in the environment variables +LAGOON_CONFIG_SSH_HOST and LAGOON_CONFIG_SSH_PORT + +These, for instance, are the amazeeio defaults - and should do for most people. +If you're on your own cluster, these are the same values that will be in your `.lagoon.yml` + +* `ssh: ssh.lagoon.amazeeio.cloud:32222` +* `api: https://api.lagoon.amazeeio.cloud/graphql` + +## Please note + +At the moment, the generator and wizard only support the most commonly used cases - files, mariadb, and postgres. +Mongodb is actually a far more complex beast, and we'll add some more support for it in the future. + + # Installing You can run `lagoon-sync` as a single binary by downloading from `https://github.com/uselagoon/lagoon-sync/releases/latest`. @@ -60,13 +176,14 @@ Usage: lagoon-sync [command] Available Commands: - completion generate the autocompletion script for the specified shell - config Print the config that is being used by lagoon-sync - generate Generate a lagoon-sync configuration stanza from a docker-compose file - help Help about any command - selfUpdate Update this tool to the latest version - sync Sync a resource type - version Print the version number of lagoon-sync + completion generate the autocompletion script for the specified shell + config Print the config that is being used by lagoon-sync + generate Generate a lagoon-sync configuration stanza from a docker-compose file + help Help about any command + interactive-config Generate a lagoon-sync configuration stanza interactively + selfUpdate Update this tool to the latest version + sync Sync a resource type + version Print the version number of lagoon-sync Flags: --config string Path to the file used to set lagoon-sync configuration @@ -148,9 +265,10 @@ Lagoon-sync configuration can be managed via yaml-formatted configuration files. The order of configuration precedence is as follows: 1. `--config` argument (e.g. `lagoon-sync [command] --config ./.custom-lagoon-sync-config.yaml`). -2. `.lagoon.yaml` files (i.e. in project root, or `lagoon` directory). If an `.lagoon.yml` is available within the project, then this file will be used as the active configuration file by default. -3. `LAGOON_SYNC_PATH` or `LAGOON_SYNC_DEFAULTS_PATH` environment variables. -4. Finally, if no config file can be found the default configuration will be used a safely written to a new '.lagoon.yml` +2. `.lagoon-sync.yml` typically contains a separate lagoon-sync configuration. Although this can, if required, be merged into the `.lagoon.yml` file +3. `.lagoon.yaml` files (i.e. in project root, or `lagoon` directory). If an `.lagoon.yml` is available within the project, then this file will be used as the active configuration file by default. +4. `LAGOON_SYNC_PATH` or `LAGOON_SYNC_DEFAULTS_PATH` environment variables. +5. Finally, if no config file can be found the default configuration will be used a safely written to a new '.lagoon.yml` There are some configuration examples in the `examples` directory of this repo. diff --git a/cmd/generate.go b/cmd/generate.go index 8bafe0a..ece414b 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -40,8 +40,6 @@ func genCommandRun(cmd *cobra.Command, args []string) { stanza, err := generator.BuildConfigStanzaFromServices(services) const yamlTemplate = ` -# Copy the following and add it to your .lagoon.yml file (see https://docs.lagoon.sh/concepts-basics/lagoon-yml/) - {{ .Sync }} ` diff --git a/cmd/wizard.go b/cmd/wizard.go index 99001d2..f168dd6 100644 --- a/cmd/wizard.go +++ b/cmd/wizard.go @@ -9,7 +9,7 @@ import ( ) var wizardCmd = &cobra.Command{ - Use: "interactive", + Use: "interactive-config", Short: "Generate a lagoon-sync configuration stanza interactively", Long: `This command shows a wizard that will help with generating a .lagoon-sync.yml style yaml stanza`, Run: genwizCommandRun, diff --git a/examples/tutorial/.lagoon-sync.yml b/examples/tutorial/.lagoon-sync.yml new file mode 100644 index 0000000..1c865ca --- /dev/null +++ b/examples/tutorial/.lagoon-sync.yml @@ -0,0 +1,28 @@ +# Below is your configuration for lagoon-sync. +# These data can live in either a separate .lagoon-sync.yml file +# or your .lagoon.yml file. + +# If your project is on anything except the amazeeio cluster, which are the defaults +# and you're running lagoon-sync from a local container, you may have to set these variables +# you can grab this information from running the lagoon cli's "lagoon config list" +# this will output the ssh endpoints and ports you need. +# Typically, though, this information is also available in the environment variables +# LAGOON_CONFIG_SSH_HOST and LAGOON_CONFIG_SSH_PORT +# +# These, for instance, are the amazeeio defaults +# ssh: ssh.lagoon.amazeeio.cloud:32222 +# api: https://api.lagoon.amazeeio.cloud/graphql + +lagoon-sync: + mariadb: + type: mariadb + config: + hostname: "${MARIADB_HOST:-mariadb}" + username: "${MARIADB_USERNAME:-lagoon}" + password: "${MARIADB_PASSWORD:-lagoon}" + port: "${MARIADB_PORT:-3306}" + database: "${MARIADB_DATABASE:-lagoon}" + cli: + type: files + config: + sync-directory: "/app/storage/" diff --git a/examples/tutorial/docker-compose.yml b/examples/tutorial/docker-compose.yml new file mode 100644 index 0000000..3162ad0 --- /dev/null +++ b/examples/tutorial/docker-compose.yml @@ -0,0 +1,95 @@ +version: '2.3' + +x-volumes: + &default-volumes + # Define all volumes you would like to have real-time mounted into the docker containers + volumes: + - ./:/app:delegated ### Local overrides to mount host filesystem. Automatically removed in CI and PROD. + +x-environment: + &default-environment + # Route that should be used locally, if you are using pygmy, this route *must* end with .docker.amazee.io + LAGOON_ROUTE: &default-url http://${COMPOSE_PROJECT_NAME:-laravel10}.docker.amazee.io + # Uncomment if you like to have the system behave like in production + #LAGOON_ENVIRONMENT_TYPE: production + # Uncomment to enable xdebug and then restart via `docker-compose up -d` + #XDEBUG_ENABLE: "true" + +x-user: + &default-user + # The default user under which the containers should run. Change this if you are on linux and run with another user than id `1000` + user: '1000' + +volumes: + files: + {} + +services: + + cli: # cli container, will be used for executing composer and any local commands (drush, drupal, etc.) + build: + context: ./ + dockerfile: .lagoon/cli.dockerfile + image: &cli-image ${COMPOSE_PROJECT_NAME:-laravel10}-cli # this image will be reused as `CLI_IMAGE` in subsequent Docker builds + labels: + # Lagoon Labels + lagoon.type: cli-persistent + lagoon.persistent.name: nginx # mount the persistent storage of nginx into this container + lagoon.persistent: /app/storage/ # location where the persistent storage should be mounted + # lando.type: php-cli-drupal + << : *default-volumes # loads the defined volumes from the top + user: root + volumes_from: ### mount the ssh-agent from the pygmy or cachalot ssh-agent. Automatically removed in CI. + - container:amazeeio-ssh-agent ### Local overrides to mount host SSH keys. Automatically removed in CI. + environment: + << : *default-environment # loads the defined environment variables from the top + + nginx: + build: + context: ./ + dockerfile: .lagoon/nginx.dockerfile + args: + CLI_IMAGE: *cli-image # Inject the name of the cli image + labels: + lagoon.type: nginx-php-persistent + lagoon.persistent: /app/storage/ # define where the persistent storage should be mounted too + << : *default-volumes # loads the defined volumes from the top + user: '1000' + depends_on: + - cli # basically just tells docker-compose to build the cli first + environment: + << : *default-environment # loads the defined environment variables from the top + LAGOON_LOCALDEV_URL: *default-url + networks: + - amazeeio-network + - default + php: + build: + context: ./ + dockerfile: .lagoon/php.dockerfile + args: + CLI_IMAGE: *cli-image + labels: + lagoon.type: nginx-php-persistent + lagoon.name: nginx # we want this service be part of the nginx pod in Lagoon + lagoon.persistent: /app/storage/ # define where the persistent storage should be mounted too + << : *default-volumes # loads the defined volumes from the top + user: '1000' + depends_on: + - cli # basically just tells docker-compose to build the cli first + environment: + << : *default-environment # loads the defined environment variables from the top + + mariadb: + image: uselagoon/mariadb-10.5:latest + labels: + lagoon.type: mariadb + ports: + - "3306" # exposes the port 3306 with a random local port, find it with `docker-compose port mariadb 3306` + user: '1000' + environment: + << : *default-environment + +networks: + amazeeio-network: + external: true \ No newline at end of file