Merge pull request #170 from peopledoc/ele_pum

Add a "Use Pum for migrations" how-to

docs/howto/use_pum_for_migrations.rst

@@ -0,0 +1,101 @@
+Use Pum for schema migrations
+-----------------------------
+
+When Procrastinate developers make changes to the Procrastinate database schema they
+write migration scripts.
+
+Here's an example of a migration script:
+
+.. code-block:: sql
+
+    ALTER TABLE procrastinate_jobs ADD COLUMN extra TEXT;
+
+Migration scripts are pure-SQL scripts, meaning that they may be applied to the database
+using any PostgreSQL client, including ``psql`` and ``PGAdmin``.
+
+The names of migration script files adhere to a certain pattern, which comes from the
+`Pum`_ project. Pum is thus a good choice for Procrastinate migrations, although other
+migration tools may be used as well.
+
+See the `Pum README`_ on GitHub to know how to use Pum. But below is a Pum quickstart,
+applied to Procrastinate.
+
+First of all, note that Pum connects to the PostgreSQL database using a `PostgreSQL
+Connection Service`_. So you have to have a Connection Service File. Here's an example
+of a Connection Service File (located at ``~/.pg_service.conf``):
+
+.. code-block::
+
+    [procrastinate]
+    host=localhost
+    port=5432
+    user=postgres
+    password=xxxxxxx
+    dbname=procrastinate
+
+You can use ``psql service=procrastinate`` to check that you can connect to
+PostgreSQL through the ``procrastinate`` service defined in your Connection Service
+File.
+
+The first time you install Procrastinate you will need to apply the Procrastinate schema
+to the database using the ``procrastinate schema --apply`` command.
+
+If you decide to use Pum for handling migrations you'll also need to create the Pum
+table (which Pum uses internally for tracking schema versions), and set the ``baseline``
+in this table.
+
+Let's say you are installing Procrastinate 0.5.0. Your baseline will then be 0.5.0. This
+the Pum command to set the baseline:
+
+.. code-block:: console
+
+    pum baseline --pg_service procrastinate --table public.pum --dir $(procrastinate schema --migrations-path) --baseline 0.5.0
+
+* ``--pg_service`` specifies the PostgreSQL Connection Service to use
+* ``--table`` specifies the name of the Pum table to create and insert the baseline into
+  (the table name must be prefixed by the schema name)
+* ``--dir`` specifies the path of the directory including migration scripts (it is
+  not relevant here, yet mandatory)
+* ``--baseline`` specifies the version number to use for the baseline
+
+.. note::
+
+    The command ``procastinate schema --migrations-path`` is not available with
+    Procrastinate <= 0.5.0. So you'll have to find the path to the migrations directory
+    (``sql/migrations`` within the ``site-packages/procrastinate`` directory).
+
+You're all set at this point! Procrastinate is ready to use, and your Pum baseline is
+set for future migrations of the Procrastinate schema.
+
+Now let's say you want to upgrade from Procrastinate 0.5.0 to 0.6.0. You will install
+Procrastinate 0.6.0 in your Python environment (for example using ``pip``), and you will
+then use Pum to apply the necessary migration scripts, that is the migration scripts the
+Procrastinate developers created for migrating from the 0.5.0 version of Procrastinate.
+
+This is how migration scripts are applied using Pum:
+
+.. code-block:: console
+
+    pum upgrade --pg_service procrastinate --table public.pum --dir $(procrastinate schema --migrations-path)
+
+The ``--pg_service``, ``--table``, and ``--dir`` flags have been described above, in the
+context of the ``pum baseline`` command.
+
+The output of the ``pum upgrade`` command should look like this:
+
+.. code-block:: console
+
+    Upgrade...
+    Applying delta 0.5.0 001_drop_started_at_column... OK
+    Applying delta 0.5.0 002_drop_started_at_column... OK
+    Applying delta 0.5.0 003_drop_procrastinate_version_table... OK
+
+In this example ``pum upgrade`` applied three migration scripts, two related to
+the dropping a column and one related to the dropping of a table.
+
+That is all! Pum includes other commands, check commands for verifying migrations in
+particular. Check out the Pum documentation for more information.
+
+.. _`Pum`: https://github.com/opengisch/pum/
+.. _`Pum README`: https://github.com/opengisch/pum/blob/master/README.md
+.. _`PostgreSQL Connection Service`: https://www.postgresql.org/docs/current/libpq-pgservice.html

docs/howto_index.rst

@@ -26,3 +26,4 @@ How-to...
     howto/remove_old_jobs
     howto/custom_json_encoder_decoder
     howto/set_database_schema
+    howto/use_pum_for_migrations

procrastinate/cli.py

@@ -245,23 +245,35 @@ def configure_job(
 @click.pass_obj
 @handle_errors()
 @click.option(
-    "--apply/--read",
-    default=True,  # a.k.a run
-    help="*Read* the schema SQL and output it, or *apply* it to the DB directly"
-    " (default)",
+    "--apply",
+    "action",
+    flag_value="apply",
+    help="Apply the schema to the DB (default)",
+    default=True,
 )
-def schema(app: procrastinate.App, apply: bool):
+@click.option(
+    "--read", "action", flag_value="read", help="Read the schema SQL and output it",
+)
+@click.option(
+    "--migrations-path",
+    "action",
+    flag_value="migrations-path",
+    help="Output the path to the directory containing the migration scripts",
+)
+def schema(app: procrastinate.App, action: str):
     """
     Apply SQL schema to the empty database. This won't work if the schema has already
     been applied.
     """
     schema_manager = app.schema_manager
-    if apply:
+    if action == "apply":
         click.echo("Applying schema")
         schema_manager.apply_schema()  # type: ignore
         click.echo("Done")
-    else:
+    elif action == "read":
         click.echo(schema_manager.get_schema(), nl=False)
+    else:
+        click.echo(schema_manager.get_migrations_path())
 
 
 @cli.command()

procrastinate/schema.py

@@ -1,8 +1,12 @@
+import pathlib
+
 from importlib_resources import read_text
 
 from procrastinate import connector as connector_module
 from procrastinate import utils
 
+migrations_path = pathlib.Path(__file__).parent / "sql" / "migrations"
+
 
 @utils.add_sync_api
 class SchemaManager:
@@ -14,6 +18,10 @@ def __init__(self, connector: connector_module.BaseConnector):
     def get_schema() -> str:
         return read_text("procrastinate.sql", "schema.sql")
 
+    @staticmethod
+    def get_migrations_path() -> str:
+        return str(migrations_path)
+
     async def apply_schema_async(self) -> None:
         queries = self.get_schema()
         await self.connector.execute_query(query=queries)

tests/integration/test_cli.py

@@ -54,6 +54,13 @@ def test_schema_read(entrypoint):
     assert result.exit_code == 0
 
 
+def test_schema_migrations_patth(entrypoint):
+    result = entrypoint("schema --migrations-path")
+
+    assert result.output.endswith("sql/migrations\n")
+    assert result.exit_code == 0
+
+
 def test_healthchecks(entrypoint, click_app, mocker):
     check_db = mocker.patch(
         "procrastinate.healthchecks.HealthCheckRunner.check_connection"

tests/unit/test_schema.py

@@ -5,6 +5,10 @@ def test_get_schema(app):
     assert app.schema_manager.get_schema().startswith("-- Procrastinate Schema")
 
 
+def test_get_migrations_path(app):
+    assert app.schema_manager.get_migrations_path().endswith("sql/migrations")
+
+
 def test_apply_schema(app, connector):
     connector.reverse_queries = defaultdict(lambda: "apply_schema")
     connector.set_schema_version_run = lambda *a, **kw: None