Syncing from upstream zalando/patroni (feature/quorum-commit)

.github/workflows/install_deps.py

@@ -9,14 +9,19 @@
 
 
 def install_requirements(what):
+    subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'pip'])
+    s = subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'wheel', 'setuptools'])
+    if s != 0:
+        return s
+
     old_path = sys.path[:]
     w = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe())))
     sys.path.insert(0, os.path.dirname(os.path.dirname(w)))
     try:
         from setup import EXTRAS_REQUIRE, read
     finally:
         sys.path = old_path
-    requirements = ['mock>=2.0.0', 'flake8', 'pytest', 'pytest-cov'] if what == 'all' else ['behave']
+    requirements = ['flake8', 'pytest', 'pytest-cov'] if what == 'all' else ['behave']
     requirements += ['coverage']
     # try to split tests between psycopg2 and psycopg3
     requirements += ['psycopg[binary]'] if sys.version_info >= (3, 8, 0) and\
@@ -28,11 +33,7 @@ def install_requirements(what):
             if not extras or what == 'all' or what in extras:
                 requirements.append(r)
 
-    subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'pip'])
-    subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'wheel'])
-    r = subprocess.call([sys.executable, '-m', 'pip', 'install'] + requirements)
-    s = subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'setuptools'])
-    return s | r
+    return subprocess.call([sys.executable, '-m', 'pip', 'install'] + requirements)
 
 
 def install_packages(what):

.github/workflows/release.yaml

@@ -11,12 +11,12 @@ jobs:
     name: Build and publish Patroni distributions to PyPI and TestPyPI
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v4
 
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.9
+        python-version: 3.11
 
     - name: Install dependencies
       run: python .github/workflows/install_deps.py
@@ -32,13 +32,13 @@ jobs:
 
     - name: Publish distribution to Test PyPI
       if: github.event_name == 'push'
-      uses: pypa/gh-action-pypi-publish@v1.5.1
+      uses: pypa/gh-action-pypi-publish@v1.8.14
       with:
         password: ${{ secrets.TEST_PYPI_API_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
 
     - name: Publish distribution to PyPI
       if: github.event_name == 'release'
-      uses: pypa/gh-action-pypi-publish@v1.5.1
+      uses: pypa/gh-action-pypi-publish@v1.8.14
       with:
         password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/run_tests.py

@@ -31,7 +31,7 @@ def main():
     else:
         if sys.platform == 'darwin':
             version = os.environ.get('PGVERSION', '16.1-1')
-            path = '/usr/local/opt/postgresql@{0}/bin:.'.format(version.split('.')[0])
+            path = '/opt/homebrew/opt/postgresql@{0}/bin:.'.format(version.split('.')[0])
             unbuffer = ['unbuffer']
         else:
             path = os.path.abspath(os.path.join('pgsql', 'bin'))

.github/workflows/tests.yaml

@@ -20,19 +20,22 @@ jobs:
         os: [ubuntu, windows, macos]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python 3.7
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.7
+      if: matrix.os != 'macos'
     - name: Install dependencies
       run: python .github/workflows/install_deps.py
+      if: matrix.os != 'macos'
     - name: Run tests and flake8
       run: python .github/workflows/run_tests.py
+      if: matrix.os != 'macos'
 
     - name: Set up Python 3.8
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.8
     - name: Install dependencies
@@ -41,7 +44,7 @@ jobs:
       run: python .github/workflows/run_tests.py
 
     - name: Set up Python 3.9
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.9
     - name: Install dependencies
@@ -50,7 +53,7 @@ jobs:
       run: python .github/workflows/run_tests.py
 
     - name: Set up Python 3.10
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
     - name: Install dependencies
@@ -59,14 +62,23 @@ jobs:
       run: python .github/workflows/run_tests.py
 
     - name: Set up Python 3.11
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.11
     - name: Install dependencies
       run: python .github/workflows/install_deps.py
     - name: Run tests and flake8
       run: python .github/workflows/run_tests.py
 
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
+    - name: Install dependencies
+      run: python .github/workflows/install_deps.py
+    - name: Run tests and flake8
+      run: python .github/workflows/run_tests.py
+
     - name: Combine coverage
       run: python .github/workflows/run_tests.py combine
 
@@ -90,7 +102,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu]
-        python-version: [3.7, '3.10']
+        python-version: [3.7, 3.12]
         dcs: [etcd, etcd3, consul, exhibitor, kubernetes, raft]
         include:
         - os: macos
@@ -104,9 +116,9 @@ jobs:
           dcs: etcd3
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - uses: nolar/setup-k3d-k3s@v1
@@ -125,7 +137,7 @@ jobs:
     - name: Run behave tests
       run: python .github/workflows/run_tests.py
     - name: Upload logs if behave failed
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: failure()
       with:
         name: behave-${{ matrix.os }}-${{ matrix.dcs }}-${{ matrix.python-version }}-logs
@@ -145,7 +157,7 @@ jobs:
     needs: unit
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
     - run: python -m pip install coveralls
     - run: python -m coveralls --service=github --finish
       env:
@@ -162,27 +174,27 @@ jobs:
   pyright:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
-    - name: Set up Python 3.11
-      uses: actions/setup-python@v4
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.11
+        python-version: 3.12
 
     - name: Install dependencies
       run: python -m pip install -r requirements.txt psycopg2-binary psycopg
 
-    - uses: jakebailey/pyright-action@v1
+    - uses: jakebailey/pyright-action@v2
       with:
-        version: 1.1.347
+        version: 1.1.356
 
   docs:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python 3.11
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.11
         cache: pip

README.rst

@@ -41,7 +41,7 @@ For additional background info, see:
 Development Status
 ==================
 
-Patroni is in active development and accepts contributions. See our `Contributing <https://github.com/zalando/patroni/blob/master/docs/CONTRIBUTING.rst>`__ section below for more details.
+Patroni is in active development and accepts contributions. See our `Contributing <https://github.com/zalando/patroni/blob/master/docs/contributing_guidelines.rst>`__ section below for more details.
 
 We report new releases information `here <https://github.com/zalando/patroni/releases>`__.
 

docker/entrypoint.sh

@@ -38,7 +38,7 @@ EOT
         exec dumb-init "$@"
         ;;
     etcd)
-        exec "$@" -advertise-client-urls "http://$DOCKER_IP:2379"
+        exec "$@" --auto-compaction-retention=1 -advertise-client-urls "http://$DOCKER_IP:2379"
         ;;
     zookeeper)
         exec /usr/share/zookeeper/bin/zkServer.sh start-foreground

docs/ENVIRONMENT.rst

@@ -80,7 +80,7 @@ Etcdv3
 Environment names for Etcdv3 are similar as for Etcd, you just need to use ``ETCD3`` instead of ``ETCD`` in the variable name. Example: ``PATRONI_ETCD3_HOST``, ``PATRONI_ETCD3_CACERT``, and so on.
 
 .. warning::
-    Keys created with protocol version 2 are not visible with protocol version 3 and the other way around, therefore it is not possible to switch from Etcd to Etcdv3 just by updating Patroni configuration.
+    Keys created with protocol version 2 are not visible with protocol version 3 and the other way around, therefore it is not possible to switch from Etcd to Etcdv3 just by updating Patroni configuration. In addition, Patroni uses Etcd's gRPC-gateway (proxy) to communicate with the V3 API, which means that TLS common name authentication is not possible.
 
 
 ZooKeeper

docs/conf.py

@@ -21,6 +21,8 @@
 
 import sys
 
+from sphinx.application import ENV_PICKLE_FILENAME
+
 sys.path.insert(0, os.path.abspath('..'))
 
 from patroni.version import __version__
@@ -243,13 +245,24 @@
 # Remove these pages from index, references, toc trees, etc.
 # If the builder is not 'html' then add the API docs modules index to pages to be removed.
 exclude_from_builder = {
-    'latex': ['modules/modules'],
-    'epub': ['modules/modules'],
+    'latex': ['modules/'],
+    'epub': ['modules/'],
 }
 # Internal holding list, anything added here will always be excluded
 _docs_to_remove = []
 
 
+def config_inited(app, config):
+    """Run during Sphinx `config-inited` phase.
+
+    rtd reuses the environment, and there is no way to customize this behavior.
+    Thus we remove the saved env.
+    """
+    pickle_file = os.path.join(app.doctreedir, ENV_PICKLE_FILENAME)
+    if on_rtd and os.path.exists(pickle_file):
+        os.remove(pickle_file)
+
+
 def builder_inited(app):
     """Run during Sphinx `builder-inited` phase.
 
@@ -263,14 +276,26 @@ def builder_inited(app):
         _docs_to_remove.extend(exclude_from_builder[app.builder.name])
 
 
+def _to_be_removed(doc):
+    for remove in _docs_to_remove:
+        if doc.startswith(remove):
+            return True
+    return False
+
+
 def env_get_outdated(app, env, added, changed, removed):
     """Run during Sphinx `env-get-outdated` phase.
 
     Remove the items listed in `docs_to_remove` from known pages.
     """
-    added.difference_update(_docs_to_remove)
-    changed.difference_update(_docs_to_remove)
-    removed.update(_docs_to_remove)
+    to_remove = set()
+    for doc in env.found_docs:
+        if _to_be_removed(doc):
+            to_remove.add(doc)
+    added.difference_update(to_remove)
+    changed.difference_update(to_remove)
+    removed.update(to_remove)
+    env.project.docnames.difference_update(to_remove)
     return []
 
 
@@ -282,8 +307,7 @@ def doctree_read(app, doctree):
     from sphinx import addnodes
     for toc_tree_node in doctree.traverse(addnodes.toctree):
         for e in toc_tree_node['entries']:
-            ref = str(e[1])
-            if ref in _docs_to_remove:
+            if _to_be_removed(str(e[1])):
                 toc_tree_node['entries'].remove(e)
 
 
@@ -304,6 +328,7 @@ def setup(app):
         app.add_stylesheet('custom.css')
 
     # Run extra steps to remove module docs when running with a non-html builder
+    app.connect('config-inited', config_inited)
     app.connect('builder-inited', builder_inited)
     app.connect('env-get-outdated', env_get_outdated)
     app.connect('doctree-read', doctree_read)

docs/dynamic_configuration.rst

@@ -30,7 +30,7 @@ In order to change the dynamic configuration you can use either :ref:`patronictl
 -  **failsafe\_mode**: Enables :ref:`DCS Failsafe Mode <dcs_failsafe_mode>`. Defaults to `false`.
 -  **postgresql**:
 
-   -  **use\_pg\_rewind**: whether or not to use pg_rewind. Defaults to `false`.
+   -  **use\_pg\_rewind**: whether or not to use pg_rewind. Defaults to `false`. Note that either the cluster must be initialized with ``data page checksums`` (``--data-checksums`` option for ``initdb``) and/or ``wal_log_hints`` must be set to ``on``, or ``pg_rewind`` will not work.
    -  **use\_slots**: whether or not to use replication slots. Defaults to `true` on PostgreSQL 9.4+.
    -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. There is no recovery.conf anymore in PostgreSQL 12, but you may continue using this section, because Patroni handles it transparently.
    -  **parameters**: list of configuration settings for Postgres.

docs/ha_multi_dc.rst

@@ -44,7 +44,7 @@ You should not use ``pg_ctl promote`` in this scenario, you need "manually promo
 
 In case you want to return to the "initial" state, there are only two ways of resolving it:
 
-- Add the standby_cluster section back and it will trigger pg_rewind, but there are chances that pg_rewind will fail.
+- Add the standby_cluster section back and it will trigger ``pg_rewind``; however, for ``pg_rewind`` to function properly, either the cluster must be initialized with ``data page checksums`` (``--data-checksums`` option for ``initdb``) and/or ``wal_log_hints`` must be set to ``on``, but there are still chances that ``pg_rewind`` might fail due to other factors.
 - Rebuild the standby cluster from scratch.
 
 Before promoting standby cluster one have to manually ensure that the source cluster is down (STONITH). When DC1 recovers, the cluster has to be converted to a standby cluster.