Merge branch 'master' of github.com:patroni/patroni into feature/perm…

…anent-members

.github/workflows/tests.yaml

@@ -211,3 +211,20 @@ jobs:
 
     - name: Generate documentation
       run: tox -m docs
+
+  isort:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
+        cache: pip
+
+    - name: isort
+      uses: isort/isort-action@master
+      with:
+        requirementsFiles: "requirements.txt requirements.dev.txt requirements.docs.txt"
+        sort-paths: "patroni tests features setup.py"

docs/dynamic_configuration.rst

@@ -27,6 +27,7 @@ In order to change the dynamic configuration you can use either :ref:`patronictl
 -  **primary\_stop\_timeout**: The number of seconds Patroni is allowed to wait when stopping Postgres and effective only when synchronous_mode is enabled. When set to > 0 and the synchronous_mode is enabled, Patroni sends SIGKILL to the postmaster if the stop operation is running for more than the value set by primary\_stop\_timeout. Set the value according to your durability/availability tradeoff. If the parameter is not set or set <= 0, primary\_stop\_timeout does not apply.
 -  **synchronous\_mode**: turns on synchronous replication mode. Possible values: ``off``, ``on``, ``quorum``. In this mode the leader takes care of management of ``synchronous_standby_names``, and only the last known leader, or one of synchronous replicas, are allowed to participate in leader race. Synchronous mode makes sure that successfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See :ref:`replication modes documentation <replication_modes>` for details.
 -  **synchronous\_mode\_strict**: prevents disabling synchronous replication if no synchronous replicas are available, blocking all client writes to the primary. See :ref:`replication modes documentation <replication_modes>` for details.
+-  **synchronous\_node\_count**: if ``synchronous_mode`` is enabled, this parameter is used by Patroni to manage the precise number of synchronous standby instances and adjusts the state in DCS and the ``synchronous_standby_names`` parameter in PostgreSQL as members join and leave. If the parameter is set to a value higher than the number of eligible nodes, it will be automatically adjusted. Defaults to ``1``.
 -  **failsafe\_mode**: Enables :ref:`DCS Failsafe Mode <dcs_failsafe_mode>`. Defaults to `false`.
 -  **postgresql**:
 

features/archive-restore.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
-import os
 import argparse
+import os
 import shutil
 
 if __name__ == "__main__":

features/callback2.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
 import sys
+
 with open("data/{0}/{0}_cb.log".format(sys.argv[1]), "a+") as log:
     log.write(" ".join(sys.argv[-3:]) + "\n")

features/environment.py

@@ -1,9 +1,8 @@
 import abc
 import datetime
 import glob
-import os
 import json
-import psutil
+import os
 import re
 import shutil
 import signal
@@ -13,11 +12,14 @@
 import tempfile
 import threading
 import time
+
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+import psutil
 import yaml
 
 import patroni.psycopg as psycopg
 
-from http.server import BaseHTTPRequestHandler, HTTPServer
 from patroni.request import PatroniRequest
 
 
@@ -498,6 +500,7 @@ def _start(self):
 
     def _is_running(self):
         from patroni.dcs.etcd import DnsCachingResolver
+
         # if etcd is running, but we didn't start it
         try:
             self._client = self._client_cls({'host': 'localhost', 'port': 2379, 'retry_timeout': 30,

features/steps/basic_replication.py

@@ -1,9 +1,11 @@
 import json
-import patroni.psycopg as pg
 
-from behave import step, then
 from time import sleep, time
 
+from behave import step, then
+
+import patroni.psycopg as pg
+
 
 @step('I start {name:w}')
 def start_patroni(context, name):

features/steps/citus.py

@@ -1,11 +1,12 @@
 import json
 import time
 
-from behave import step, then
-from dateutil import tz
 from datetime import datetime
 from functools import partial
-from threading import Thread, Event
+from threading import Event, Thread
+
+from behave import step, then
+from dateutil import tz
 
 tzutc = tz.tzutc()
 

features/steps/patroni_api.py

@@ -1,14 +1,16 @@
 import json
-import parse
 import shlex
 import subprocess
 import sys
 import time
+
+from datetime import datetime, timedelta
+
+import parse
 import yaml
 
 from behave import register_type, step, then
 from dateutil import tz
-from datetime import datetime, timedelta
 
 tzutc = tz.tzutc()
 

features/steps/slots.py

@@ -2,6 +2,7 @@
 import time
 
 from behave import step, then
+
 import patroni.psycopg as pg
 
 

features/steps/watchdog.py

@@ -1,6 +1,7 @@
-from behave import step, then
 import time
 
+from behave import step, then
+
 
 def polling_loop(timeout, interval=1):
     """Returns an iterator that returns values until timeout has passed. Timeout is measured from start of iteration."""

patroni/__main__.py

@@ -13,7 +13,7 @@
 from typing import Any, Dict, List, Optional, TYPE_CHECKING
 
 from patroni import MIN_PSYCOPG2, MIN_PSYCOPG3, parse_version
-from patroni.daemon import AbstractPatroniDaemon, abstract_main, get_base_arg_parser
+from patroni.daemon import abstract_main, AbstractPatroniDaemon, get_base_arg_parser
 from patroni.tags import Tags
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -268,8 +268,8 @@ def process_arguments() -> Namespace:
         generate_config(args.configfile, False, args.dsn)
         sys.exit(0)
     elif args.validate_config:
-        from patroni.validator import schema
         from patroni.config import Config, ConfigParseError
+        from patroni.validator import schema
 
         try:
             Config(args.configfile, validator=schema)

patroni/api.py

@@ -7,32 +7,32 @@
 """
 
 import base64
+import datetime
 import hmac
 import json
 import logging
-import time
-import traceback
-import dateutil.parser
-import datetime
 import os
 import socket
 import sys
+import time
+import traceback
 
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from ipaddress import ip_address, ip_network, IPv4Network, IPv6Network
 from socketserver import ThreadingMixIn
 from threading import Thread
-from urllib.parse import urlparse, parse_qs
-
 from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union
+from urllib.parse import parse_qs, urlparse
+
+import dateutil.parser
 
 from . import global_config, psycopg
 from .__main__ import Patroni
 from .dcs import Cluster
 from .exceptions import PostgresConnectionException, PostgresException
 from .postgresql.misc import postgres_version_to_int
-from .utils import deep_compare, enable_keepalive, parse_bool, patch_config, Retry, \
-    RetryFailedError, parse_int, split_host_port, tzutc, uri, cluster_as_json
+from .utils import cluster_as_json, deep_compare, enable_keepalive, parse_bool, \
+    parse_int, patch_config, Retry, RetryFailedError, split_host_port, tzutc, uri
 
 logger = logging.getLogger(__name__)
 

patroni/config.py

@@ -1,24 +1,25 @@
 """Facilities related to Patroni configuration."""
-import re
 import json
 import logging
 import os
+import re
 import shutil
 import tempfile
-import yaml
 
 from collections import defaultdict
 from copy import deepcopy
-from typing import Any, Callable, Collection, Dict, List, Optional, Union, TYPE_CHECKING
+from typing import Any, Callable, Collection, Dict, List, Optional, TYPE_CHECKING, Union
+
+import yaml
 
 from . import PATRONI_ENV_PREFIX
 from .collections import CaseInsensitiveDict, EMPTY_DICT
 from .dcs import ClusterConfig
 from .exceptions import ConfigParseError
 from .file_perm import pg_perm
 from .postgresql.config import ConfigHandler
-from .validator import IntValidator
 from .utils import deep_compare, parse_bool, parse_int, patch_config
+from .validator import IntValidator
 
 logger = logging.getLogger(__name__)
 
@@ -676,23 +677,6 @@ def _get_auth(name: str, params: Collection[str] = _AUTH_ALLOWED_PARAMETERS[:2])
             if dcs in ret:
                 ret[dcs].update(_get_auth(dcs))
 
-        users = {}
-        for param in list(os.environ.keys()):
-            if param.startswith(PATRONI_ENV_PREFIX):
-                name, suffix = (param[len(PATRONI_ENV_PREFIX):].rsplit('_', 1) + [''])[:2]
-                # PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
-                # CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
-                if name and suffix == 'PASSWORD':
-                    password = os.environ.pop(param)
-                    if password:
-                        users[name] = {'password': password}
-                        options = os.environ.pop(param[:-9] + '_OPTIONS', None)  # replace "_PASSWORD" with "_OPTIONS"
-                        options = options and _parse_list(options)
-                        if options:
-                            users[name]['options'] = options
-        if users:
-            ret['bootstrap']['users'] = users
-
         return ret
 
     def _build_effective_configuration(self, dynamic_configuration: Dict[str, Any],

patroni/config_generator.py

@@ -2,14 +2,16 @@
 import abc
 import logging
 import os
-import psutil
 import socket
 import sys
-import yaml
 
-from getpass import getuser, getpass
 from contextlib import contextmanager
+from getpass import getpass, getuser
 from typing import Any, Dict, Iterator, List, Optional, TextIO, Tuple, TYPE_CHECKING, Union
+
+import psutil
+import yaml
+
 if TYPE_CHECKING:  # pragma: no cover
     from psycopg import Cursor
     from psycopg2 import cursor
@@ -23,7 +25,6 @@
 from .postgresql.misc import postgres_major_version_to_int
 from .utils import get_major_version, parse_bool, patch_config, read_stripped
 
-
 # Mapping between the libpq connection parameters and the environment variables.
 # This dict should be kept in sync with `patroni.utils._AUTH_ALLOWED_PARAMETERS`
 # (we use "username" in the Patroni config for some reason, other parameter names are the same).

patroni/ctl.py

@@ -12,12 +12,9 @@
     If it is also missing in the configuration file we assume that this is just a normal Patroni cluster (not Citus).
 """
 
-import click
 import codecs
 import copy
 import datetime
-import dateutil.parser
-import dateutil.tz
 import difflib
 import io
 import json
@@ -28,15 +25,21 @@
 import subprocess
 import sys
 import tempfile
-import urllib3
 import time
-import yaml
 
 from collections import defaultdict
 from contextlib import contextmanager
-from prettytable import ALL, FRAME, PrettyTable
+from typing import Any, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union
 from urllib.parse import urlparse
-from typing import Any, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECKING
+
+import click
+import dateutil.parser
+import dateutil.tz
+import urllib3
+import yaml
+
+from prettytable import ALL, FRAME, PrettyTable
+
 if TYPE_CHECKING:  # pragma: no cover
     from psycopg import Cursor
     from psycopg2 import cursor
@@ -52,12 +55,12 @@
 
 from . import global_config
 from .config import Config
-from .dcs import get_dcs as _get_dcs, AbstractDCS, Cluster, Member
+from .dcs import AbstractDCS, Cluster, get_dcs as _get_dcs, Member
 from .exceptions import PatroniException
 from .postgresql.misc import postgres_version_to_int
 from .postgresql.mpp import get_mpp
-from .utils import cluster_as_json, patch_config, polling_loop
 from .request import PatroniRequest
+from .utils import cluster_as_json, patch_config, polling_loop
 from .version import __version__
 
 CONFIG_DIR_PATH = click.get_app_dir('patroni')

patroni/dcs/__init__.py

@@ -5,22 +5,22 @@
 import logging
 import re
 import time
+
 from collections import defaultdict
 from copy import deepcopy
 from random import randint
 from threading import Event, Lock
 from typing import Any, Callable, Collection, Dict, Iterator, List, \
     NamedTuple, Optional, Set, Tuple, Type, TYPE_CHECKING, Union
-from urllib.parse import urlparse, urlunparse, parse_qsl
+from urllib.parse import parse_qsl, urlparse, urlunparse
 
 import dateutil.parser
 
 from .. import global_config
 from ..dynamic_loader import iter_classes, iter_modules
 from ..exceptions import PatroniFatalException
-from ..utils import deep_compare, uri
 from ..tags import Tags
-from ..utils import parse_int
+from ..utils import deep_compare, parse_int, uri
 
 if TYPE_CHECKING:  # pragma: no cover
     from ..config import Config