Skip to content

Commit

Permalink
Patroni 2.1.4 (#737)
Browse files Browse the repository at this point in the history 
- ensure all packages are upgraded to the latest version (apt-get -y upgrade)
- allow starting etcd on non-x86 platforms
- refactor `get_dcs_config()` function

Close #736, #726
  • Loading branch information
Alexander Kukushkin authored Jun 17, 2022
1 parent 22c9247 commit cdae614
Show file tree
Hide file tree
Showing 4 changed files with 45 additions and 35 deletions.
9 changes: 6 additions & 3 deletions postgres-appliance/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ ARG ADDITIONAL_LOCALES=
RUN export DEBIAN_FRONTEND=noninteractive \
&& echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
&& apt-get update \
&& apt-get -y upgrade \
&& apt-get install -y curl ca-certificates less locales jq vim-tiny gnupg1 cron runit dumb-init libcap2-bin rsync sysstat \
&& ln -s chpst /usr/bin/envdir \
# Make it possible to use the following utilities without root (if container runs without "no-new-privileges:true")
Expand Down Expand Up @@ -137,7 +138,8 @@ ENV POSTGIS_VERSION=3.2 \
PLPROFILER=REL4_1 \
PAM_OAUTH2=v1.0.1 \
PLANTUNER_COMMIT=800d81bc85da64ff3ef66e12aed1d4e1e54fc006 \
PG_PERMISSIONS_COMMIT=314b9359e3d77c0b2ef7dbbde97fa4be80e31925
PG_PERMISSIONS_COMMIT=314b9359e3d77c0b2ef7dbbde97fa4be80e31925 \
PG_TM_AUX_COMMIT=6c012d38a4c1b0ba4a36952d60b0ce3a22ac9c3d

RUN export DEBIAN_FRONTEND=noninteractive \
&& export MAKEFLAGS="-j $(grep -c ^processor /proc/cpuinfo)" \
Expand Down Expand Up @@ -183,6 +185,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
&& curl -sL https://github.com/CyberDem0n/bg_mon/archive/$BG_MON_COMMIT.tar.gz | tar xz \
&& curl -sL https://github.com/sdudoladov/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.tar.gz | tar xz \
&& curl -sL https://github.com/cybertec-postgresql/pg_permissions/archive/$PG_PERMISSIONS_COMMIT.tar.gz | tar xz \
&& curl -sL https://github.com/x4m/pg_tm_aux/archive/$PG_TM_AUX_COMMIT.tar.gz | tar xz \
&& git clone -b $SET_USER https://github.com/pgaudit/set_user.git \
&& git clone https://github.com/timescale/timescaledb.git \
\
Expand Down Expand Up @@ -258,7 +261,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
EXTRA_EXTENSIONS=""; \
fi \
\
&& for n in bg_mon-${BG_MON_COMMIT} pg_auth_mon-${PG_AUTH_MON_COMMIT} set_user pg_permissions-${PG_PERMISSIONS_COMMIT} $EXTRA_EXTENSIONS; do \
&& for n in bg_mon-${BG_MON_COMMIT} pg_auth_mon-${PG_AUTH_MON_COMMIT} set_user pg_permissions-${PG_PERMISSIONS_COMMIT} pg_tm_aux-${PG_TM_AUX_COMMIT} $EXTRA_EXTENSIONS; do \
make -C $n USE_PGXS=1 clean install-strip; \
done; \
done \
Expand Down Expand Up @@ -399,7 +402,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
COPY --from=dependencies-builder /builddeps/wal-g /usr/local/bin/

# Install patroni and wal-e
ENV PATRONIVERSION=2.1.3
ENV PATRONIVERSION=2.1.4
ENV WALE_VERSION=1.1.1
RUN export DEBIAN_FRONTEND=noninteractive \
&& set -ex \
Expand Down
2 changes: 1 addition & 1 deletion postgres-appliance/dependencies/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ ENV PACKAGES="libgdal20"
RUN export DEBIAN_FRONTEND=noninteractive \
&& echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend \
&& echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend \
&& && sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list \
&& sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list \
&& apt-get update \
&& apt-get install -y devscripts equivs \
&& mk-build-deps $SOURCES \
Expand Down
2 changes: 1 addition & 1 deletion postgres-appliance/runit/etcd/run
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/sh -e

exec 2>&1
exec env -i /bin/etcd --data-dir /run/etcd.data
exec env -i ETCD_UNSUPPORTED_ARCH=$(dpkg --print-architecture) /bin/etcd --data-dir /run/etcd.data
67 changes: 37 additions & 30 deletions postgres-appliance/scripts/configure_spilo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@
PROVIDER_UNSUPPORTED = "unsupported"
USE_KUBERNETES = os.environ.get('KUBERNETES_SERVICE_HOST') is not None
KUBERNETES_DEFAULT_LABELS = '{"application": "spilo"}'
MEMORY_LIMIT_IN_BYTES_PATH = '/sys/fs/cgroup/memory/memory.limit_in_bytes'
PATRONI_DCS = ('zookeeper', 'exhibitor', 'consul', 'etcd3', 'etcd')
PATRONI_DCS = ('kubernetes', 'zookeeper', 'exhibitor', 'consul', 'etcd3', 'etcd')
AUTO_ENABLE_WALG_RESTORE = ('WAL_S3_BUCKET', 'WALE_S3_PREFIX', 'WALG_S3_PREFIX', 'WALG_AZ_PREFIX', 'WALG_SSH_PREFIX')
WALG_SSH_NAMES = ['WALG_SSH_PREFIX', 'SSH_PRIVATE_KEY_PATH', 'SSH_USERNAME', 'SSH_PORT']

Expand Down Expand Up @@ -647,9 +646,18 @@ def get_placeholders(provider):
'envdir "{WALE_ENV_DIR}" {WALE_BINARY} wal-push "%p"'.format(**placeholders) \
if placeholders['USE_WALE'] else '/bin/true'

if os.path.exists(MEMORY_LIMIT_IN_BYTES_PATH):
with open(MEMORY_LIMIT_IN_BYTES_PATH) as f:
cgroup_memory_limit_path = '/sys/fs/cgroup/memory/memory.limit_in_bytes'
cgroup_v2_memory_limit_path = '/sys/fs/cgroup/memory.max'

if os.path.exists(cgroup_memory_limit_path):
with open(cgroup_memory_limit_path) as f:
os_memory_mb = int(f.read()) / 1048576
elif os.path.exists(cgroup_v2_memory_limit_path):
with open(cgroup_v2_memory_limit_path) as f:
try:
os_memory_mb = int(f.read()) / 1048576
except Exception: # string literal "max" is a possible value
os_memory_mb = 0x7FFFFFFFFFF
else:
os_memory_mb = sys.maxsize
os_memory_mb = min(os_memory_mb, os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') / 1048576)
Expand Down Expand Up @@ -689,39 +697,38 @@ def pystache_render(*args, **kwargs):


def get_dcs_config(config, placeholders):
# (KUBERNETES|ZOOKEEPER|EXHIBITOR|CONSUL|ETCD3|ETCD)_(HOSTS|HOST|PORT|...)
dcs_configs = defaultdict(dict)
for name, value in placeholders.items():
if '_' not in name:
continue
dcs, param = name.lower().split('_', 1)
if dcs in PATRONI_DCS:
if param == 'hosts':
if not (value.strip().startswith('-') or '[' in value):
value = '[{0}]'.format(value)
value = yaml.safe_load(value)
elif param == 'discovery_domain':
param = 'discovery_srv'
dcs_configs[dcs][param] = value

if USE_KUBERNETES and placeholders.get('DCS_ENABLE_KUBERNETES_API'):
config = {'kubernetes': dcs_configs['kubernetes']}
try:
kubernetes_labels = json.loads(placeholders.get('KUBERNETES_LABELS'))
kubernetes_labels = json.loads(config['kubernetes'].get('labels'))
except (TypeError, ValueError) as e:
logging.warning("could not parse kubernetes labels as a JSON: {0}, "
"reverting to the default: {1}".format(e, KUBERNETES_DEFAULT_LABELS))
logging.warning("could not parse kubernetes labels as a JSON: %r, "
"reverting to the default: %s", e, KUBERNETES_DEFAULT_LABELS)
kubernetes_labels = json.loads(KUBERNETES_DEFAULT_LABELS)
config['kubernetes']['labels'] = kubernetes_labels

config = {'kubernetes': {'role_label': placeholders.get('KUBERNETES_ROLE_LABEL'),
'scope_label': placeholders.get('KUBERNETES_SCOPE_LABEL'),
'labels': kubernetes_labels}}
if not placeholders.get('KUBERNETES_USE_CONFIGMAPS'):
config['kubernetes'].update({'use_endpoints': True, 'pod_ip': placeholders['instance_data']['ip'],
'ports': [{'port': 5432, 'name': 'postgresql'}]})
if str(placeholders.get('KUBERNETES_BYPASS_API_SERVICE')).lower() == 'true':
if not config['kubernetes'].pop('use_configmaps'):
config['kubernetes'].update({'use_endpoints': True, 'ports': [{'port': 5432, 'name': 'postgresql'}]})
if str(config['kubernetes'].pop('bypass_api_service', None)).lower() == 'true':
config['kubernetes']['bypass_api_service'] = True
else:
# (ZOOKEEPER|EXHIBITOR|CONSUL|ETCD3|ETCD)_(HOSTS|HOST|PORT|...)
dcs_configs = defaultdict(dict)
for name, value in placeholders.items():
if '_' not in name:
continue
dcs, param = name.lower().split('_', 1)
if dcs in PATRONI_DCS:
if param == 'hosts':
if not (value.strip().startswith('-') or '[' in value):
value = '[{0}]'.format(value)
value = yaml.safe_load(value)
elif param == 'discovery_domain':
param = 'discovery_srv'
dcs_configs[dcs][param] = value
for dcs in PATRONI_DCS:
if dcs in dcs_configs:
if dcs != 'kubernetes' and dcs in dcs_configs:
config = {dcs: dcs_configs[dcs]}
break
else:
Expand Down Expand Up @@ -1026,7 +1033,7 @@ def main():
user_config_copy = deepcopy(user_config)
config = deep_update(user_config_copy, config)

if provider == PROVIDER_LOCAL and not any(1 for key in config.keys() if key == 'kubernetes' or key in PATRONI_DCS):
if provider == PROVIDER_LOCAL and not any(1 for key in config.keys() if key in PATRONI_DCS):
link_runit_service(placeholders, 'etcd')
config['etcd'] = {'host': '127.0.0.1:2379'}

Expand Down

0 comments on commit cdae614

Please sign in to comment.