Merge branch 'develop' into nginx-apache-exporters

openfoodfacts · May 28, 2024 · b4f2480 · b4f2480
2 parents 0c375a2 + 4de1005
commit b4f2480
Show file tree

Hide file tree

Showing 24 changed files with 58 additions and 33 deletions.
diff --git a/confs/matomo/matomo/archive-2.env b/confs/matomo/matomo/archive-2.env
@@ -1,4 +1,5 @@
 # ENV file called by [email protected]
 # This is for website archiving of openfoodfacts.org
-MATOMO_OPTIONS="--force-idsites=2 --concurrent-archivers=0"
+# Note: comma is mandatory for --force-idsites to be taken into account correctly
+MATOMO_OPTIONS="--force-idsites=2, --concurrent-archivers=0"
 
diff --git a/confs/matomo/matomo/archive-5.env b/confs/matomo/matomo/archive-5.env
@@ -1,4 +1,5 @@
 # ENV file called by [email protected]
 # This is for generating archive of mobile app
-MATOMO_OPTIONS="--force-idsites=5 --concurrent-archivers=0"
+# Note: comma is mandatory for --force-idsites to be taken into account correctly
+MATOMO_OPTIONS="--force-idsites=5, --concurrent-archivers=0"
 
diff --git a/confs/matomo/matomo/archive-main.env b/confs/matomo/matomo/archive-main.env
@@ -1,4 +1,4 @@
 # ENV file called by [email protected]
 # This is for main archive generation, that is excluding specific high traffic services
-MATOMO_OPTIONS="--skip-idsites 5,2 --concurrent-archivers=0"
+MATOMO_OPTIONS="--skip-idsites=5,2 --concurrent-archivers=0"
 
diff --git a/confs/matomo/mysql/mariadb.conf.d/90-off-configs.cnf b/confs/matomo/mysql/mariadb.conf.d/90-off-configs.cnf
@@ -1,10 +1,12 @@
 [mysqld]
 # following https://matomo.org/faq/troubleshooting/faq_183/
 wait_timeout=28800
+interactive_timeout=28800
 net_read_timeout=28800
 net_write_timeout=28800
 innodb_log_file_size=2048M
 max_allowed_packet=2048M
+max_connections=400
 
 # following https://matomo.org/faq/on-premise/how-to-configure-matomo-for-speed/
 # see also https://mariadb.com/resources/blog/10-database-tuning-tips-for-peak-workloads/

diff --git a/confs/matomo/systemd/mariadb.service.d/override.conf b/confs/matomo/systemd/mariadb.service.d/override.conf
@@ -1,3 +1,3 @@
 [Unit]
-OnFailure=email-failures@mariadb-%H.service
+[email protected]
 
diff --git a/confs/matomo/systemd/[email protected] b/confs/matomo/systemd/[email protected]
@@ -20,4 +20,3 @@ StandardError=append:/var/log/matomo/matomo-archive-%i-err.log
 # place a timeout for bad cases as this process sometimes hangs
 # note: we have to use TimeoutStartSec as it's a Type=oneshot service
 TimeoutStartSec=5h40m
-
diff --git a/confs/matomo/systemd/php7.3-fpm.service.d/override.conf b/confs/matomo/systemd/php7.3-fpm.service.d/override.conf
@@ -1,3 +1,3 @@
 [Unit]
-[email protected]-%H.service
+[email protected]
 
diff --git a/confs/off2-reverse-proxy/systemd/system/nginx.service.d/override.conf b/confs/off2-reverse-proxy/systemd/system/nginx.service.d/override.conf
@@ -1,3 +1,7 @@
+[Service]
+# let nginx refine this
+LimitNOFILE=65535
+
 [Unit]
 [email protected]
-Restart=always
+Restart=always
diff --git a/confs/off2-reverse-proxy/systemd/system/prometheus-nginx-exporter.service.d/override.conf b/confs/off2-reverse-proxy/systemd/system/prometheus-nginx-exporter.service.d/override.conf
@@ -0,0 +1,3 @@
+[Service]
+# ONLY listen on 127.0.0.1 for security reasons
+Environment="LISTEN_ADDRESS=127.0.0.1:9113"
diff --git a/confs/proxy-off/nginx/free-exporters.openfoodfacts.org b/confs/proxy-off/nginx/free-exporters.openfoodfacts.org
@@ -19,9 +19,16 @@ server {
 
 # map from service to exporter
 map $uri $exporter {
+    # mongodb
     "/mongo/mongodb/metrics" 10.1.0.102:9216;
+    # nginx on this proxy
+    "/proxy/nginx/metrics" 127.0.0.1:9113;
+    # nginx on off
+    "/off/nginx/metrics" 10.1.0.113:9113;
 }
 
+#log_format debug_exporter "$time_local $scheme://$host:$server_port$request_uri $exporter";
+
 # https server
 server {
     listen 443 ssl http2;
@@ -33,15 +40,17 @@ server {
     ssl_certificate_key /etc/letsencrypt/live/free-exporters.openfoodfacts.org/privkey.pem;
     ssl_trusted_certificate /etc/letsencrypt/live/free-exporters.openfoodfacts.org/chain.pem;
 
-    # logs location
-    access_log  /var/log/nginx/free-exporters.log  main;
+    # logs location, avoid access_log, no need
+    #access_log  /var/log/nginx/free-exporters.log  debug_exporter;
+    #access_log  /var/log/nginx/free-exporters.log  main;
+    access_log  off;
     error_log   /var/log/nginx/free-exporters.errors.log;
 
     # protect
     auth_basic "Exporter metrics";
     auth_basic_user_file "/etc/nginx/.htpasswd/free-exporters";
 
-    # mongodb exporter
+    # url to join is defined thanks to exporter variable, itself based upon uri
     location / {
 
         if ($exporter = "") {

diff --git a/confs/proxy-off/nginx/nginx.conf b/confs/proxy-off/nginx/nginx.conf
@@ -4,7 +4,7 @@ pid /run/nginx.pid;
 include /etc/nginx/modules-enabled/*.conf;
 
 events {
-	worker_connections 768;
+	worker_connections 4096;
 	# multi_accept on;
 }
 

diff --git a/confs/proxy-off/nginx/stub_status b/confs/proxy-off/nginx/stub_status
@@ -0,0 +1,6 @@
+# This enables prometheus exporter to get data from nginx
+server {
+	listen 127.0.0.1:8080;
+        stub_status on;
+}
+
diff --git a/docs/linux-server.md b/docs/linux-server.md
@@ -27,7 +27,7 @@ We use either postfix or exim as a satellite of a smart_host.
 Every outgoing mail must pass through the proxmox mail gateway,
 which is registered in spf record and adds DKIM signature.
 
-For configuration, see [mail - Servers](./mail.md#Servers)
+For configuration, see [mail - Servers](./mail.md#servers)
 
 ## Iptables
 

diff --git a/docs/logs-off2.md b/docs/logs-off2.md
@@ -5,7 +5,7 @@ Keep things short or write a report.
 
 ## 2023-01-19
 
-see [2024-01-19 Certbot failure on off2 reverse proxy](./reports/2024-01-19-certbot-failure-off2-proxy)
+see [2024-01-19 Certbot failure on off2 reverse proxy](./reports/2024-01-19-certbot-failure-off2-proxy.md)
 
 ## 2023-01-02
 

diff --git a/docs/mail.md b/docs/mail.md
@@ -83,7 +83,7 @@ To add a new machine
 ```
 sudo iptables -t nat -A PREROUTING -s 213.36.253.206,213.36.253.208,146.59.148.140,51.210.154.203,1.210.32.79 -d pmg.openfoodfacts.org -p tcp  --dport 25 -j DNAT --to 10.1.0.102:25
 ```
-don't forget [to save iptables](./linux-servers#iptables)
+don't forget [to save iptables](./linux-server.md#iptables)
 
 (a generic masquerading rule for VM also exists)
 

diff --git a/docs/overview.md b/docs/overview.md
@@ -20,7 +20,7 @@ We have servers hosted by two providers:
 ### web traffic proxies
 
 - Most services are hosted on ovh,
-  and pass through an nginx proxy (see [promox - HTTP Reverse Proxy](./promox.md#http-reverse-proxy)) 
+  and pass through an nginx proxy (see [proxmox - HTTP Reverse Proxy](./proxmox.md#http-reverse-proxy)) 
   hosted on 101 VM on ovh1 which has a bridge with it's own ip.
 
 - product opener instances (openfoodfacts.org and its cousins) have their own proxy on [off1](#off1)
@@ -70,12 +70,12 @@ Main services:
 
 Located at ovh Strasbourg (sbg3)
 
-Uses [proxmox](./promox.md)
+Uses [proxmox](./proxmox.md)
 
-Part of [proxmox](./promox.md) cluster.
+Part of [proxmox](./proxmox.md) cluster.
 
 Contains lots of small services, as proxmox containers:
-- [http reverse proxy](./promox.md#http-reverse-proxy)
+- [http reverse proxy](./proxmox.md#http-reverse-proxy)
 - [Proxmox mail gateway](./mail.md)
 - blog
 - [odoo (connect)](./odoo.md)
@@ -85,7 +85,7 @@ Contains lots of small services, as proxmox containers:
 
 Located at ovh Roubaix (rbx8)
 
-Part of [proxmox](./promox.md) cluster.
+Part of [proxmox](./proxmox.md) cluster.
 
 Contains two big QEMU VMs hosting lots of docker services.
 One for staging, one for production.

diff --git a/docs/reports/2023-03-14-off2-opff-reinstall.md b/docs/reports/2023-03-14-off2-opff-reinstall.md
@@ -64,7 +64,7 @@ I add a problem running `ct_postinstall` as it wasn't able to fetch debian archi
 
 I then simply install `nginx` and `stunnel4` using apt.
 
-I also [configure postfix](../mail#postfix-configuration) and tested it.
+I also [configure postfix](../mail.md#postfix-configuration) and tested it.
 
 ### Adding the IP
 
@@ -288,7 +288,7 @@ sudo rsync --info=progress2 -a -x 10.0.0.1:/srv/opff/html/images/products  /zfs-
 ```
 this took 12 minutes.
 
-Then I sync to ovh3 (see also below [sanoid usage](#sanoid)):
+Then I sync to ovh3 (see also below [sanoid usage](#snapshots-and-syncs-with-sanoid)):
 
 ```bash
 time sudo  syncoid --no-sync-snap zfs-hdd/opff/images [email protected]:rpool/opff/images
@@ -504,7 +504,7 @@ I also did it for the data and cache dataset:
 I created a CT followings [How to create a new Container](../proxmox.md#how-to-create-a-new-container) it went all smooth.
 I choosed a 30Gb disk, 0B swap, 4 Cores and 6 Gb memory.
 
-I also [configure postfix](../mail#postfix-configuration) and tested it.
+I also [configure postfix](../mail.md#postfix-configuration) and tested it.
 
 
 ### Installing packages

diff --git a/docs/reports/2023-06-07-off2-opf-obf-reinstall.md b/docs/reports/2023-06-07-off2-opf-obf-reinstall.md
@@ -161,7 +161,7 @@ And on ovh3 add them to `sanoid.conf` with `synced_data` template
 I created a CT for obf followings [How to create a new Container](../proxmox.md#how-to-create-a-new-container) it went all smooth.
 I choosed a 30Gb disk, 0B swap, 4 Cores and 6 Gb memory.
 
-I also [configure postfix](../mail#postfix-configuration) and tested it.
+I also [configure postfix](../mail.md#postfix-configuration) and tested it.
 
 **Important:** do not create any user until you changed id maping in lxc conf (see [Mounting volumes](#mounting-volumes)). And also think about creating off user before any other user to avoid having to change users uids, off must have uid 1000.
 
@@ -1596,7 +1596,7 @@ scss:
   templates
 ```
 
-## Annex files difference for obf in git vs server
+## Annex files difference for opf in git vs server
 
 
 **TO DECIDE**

diff --git a/docs/reports/2023-07-off2-off-reinstall.md b/docs/reports/2023-07-off2-off-reinstall.md
@@ -45,7 +45,7 @@ I added a disk on zfs-nvme mounted on /var/lib/postgresql/ with 5Gb size and noa
 
 I did not create a user.
 
-I also [configure postfix](../mail#postfix-configuration) and tested it.
+I also [configure postfix](../mail.md#postfix-configuration) and tested it.
 
 ### Installed Postgres
 
@@ -108,7 +108,7 @@ I created a CT for OFF followings [How to create a new Container](../proxmox.md#
 It's 121 (off-memcached)
 I choosed a 15Gb disk on zfs-hdd, 0B swap, 2 Cores and 4 Gb memory.
 
-I also [configure postfix](../mail#postfix-configuration) and tested it.
+I also [configure postfix](../mail.md#postfix-configuration) and tested it.
 
 I did not create a user.
 
@@ -521,7 +521,7 @@ I choosed a 30Gb disk, 0B swap, 8 Cores and 40 Gb memory.
 
 Note that my first container creation failed because unable to mount the ZFS volume ("zfs dataset is busy"…), I had to destroy the dataset and re-create the container.
 
-I also [configure postfix](../mail#postfix-configuration) and tested it.
+I also [configure postfix](../mail.md#postfix-configuration) and tested it.
 
 **Important:** do not create any user until you changed id maping in lxc conf (see [Mounting volumes](#mounting-volumes)). And also think about creating off user before any other user to avoid having to change users uids, off must have uid 1000.
 

diff --git a/docs/reports/2023-12-08-off1-upgrade.md b/docs/reports/2023-12-08-off1-upgrade.md
@@ -114,7 +114,7 @@ We decided to disable the SSD card and continue installation.
 
 ### Installing Proxmox
 
-We boot on the USB key that we had brought with promox install CD on it.
+We boot on the USB key that we had brought with proxmox install CD on it.
 
 For Hard disk options, we choose ZFS - RAID1, ashift 12, compress an checksum on, disk size 64G
 
@@ -466,7 +466,7 @@ Simply following [our Munin doc on how to configure a server](../munin.md#how-to
 
 ## Configuring snapshots and syncoid
 
-I first installed sanoid following [install instructions](../sanoid.md#building-sanoid-deb)
+I first installed sanoid following [install instructions](../sanoid.md#how-to-build-and-install-sanoid-deb)
 
 We want to pull snapshots from off1 and to let ovh3 pull our snapshots.
 

diff --git a/docs/reports/2024-01-04-setting-up-stunnel.md b/docs/reports/2024-01-04-setting-up-stunnel.md
@@ -118,7 +118,7 @@ I also [configured email](../mail.md#postfix-configuration) in the container.
 
 ## Setting up stunnel on ovh1 stunnel-client
 
-Did the same as above to [set up stunnel on ovh1 proxy](./#setting-up-stunnel-on-off1-proxy).
+Did the same as above to [set up stunnel on ovh1 proxy](#setting-up-stunnel-on-off1-proxy).
 
 I created a key with `ssh-keygen -t ed25519 -C "[email protected]"` 
 add it as a deploy key to this projects 

diff --git a/docs/reports/2024-02-08-prod-redis-install.md b/docs/reports/2024-02-08-prod-redis-install.md
@@ -11,7 +11,7 @@ I created a CT on off2 followings [How to create a new Container](../proxmox.md#
 
 I did not create a user.
 
-I also [configure postfix](../mail#postfix-configuration) and [tested it](../mail#testing-that-the-gateway-is-well-configured).
+I also [configure postfix](../mail.md#postfix-configuration) and [tested it](../mail.md#testing-that-the-gateway-is-well-configured).
 
 Cloned this repository in [/opt using a root key as deploy key](../how-to-have-server-config-in-git.md)
 

diff --git a/docs/reports/2024-03-separate-hdd-docker-vms.md b/docs/reports/2024-03-separate-hdd-docker-vms.md
@@ -1,7 +1,7 @@
 # 2024-03 separate HDD on docker VMs
 
 We have docker VMs for staging and production.
-Disk are quite big. They use ext4 over a zfs in block mode (as proposed by promox).
+Disk are quite big. They use ext4 over a zfs in block mode (as proposed by proxmox).
 To keep things manageable, in case we have to switch server,
 I propose to split data from system. It might also help with backups.
 

diff --git a/scripts/readme-writer/requirements.txt b/scripts/readme-writer/requirements.txt
@@ -16,7 +16,7 @@ pyparsing==2.4.7
 pytablewriter==0.62.0
 python-dateutil==2.8.2
 pytz==2021.1
-requests==2.31.0
+requests==2.32.0
 six==1.16.0
 tabledata==1.2.0
 tcolorpy==0.1.1
Original file line number	Diff line number	Diff line change
Expand Up		@@ -20,4 +20,3 @@ StandardError=append:/var/log/matomo/matomo-archive-%i-err.log
		# place a timeout for bad cases as this process sometimes hangs
		# note: we have to use TimeoutStartSec as it's a Type=oneshot service
		TimeoutStartSec=5h40m