[Phoenix] Python version of phoenix (enhanced)

Makefiles/server.mk

@@ -15,7 +15,7 @@ OARDIR_BINFILES = $(SRCDIR)/tools/oar_resources_init \
 		  # $(SRCDIR)/tools/oar_resources_init \
 		  # $(SRCDIR)/tools/oar_resources_add
 
-OARCONFDIR_BINFILES = $(SRCDIR)/tools/oar_phoenix.pl
+OARCONFDIR_BINFILES = $(SRCDIR)/tools/oar_phoenix.py
 
 MANDIR_FILES = $(SRCDIR)/../docs/man/man1/almighty.1 \
 	       $(SRCDIR)/../docs/man/man1/oaraccounting.1 \
@@ -61,7 +61,7 @@ clean: clean_shared
 	# $(OARDO_CLEAN) CMD_WRAPPER=$(OARDIR)/oarmonitor CMD_TARGET=$(DESTDIR)$(SBINDIR)/oarmonitor
 	$(OARDO_CLEAN) CMD_WRAPPER=$(OARDIR)/oar_resources_init CMD_TARGET=$(DESTDIR)$(SBINDIR)/oar_resources_init
 	$(OARDO_CLEAN) CMD_WRAPPER=$(OARDIR)/oar_resources_add CMD_TARGET=$(DESTDIR)$(SBINDIR)/oar_resources_add
-	$(OARDO_CLEAN) CMD_WRAPPER=$(OARCONFDIR)/oar_phoenix.pl CMD_TARGET=$(DESTDIR)$(SBINDIR)/oar_phoenix
+	$(OARDO_CLEAN) CMD_WRAPPER=$(OARCONFDIR)/oar_phoenix.py CMD_TARGET=$(DESTDIR)$(SBINDIR)/oar_phoenix
 
 build: build_shared
 	$(MAKE) -f Makefiles/man.mk build
@@ -74,7 +74,7 @@ build: build_shared
 	#$(OARDO_BUILD) CMD_WRAPPER=$(OARDIR)/oaradmissionrules CMD_TARGET=$(DESTDIR)$(SBINDIR)/oaradmissionrules
 	$(OARDO_BUILD) CMD_WRAPPER=$(OARDIR)/oar_resources_init CMD_TARGET=$(DESTDIR)$(SBINDIR)/oar_resources_init
 	$(OARDO_BUILD) CMD_WRAPPER=$(OARDIR)/oar_resources_add CMD_TARGET=$(DESTDIR)$(SBINDIR)/oar_resources_add
-	$(OARDO_BUILD) CMD_WRAPPER=$(OARCONFDIR)/oar_phoenix.pl CMD_TARGET=$(DESTDIR)$(SBINDIR)/oar_phoenix
+	$(OARDO_BUILD) CMD_WRAPPER=$(OARCONFDIR)/oar_phoenix.py CMD_TARGET=$(DESTDIR)$(SBINDIR)/oar_phoenix
 
 install: build install_shared
 	install -d $(DESTDIR)$(OARCONFDIR)
@@ -100,7 +100,7 @@ install: build install_shared
 	# $(OARDO_INSTALL) CMD_WRAPPER=$(OARDIR)/oarmonitor CMD_TARGET=$(DESTDIR)$(SBINDIR)/oarmonitor
 	$(OARDO_INSTALL) CMD_WRAPPER=$(OARDIR)/oar_resources_init CMD_TARGET=$(DESTDIR)$(SBINDIR)/oar_resources_init
 	$(OARDO_INSTALL) CMD_WRAPPER=$(OARDIR)/oar_resources_add CMD_TARGET=$(DESTDIR)$(SBINDIR)/oar_resources_add
-	$(OARDO_INSTALL) CMD_WRAPPER=$(OARCONFDIR)/oar_phoenix.pl CMD_TARGET=$(DESTDIR)$(SBINDIR)/oar_phoenix
+	$(OARDO_INSTALL) CMD_WRAPPER=$(OARCONFDIR)/oar_phoenix.py CMD_TARGET=$(DESTDIR)$(SBINDIR)/oar_phoenix
 
 uninstall: uninstall_shared
 	# @for file in $(OARCONFDIR_FILES); do rm -f $(DESTDIR)$(OARCONFDIR)/`basename $$file`; done
@@ -114,13 +114,13 @@ uninstall: uninstall_shared
 	$(OARDO_UNINSTALL) CMD_WRAPPER=$(OARDIR)/.oarproperty CMD_TARGET=$(DESTDIR)$(SBINDIR)/oarproperty
 	$(OARDO_UNINSTALL) CMD_WRAPPER=$(OARDIR)/.oarremoveresource CMD_TARGET=$(DESTDIR)$(SBINDIR)/oarremoveresource
 	$(OARDO_UNINSTALL) CMD_WRAPPER=$(OARDIR)/.oaraccounting CMD_TARGET=$(DESTDIR)$(SBINDIR)/oaraccounting
-	$(OARDO_UNINSTALL) CMD_WRAPPER=$(OARDIR)/.oarnotify CMD_TARGET=$(DESTDIR)$(SBINDIR)/oarnotify	
+	$(OARDO_UNINSTALL) CMD_WRAPPER=$(OARDIR)/.oarnotify CMD_TARGET=$(DESTDIR)$(SBINDIR)/oarnotify
 	$(OARDO_UNINSTALL) CMD_WRAPPER=$(OARDIR)/.oarqueue CMD_TARGET=$(DESTDIR)$(SBINDIR)/oarqueue
 	#$(OARDO_UNINSTALL) CMD_WRAPPER=$(OARDIR)/oaradmissionrules CMD_TARGET=$(DESTDIR)$(SBINDIR)/oaradmissionrules
 	#$(OARDO_UNINSTALL) CMD_WRAPPER=$(OARDIR)/oarmonitor CMD_TARGET=$(DESTDIR)$(SBINDIR)/oarmonitor
 	$(OARDO_UNINSTALL) CMD_WRAPPER=$(OARDIR)/oar_resources_init CMD_TARGET=$(DESTDIR)$(SBINDIR)/oar_resources_init
 	$(OARDO_UNINSTALL) CMD_WRAPPER=$(OARDIR)/oar_resources_add CMD_TARGET=$(DESTDIR)$(SBINDIR)/oar_resources_add
-	$(OARDO_UNINSTALL) CMD_WRAPPER=$(OARCONFDIR)/oar_phoenix.pl CMD_TARGET=$(DESTDIR)$(SBINDIR)/oar_phoenix	
+	$(OARDO_UNINSTALL) CMD_WRAPPER=$(OARCONFDIR)/oar_phoenix.py CMD_TARGET=$(DESTDIR)$(SBINDIR)/oar_phoenix
 
 .PHONY: install setup uninstall build clean
 

etc/oar.conf

@@ -54,7 +54,7 @@ LOG_CATEGORIES="all"
 
 # Enable debug log for managed remote commandes (disable by default)
 #DEBUG_REMOTE_COMMANDS="no"
-   
+
 # If you want to debug oarexec on nodes then affect 1 (only effective if DETACH_JOB_FROM_SERVER = 1)
 OAREXEC_DEBUG_MODE="0"
 
@@ -78,7 +78,7 @@ COSYSTEM_HOSTNAME="127.0.0.1"
 
 # Specify the database field that will be considered to fill the node file used
 # by the user on the first node of the job. for each different value of this
-# field then OAR will put 1 line in the node file (default: resource_id). 
+# field then OAR will put 1 line in the node file (default: resource_id).
 #NODE_FILE_DB_FIELD_DISTINCT_VALUES="resource_id"
 
 # If you want to free a process per job on the server you can change this tag
@@ -126,7 +126,7 @@ OPENSSH_CMD="/usr/bin/ssh -p 6667"
 FINAUD_FREQUENCY="300"
 
 # Set time after which Suspected resources become Dead (default is 0 and it
-# means never) 
+# means never)
 #DEAD_SWITCH_TIME="600"
 
 # Set to yes if you want to check the nodes that runs jobs. (no is the default value)
@@ -140,7 +140,7 @@ FINAUD_FREQUENCY="300"
 
 # sentinelle.pl
 # If you want to use sentinelle.pl then you must use this tag.
-# (sentinelle.pl is like a "for" of ssh but it adds timeout and window to 
+# (sentinelle.pl is like a "for" of ssh but it adds timeout and window to
 # avoid overloading the server)
 # (sentinelle.pl is provided with OAR in the install directory)
 PINGCHECKER_SENTINELLE_SCRIPT_COMMAND="%%OARDIR%%/sentinelle.pl -t 30 -w 20"
@@ -212,8 +212,8 @@ PINGCHECKER_SENTINELLE_SCRIPT_COMMAND="%%OARDIR%%/sentinelle.pl -t 30 -w 20"
 #SERVER_EPILOGUE_EXEC_FILE="/path/to/prog"
 
 #
-# File to execute just after a group of resources has been supected. It may 
-# be used to trigger automatic actions to heal the resources. The script is 
+# File to execute just after a group of resources has been supected. It may
+# be used to trigger automatic actions to heal the resources. The script is
 # started with the list of resources put in its STDIN: resource_id followed
 # by a space and the network_address (one line per resource)
 #SUSPECTED_HEALING_EXEC_FILE="/path/to/prog"
@@ -233,7 +233,7 @@ SCHEDULER_TIMEOUT="30"
 SCHEDULER_JOB_SECURITY_TIME="60"
 
 # Number of seconds before the start of an advance reservation, which besteffort
-# jobs must be killed at (in order to let time to get nodes back and healthy 
+# jobs must be killed at (in order to let time to get nodes back and healthy
 # for the advance reservation). Default is 0 seconds.
 # If SCHEDULER_BESTEFFORT_KILL_DURATION_BEFORE_RESERVATION < SCHEDULER_JOB_SECURITY_TIME
 # then SCHEDULER_JOB_SECURITY_TIME is used instead.
@@ -248,9 +248,9 @@ SCHEDULER_GANTT_HOLE_MINIMUM_TIME="300"
 # system(SQL ORDER syntax)
 SCHEDULER_RESOURCE_ORDER="scheduler_priority ASC, state_num ASC, available_upto DESC, suspended_jobs ASC, network_address ASC, resource_id ASC"
 
-# If next line is uncommented then OAR will automatically update the value of 
+# If next line is uncommented then OAR will automatically update the value of
 # "scheduler_priority" field corresponding to the besteffort jobs.
-# The syntax is field names separated by "/". The value affected to 
+# The syntax is field names separated by "/". The value affected to
 # "scheduler_priority" depends of the position of the field name.
 SCHEDULER_PRIORITY_HIERARCHY_ORDER="resource_id"
 
@@ -283,20 +283,20 @@ SCHEDULER_AVAILABLE_SUSPENDED_RESOURCE_TYPE="default"
 #RESERVATION_WAITING_RESOURCES_TIMEOUT="300"
 
 # Set the granularity of the OAR accounting feature (in seconds)
-# Used by the oaraccounting command and the 
+# Used by the oaraccounting command and the
 # oar_sched_gantt_with_timesharing_and_fairsharing* to calculate the timesharing
 # policy. Default is 1 day (86400s)
 #ACCOUNTING_WINDOW="86400"
 
 ################################
 # KAMELOT Scheduler parameters #
 ###############################################################################
-# Be carefull hierarchy labels must be declare below to be use in resource 
+# Be carefull hierarchy labels must be declare below to be use in resource
 # request. Labels' order does not matter here. Default value is
-# "resource_id,network_address,cpu,core" 
-HIERARCHY_LABELS="resource_id,network_address,cpu,core" 
+# "resource_id,network_address,cpu,core"
+HIERARCHY_LABELS="resource_id,network_address,cpu,core"
 
-# Number of jobs which will be scheduled by scheduling round for each queue where Kamelot is used 
+# Number of jobs which will be scheduled by scheduling round for each queue where Kamelot is used
 # ***NOT LIMITED by default***
 #MAX_JOB_PER_SCHEDULING_ROUND=1000
 
@@ -477,11 +477,11 @@ SCHEDULER_FAIRSHARING_MAX_JOB_PER_USER=30
 # - nodes are suspected if they do not wake up before a timeout
 # - some nodes can be kept always alive depending on some properties
 # - the launching of wakeup/shutdown commands can be windowized to prevent
-#   from electric peeks 
+#   from electric peeks
 # Possible values are "yes" and "no"
 ENERGY_SAVING_INTERNAL="no"
 
-# Path to the script used by the energy saving module to wake up nodes. 
+# Path to the script used by the energy saving module to wake up nodes.
 # This command is executed from the oar server host.
 # OAR puts the node list on its STDIN (one hostname by line).
 # The scheduler looks at the available_upto field in the resources table to know
@@ -500,12 +500,12 @@ ENERGY_SAVING_INTERNAL="no"
 # have to wake up and 2000 seconds otherwise.
 #ENERGY_SAVING_NODE_MANAGER_WAKEUP_TIMEOUT="900"
 
-# You can set up a number of nodes that must always be on. You can use the 
+# You can set up a number of nodes that must always be on. You can use the
 # syntax in the examples if you want a number of alive nodes of different types
 # (corresponding to a specific sql properties requierement).
 # Example 1: keep alive 10 nodes on the whole cluster:
 #ENERGY_SAVING_NODES_KEEPALIVE="type='default':10"
-# Example 2: keep alive 4 nodes on the paradent cluster AND 6 nodes on the 
+# Example 2: keep alive 4 nodes on the paradent cluster AND 6 nodes on the
 # paraquad cluster AND 2 nodes accepting besteffort
 #ENERGY_SAVING_NODES_KEEPALIVE="cluster='paradent':4,cluster='paraquad':6,besteffort='YES':2"
 # By default, keepalive is disabled:
@@ -519,8 +519,8 @@ ENERGY_SAVING_INTERNAL="no"
 
 # Parameter to bypas the window mechanism embedded in the energy saving module.
 # Possible values are "yes" and "no"
-# When set to "yes", the list of nodes to wake up or shut down is passed to 
-# ENERGY_SAVING_NODE_MANAGER_*_CMD through stdin. 
+# When set to "yes", the list of nodes to wake up or shut down is passed to
+# ENERGY_SAVING_NODE_MANAGER_*_CMD through stdin.
 #ENERGY_SAVING_WINDOW_FORKER_BYPASS="no"
 
 # Time in second between execution of each window.
@@ -533,7 +533,7 @@ ENERGY_SAVING_INTERNAL="no"
 #ENERGY_SAVING_WINDOW_TIMEOUT="120"
 
 # The energy saving module can be automatically restarted after reaching
-# this number of cycles. This is a workaround for some DBD modules that do 
+# this number of cycles. This is a workaround for some DBD modules that do
 # not always free memory correctly.
 #ENERGY_MAX_CYCLES_UNTIL_REFRESH=5000
 
@@ -556,7 +556,7 @@ ENERGY_SAVING_INTERNAL="no"
 
 
 # To allow users to hold or resume their jobs. By default it's restricted to oar and
-# root users due to global scheduling impact and possible priority bypassing.  
+# root users due to global scheduling impact and possible priority bypassing.
 #USERS_ALLOWED_HOLD_RESUME="yes"
 
 ###############################################################################
@@ -608,26 +608,26 @@ OARSH_OARSTAT_CMD="%%BINDIR%%/oarstat"
 # The following variable gives the OpenSSH options which the oarsh command must
 # understand in order to parse user commands.
 # The value of OPENSSH_OPTSTR must match the option string of OpenSSH ssh
-# command installed on the system (mind checking it is up-to-date, as OpenSSH 
-# development is very active), which can be found in the ssh.c file of the 
+# command installed on the system (mind checking it is up-to-date, as OpenSSH
+# development is very active), which can be found in the ssh.c file of the
 # OpenSSH sources. For instance, the following command extracts the option
 # string of OpenSSH 7.2p2 :
 #
-# $ cd path/to/openssh/sources/ 
+# $ cd path/to/openssh/sources/
 # $ grep getopt -A1 ssh.c | sed 's/.*"\(.\+\)".*/\1/' | xargs | sed 's/ //g'
 # 1246ab:c:e:fgi:kl:m:no:p:qstvxACD:E:F:GI:KL:MNO:PQ:R:S:TVw:W:XYy
 #
 OPENSSH_OPTSTR="1246ab:c:e:fgi:kl:m:no:p:qstvxACD:E:F:GI:J:KL:MNO:PQ:R:S:TVw:W:XYy"
 
 # The following variable sets the OpenSSH options which oarsh actually uses.
-# Any option which is filtered out from the OPENSSH_OPTSTR variable above is 
+# Any option which is filtered out from the OPENSSH_OPTSTR variable above is
 # just ignored (see oarsh -v for debug)
 # WARNING: if not fitlered out, some options may allow root exploit using oarsh.
-# At least the following OpenSSH options are recommanded to be fitlered out: 
+# At least the following OpenSSH options are recommanded to be fitlered out:
 #  -a -A -i -l -o -p -E -F -G -I -w
 OPENSSH_OPTSTR_FILTERED="1246b:c:e:fgkm:nqstvxCD:KL:MNO:PQ:R:S:TVW:XYy"
 
-# The following variable forces OpenSSH configuration options for the ssh call 
+# The following variable forces OpenSSH configuration options for the ssh call
 # made by in oarsh, so that, for security reasons, they cannot be set by the
 # user (whenever "o:" is not filtered out in the OPENSSH_OPTSTR_FILTERED
 # variable above).
@@ -692,16 +692,16 @@ API_ACCESS_TOKEN_EXPIRE_MINUTES="524160" # One year
 #API_JOBS_URI_DEFAULT_PARAMS="state=Finishing,Running,Resuming,Suspended,Launching,toLaunch,Waiting,toAckReservation,Hold"
 
 # Set to 0 if you want the API to provide relative uris.
-# Relative uris may help if your API is behind a reverse proxy, 
-# as you don't have to rewrite the uris, but due to the possible confusion 
-# between "resources" and "resources/", it may not work with some libraries 
+# Relative uris may help if your API is behind a reverse proxy,
+# as you don't have to rewrite the uris, but due to the possible confusion
+# between "resources" and "resources/", it may not work with some libraries
 # like "ruby Restfully".
 # In addition, you can use the X_API_PATH_PREFIX http header variable to prepend each uris
 # returned by the API by a given prefix. (for example: curl -i -H'X_API_PATH_PREFIX: http://prefix_a/la/noix/' ...)
 #API_ABSOLUTE_URIS=1
 
 # Api stress_factor script
-# This script should return at least a real value between 0 and 1 that is given by 
+# This script should return at least a real value between 0 and 1 that is given by
 # the OAR api for the "GET /stress_factor" URI.
 # Warning: this script is run by root and the output is parsed as a list of
 # variables as is!
@@ -729,7 +729,44 @@ API_ACCESS_TOKEN_EXPIRE_MINUTES="524160" # One year
 # The API will automatically append .<timestamp>.hdf5
 #API_COLMET_HDF5_PATH_PREFIX="/var/lib/colmet/hdf5/cluster"
 
-
+###########
 # Activate admission rules as files
+###############################################################################
+
 # The location for AR is /etc/oar/admission_rules.d
 ADMISSION_RULES_IN_FILES="yes"
+
+###########
+# PHOENIX #
+###############################################################################
+# System monitoring tool that checks for suspected nodes and performs reboots
+# if necessary.
+# Phoenix is an external tool that is run periodically from crontab
+
+# File where phoenix saves it's state
+#PHOENIX_DBFILE=/var/lib/oar/phoenix/oar_phoenix.db
+
+# Directory where logfiles are created in case of problems
+# PHOENIX_LOGDIR="/var/lib/oar/phoenix/"
+
+# Command sent to reboot a node (first attempt)
+# PHOENIX_SOFT_REBOOTCMD="ssh -p 6667 {NODENAME} oardodo /sbin/reboot"
+
+# Timeout for a soft rebooted node to be considered hard rebootable
+# PHOENIX_SOFT_TIMEOUT=300
+
+# Command sent to reboot a node (second attempt)
+# PHOENIX_HARD_REBOOTCMD="oardodo ipmitool -I lanplus -U admin -f /etc/ipmipasswd -H {NODENAME}-bmc power off || true;sleep 30;oardodo ipmitool -I lanplus -U admin -f /etc/ipmipasswd -H {NODENAME}-bmc power on;sleep 5;oardodo ipmitool -I lanplus -U admin -f /etc/ipmipasswd -H {NODENAME}-bmc power on"
+
+# Timeout (s) for a hard rebooted node to be considered really broken, then an email is sent
+# PHOENIX_HARD_TIMEOUT=300
+
+# Max number of simultaneous reboots (soft OR hard)
+# PHOENIX_MAX_REBOOTS=20
+
+# Timeout (s) for unix commands
+# PHOENIX_CMD_TIMEOUT=60
+
+# Properties of the broken nodes (SQL where clause)
+# PHOENIX_BROKEN_NODES="state='Suspected' and network_address NOT IN (SELECT distinct(network_address) FROM resources where resource_id IN (SELECT resource_id  FROM assigned_resources WHERE assigned_resource_index = 'CURRENT')) and network_address not like 'luke%' and network_address != 'dahu33';"
+

oar/api/query.py

@@ -6,7 +6,7 @@
 from oar.lib.basequery import BaseQuery, BaseQueryCollection
 
 # from oar.lib.models import (db, Job, Resource)
-from oar.lib.utils import cached_property, row2dict
+from oar.lib.utils import row2dict
 
 # from flask import abort, current_app
 # TODO: This whole file is to review since it has been adapted from flask and now use in fastapi.