nodeinfo

#!/bin/bash
#
# nodeinfo script collecting system information
#
# Copyright (c) 2015-2024 Christoph Niethammer <niethammer@hlrs.de>
#
# Usage:  nodeinfo
#

function infoCommand() {
    title=$1
    cmd=$2
    shift 2
    args=$@
    echo "<info title='$title' command='$cmd${args:+ $args}'>"
    if ! type $cmd >/dev/null 2>/dev/null ; then
        echo "<!-- '$cmd' not available on this system -->"
    else
        echo "<commandtype><![CDATA["
        type $cmd
        echo "]]></commandtype>"
        echo "<commandoutput><![CDATA["
        $cmd $args
        echo "]]></commandoutput>"
    fi
    echo "</info>"
}

function infoFile() {
    title=$1
    cmd=$2
    file=$3
    echo "<info title='$title' command='$cmd $file'>"
    if [ ! -r "$file" ] ; then
        echo "<!-- '$file' not available on this system -->"
    else
        echo "<filetype><![CDATA["
        file $file
        echo "]]></filetype>"
        echo "<realfiletype><![CDATA["
        file $(realpath $file)
        echo "]]></realfiletype>"
        echo "<commandoutput><![CDATA["
        $cmd "$file"
        echo "]]></commandoutput>"
    fi
    echo "</info>"
}

function infoKernelModulesInfo() {
    echo "<info title='Kernel Module Infos' command='modinfo'>"
    cmd="modinfo"
    if ! type $cmd >/dev/null 2>/dev/null ; then
        echo "<!-- '$cmd' not available on this system -->"
    else
        echo "<commandtype><![CDATA["
        type $cmd
        echo "]]></commandtype>"
        echo "<commandoutput><![CDATA["
        kmodules=$(cut -d ' ' -f 1 /proc/modules | tail -n +2)
        for km in $kmodules; do
            $cmd "$km"
        done
        echo "]]></commandoutput>"
    fi
    echo "</info>"
}


echo "<nodeinfo>"

echo "<node hostname=\"${HOSTNAME}\">"
infoCommand "Host" "hostname"
infoCommand "Network interfaces" "ifconfig -a"
infoFile    "OS release" "cat" "/etc/os-release"
infoCommand "Environment" "env"
infoCommand "ulimit" "ulimit" "-a"
infoFile    "Kernel config" "zcat" "/proc/config.gz"
infoCommand "Filesystem mountpoints" "mount"
infoCommand "Block devices" "lsblk"
infoCommand "Filesystem usage" "df"
infoCommand "Lustre filesystem disk usage" "lfs" "df -h"
infoCommand "Quota info" "quota"
infoCommand "PCI devices" "lspci" "-tv"
infoCommand "PCI device details" "lspci" "-vvv"
infoCommand "Topology information" "lstopo" "--of xml"
infoCommand "lscpu" "lscpu"
infoFile    "CPU info" "cat" "/proc/cpuinfo"
infoFile    "Memory info" "cat" "/proc/meminfo"
infoCommand "numactl info show" "numactl" "--show"
infoCommand "numactl info hardware" "numactl" "--hardware"
infoCommand "likwid-topology" "likwid-topology" "-c -C -g"
infoCommand "Network configuration" "ip" "address show"
infoCommand "IB info" "ibstat" "-v"
infoCommand "OFED info" "ofed_info"
infoCommand "OpenCL device info" "clinfo"
infoCommand "NVIDIA GPU info" "nvidia-smi" "-q"
infoCommand "running processes:" "ps" "aux"
infoCommand "Kernel version" "uname" "-a"
infoCommand "Kernel modules" "lsmod"
infoKernelModulesInfo
infoCommand "Kernel module params" "tail" "-n+1 $(echo /sys/module/*/parameters/*)" 2>/dev/null
infoCommand "Kernel messages" "dmesg"
infoCommand "Executable format" "file" "/bin/ls"
infoCommand "Open MPI info" "ompi_info" "--all --all --parsable"
infoCommand "UCX info version" "ucx_info" "-v"
infoCommand "UCX info devices" "ucx_info" "-d"
infoCommand "UCX info config" "ucx_info" "-c"
infoCommand "UCX info build" "ucx_info" "-b"
infoCommand "PAPI version" "papi_version"
infoCommand "PAPI avail" "papi_avail"
infoCommand "Score-P info" "scorep-info" "config-summary"
# modules command writes output to stderr so we have to redirect stderr pipe here
infoCommand "Environment modules" "module" "avail" 2>&1
infoCommand "Lmod modules" "module" "spider" 2>&1

# check if we run in a job scheduling environment
if [[ ! -z $PBS_JOBID ]] ; then
    infoCommand "BPS jobs" "qstat" "-a"
    infoCommand "PBS job info" "qstat" "-f $PBS_JOBID"
    infoCommand "PBS nodes" "cat" "$PBS_NODEFILE"
    NODELIST=$(cat $PBS_NODEFILE)
fi
if [[ ! -z $SLURM_JOBID ]] ; then
    infoCommand "SLURM partitions" "sinfo" "-a"
    infoCommand "SLURM job info" "scontrol" "show -d job"
    infoCommand "SLURM nodes" "scontrol" "show hostname $SLURM_NODELIST"
    NODELIST=$(scontrol show hostname $SLURM_NODELIST)
fi
echo "</node>"

echo "</nodeinfo>"