diff --git a/INSTALL.md b/INSTALL.md
new file mode 100644
index 0000000000..3574e8625d
--- /dev/null
+++ b/INSTALL.md
@@ -0,0 +1,241 @@
+# Testing foreman
+
+This is a rough guide for applying the code base on an existing Foreman 3.8/3.9
+installation on AlmaLinux 8.
+
+This works by introducing a nother site, called `test`. The idea is to have
+test systems that are isolated from the rest of the LSST infrastructure. For
+example IPA isn't managed and no real route53 keys are provided.
+
+## Setup the VM
+
+We use Hetzner for cloud instances to test setups:
+
+```
+hcloud server create --image=alma-8 --name=$host --type=cpx41 --ssh-key='bastelfreak'
+hcloud server set-rdns $host --ip=95.217.179.41 --hostname=$host
+hcloud server set-rdns $host --ip=2a01:4f9:c012:acee::1 --hostname=$host
+```
+
+(Now also add matching A/AAAA records to make this easier)
+
+```
+ssh-keygen -f ~/.ssh/known_hosts -R $host
+ssh-keyscan $host >> ~/.ssh/known_hosts
+```
+
+## Patching
+
+```
+sed --in-place 's/SELINUX=permissive/SELINUX=disabled/' /etc/selinux/config
+echo 'if [ $TERM == "alacritty" ]; then export TERM=xterm-256color; fi' > /etc/profile.d/terminal.sh
+LC_ALL=en_US.UTF-8 dnf -y update
+LC_ALL=en_US.UTF-8 dnf -y install vim glibc-all-langpacks git bash-completion epel-release
+crb enable
+sync
+reboot
+```
+
+### Make vim less shitty
+
+also this provides a persistent undo history in case I derp in config files
+
+```
+mkdir -p ~/.vim/{backupdir,undodir}
+wget https://gist.githubusercontent.com/bastelfreak/a3cfa50db2a7be92c47f246f8f22ca5c/raw/dab14889680d4a8bbcb83580185ca2e5040d5947/vla.vimrc -O ~/.vimrc
+```
+
+### Helpful tools
+
+Those are helpful during testing
+
+```
+dnf -y install htop tig jq
+```
+
+## install Puppet + Foreman
+
+```
+dnf -y install https://yum.puppet.com/puppet7-release-el-8.noarch.rpm
+dnf -y install https://yum.theforeman.org/releases/3.8/el8/x86_64/foreman-release.rpm
+dnf -y module enable foreman:el8
+dnf -y install foreman-installer
+foreman-installer --enable-foreman-plugin-puppetdb
+dnf -y install puppetdb puppetdb-termini postgresql-contrib
+```
+
+Output from the installer should be like this:
+
+```
+[root@lsst ~]# foreman-installer --enable-foreman-plugin-puppetdb
+2024-02-11 18:57:31 [NOTICE] [root] Loading installer configuration. This will take some time.
+2024-02-11 18:57:33 [NOTICE] [root] Running installer with log based terminal output at level NOTICE.
+2024-02-11 18:57:33 [NOTICE] [root] Use -l to set the terminal output log level to ERROR, WARN, NOTICE, INFO, or DEBUG. See --full-help for definitions.
+2024-02-11 18:57:35 [NOTICE] [configure] Starting system configuration.
+2024-02-11 18:58:30 [NOTICE] [configure] 250 configuration steps out of 1244 steps complete.
+2024-02-11 18:58:39 [NOTICE] [configure] 500 configuration steps out of 1247 steps complete.
+2024-02-11 18:58:45 [NOTICE] [configure] 750 configuration steps out of 1272 steps complete.
+2024-02-11 18:58:56 [NOTICE] [configure] 1000 configuration steps out of 1272 steps complete.
+2024-02-11 19:00:12 [NOTICE] [configure] 1250 configuration steps out of 1272 steps complete.
+2024-02-11 19:00:15 [NOTICE] [configure] System configuration has finished.
+Executing: foreman-rake upgrade:run
+  Success!
+  * Foreman is running at https://foreman
+      Initial credentials are admin / s2hYUi7oEksKxaNM
+  * Foreman Proxy is running at https://foreman
+
+The full log is at /var/log/foreman-installer/foreman.log
+[root@lsst ~]#
+```
+
+### Configure r10k
+
+# Install r10k + control-repo
+
+First we want to stop puppet so it doesn't make unexpected changes in the
+background after code got deployed.
+
+```
+systemctl disable --now puppet
+```
+
+Now install r10k
+
+```
+source /etc/profile.d/puppet-agent.sh
+# required if we're on Puppet 7, which contains Ruby 2.7. newer faraday wants ruby 3
+puppet resource package faraday ensure=2.8.1 provider=puppet_gem
+puppet resource package r10k ensure=installed provider=puppet_gem
+ln -s /opt/puppetlabs/puppet/bin/r10k /usr/local/bin/
+```
+
+configure r10k
+
+```
+mkdir -p /etc/puppetlabs/r10k
+cat > /etc/puppetlabs/r10k/r10k.yaml << EOF
+---
+pool_size: 8
+deploy:
+  generate_types: true
+  purge_levels:
+  - deployment
+  exclude_spec: true
+  incremental: true
+:postrun: []
+:cachedir: /opt/puppetlabs/puppet/cache/r10k
+:sources:
+  puppet:
+    basedir: /etc/puppetlabs/code/environments
+    remote: https://github.com/bastelfreak/lsst-control
+EOF
+```
+
+deploy the code
+
+```
+r10k deploy environment production bastelfreak --modules --verbose --color
+```
+
+## Configure PuppetDB
+
+Setup the database and user
+
+```
+su --login postgres --command 'createuser --no-createdb --no-createrole --no-superuser puppetdb'
+su --login postgres --command 'createuser --no-createdb --no-createrole --no-superuser puppetdb_read'
+su --login postgres --command 'createdb --encoding UTF8 --owner postgres puppetdb'
+su --login postgres --command "psql puppetdb --command 'revoke create on schema public from public'"
+su --login postgres --command "psql puppetdb --command 'grant create on schema public to puppetdb'"
+su --login postgres --command "psql puppetdb --command 'alter default privileges for user puppetdb in schema public grant select on tables to puppetdb_read'"
+su --login postgres --command "psql puppetdb --command 'alter default privileges for user puppetdb in schema public grant usage on sequences to puppetdb_read'"
+su --login postgres --command "psql puppetdb --command 'alter default privileges for user puppetdb in schema public grant execute on functions to puppetdb_read'"
+su --login postgres --command "psql puppetdb --command 'create extension pg_trgm'"
+su --login postgres --command "psql puppetdb --command \"ALTER USER puppetdb WITH PASSWORD 'PASSWORD'\""
+su --login postgres --command "psql puppetdb --command \"ALTER USER puppetdb_read WITH PASSWORD 'PASSWORD'\""
+```
+
+Tell PuppetDB to use the database
+
+```
+echo '[database]' > /etc/puppetlabs/puppetdb/conf.d/database.ini
+echo 'subname = //127.0.0.1:5432/puppetdb' >> /etc/puppetlabs/puppetdb/conf.d/database.ini
+echo 'username = puppetdb' >> /etc/puppetlabs/puppetdb/conf.d/database.ini
+echo 'password = PASSWORD' >> /etc/puppetlabs/puppetdb/conf.d/database.ini
+echo '[read-database]' >> /etc/puppetlabs/puppetdb/conf.d/database.ini
+echo 'subname = //127.0.0.1:5432/puppetdb' >> /etc/puppetlabs/puppetdb/conf.d/database.ini
+echo 'username = puppetdb_read' >> /etc/puppetlabs/puppetdb/conf.d/database.ini
+echo 'password = PASSWORD' >> /etc/puppetlabs/puppetdb/conf.d/database.ini
+```
+
+Start PuppetDB
+
+```
+systemctl enable --now puppetdb
+```
+
+Update Puppetserver to talk to PuppetDB
+```
+puppet config set --section server storeconfigs true
+puppet config set --section main reports foreman,puppetdb
+echo -e "[main]\nserver_urls = https://$(hostname -f):8081/\nsoft_write_failure = true" > /etc/puppetlabs/puppet/puppetdb.conf
+systemctl restart puppetserver
+```
+
+## configure node in foreman
+
+We need to ensure foreman knows the environment `bastelfreak` before we can
+assign it
+
+* login at https://foreman/
+* got to https://foreman/foreman_puppet/environments, import new environments
+
+We need to set the environment in foreman
+
+* login at https://foreman/
+* select the node, click edit
+    * should bring you to https://foreman/hosts/foreman/edit
+* At environment, select `bastelfreak`
+* save
+
+We need to set the role and site
+
+* login at https://foreman/
+* At https://foreman/hosts/foreman/edit, go to `Parameters`
+* Select `Add Parameter`
+* Name=site, Value=test; save
+* Repeat: Name=role, Value=foreman; save
+
+
+At the moment a full puppet run doesn't succeed, but we can apply the following tags:
+
+```
+puppet agent -t --tags accounts,prometheus,chrony,yumrepo,auditd,tftp,convenience,debugutils,rsyslog,discovery,puppetserver,host,irqbalance,ssh,lldpd,sysstat,r10k,webhook,timezone,selinux,yum,docker,firewall,foreman_envsync,resolv_conf,sudo,postgresql_conf,udevd,reboot.target
+```
+
+Due to this we miss some migrations:
+
+```
+systemctl restart foreman
+foreman-rake db:migrate
+```
+
+Then we can reboot:
+
+```
+sync; reboot
+```
+
+## Rebuilding the instance
+
+```
+hcloud server rebuild $host --image=alma-8
+ssh-keygen -f ~/.ssh/known_hosts -R $host
+ssh-keyscan $host >> ~/.ssh/known_hosts
+```
+
+## Final updates
+
+**update**: After a bit of playing with Hiera, Puppet now succeeds within two
+runs. The `foreman_config_entry` resources only work on the second run, maybe
+because they have a missing dependency to one of the foreman packages.
diff --git a/Puppetfile b/Puppetfile
index 0f9f986057..811ca7fb9b 100644
--- a/Puppetfile
+++ b/Puppetfile
@@ -112,7 +112,7 @@ mod 'stm/debconf', '5.0.0'
 mod 'syseleven/restic', '2.6.1'
 mod 'theforeman/dhcp', git: 'https://github.com/lsst-it/puppet-dhcp', ref: '4d48173'  # https://github.com/theforeman/puppet-dhcp/pull/226
 mod 'theforeman/dns', '10.1.0'
-mod 'theforeman/foreman', git: 'https://github.com/lsst-it/puppet-foreman', ref: '70b70bc'  # 20.2.0 + dep updates
+mod 'theforeman/foreman', git: 'https://github.com/theforeman/puppet-foreman', ref: '24.1.0'
 mod 'theforeman/foreman_proxy', git: 'https://github.com/lsst-it/puppet-foreman_proxy', ref: '39ef803'  # https://github.com/theforeman/puppet-foreman_proxy/pull/772 https://github.com/theforeman/puppet-foreman_proxy/pull/816
 mod 'theforeman/puppet', git: 'https://github.com/lsst-it/puppet-puppet', ref: '8ef01c3'  # https://github.com/theforeman/puppet-puppet/pull/891
 mod 'theforeman/puppetserver_foreman', '2.4.0'
diff --git a/hieradata/role/foreman.yaml b/hieradata/role/foreman.yaml
index 6a4fc584aa..a1f2c6dfe5 100644
--- a/hieradata/role/foreman.yaml
+++ b/hieradata/role/foreman.yaml
@@ -81,6 +81,7 @@ foreman::oauth_active: true
 #foreman::oauth_consumer_key:  # secret
 #foreman::oauth_consumer_secret:  # secret
 foreman::passenger: false  # use puma; param removed in theforman/foreman >= 17.0.0
+# we need to figure out how to configure columns in newer foreman versions
 foreman::plugin::column_view::columns:
   role:
     title: "Role"
@@ -202,8 +203,9 @@ profile::core::foreman::foreman_config:
   bmc_credentials_accessible: {value: false}  # disable bmc pass in enc yaml
   default_pxe_item_global: {value: "discovery"}
   destroy_vm_on_host_delete: {value: true}
-  discovery_fact_column: {value: "ipmi_ipaddress,ipmi_macaddress"}
-  discovery_hostname: {value: "ipmi_macaddress,discovery_bootif"}
+  # on older foreman/puppet-foreman it wasn't idempotent to set this, but it works withforeman 3.8/ puppet-foreman 24.1.0
+  discovery_fact_column: {value: '["ipmi_ipaddress","ipmi_macaddress"]'}
+  discovery_hostname: {value: '["ipmi_macaddress","discovery_bootif"]'}
   entries_per_page: {value: 100}
   # remove "docker*" from default excluded_facts
   # XXX using block scalar style results in the double quotes being preceeded
@@ -214,7 +216,9 @@ profile::core::foreman::foreman_config:
   #excluded_facts:
   #  value: '["lo", "en*v*", "usb*", "vnet*", "macvtap*", ";vdsmdummy;", "veth*", "tap*", "qbr*", "qvb*", "qvo*", "qr-*", "qg-*", "vlinuxbr*", "vovsbr*", "br-int", "vif*", "load_averages::*", "memory::swap::available*", "memory::swap::capacity", "memory::swap::used*", "memory::system::available*", "memory::system::capacity", "memory::system::used*", "memoryfree", "memoryfree_mb", "swapfree", "swapfree_mb", "uptime_hours", "uptime_days"]'
   host_details_ui: {value: false}  # https://projects.theforeman.org/issues/35115
-  host_power_status: {value: false}
+  # since isn't required/doesn't work anymore since https://github.com/theforeman/foreman/pull/9462/files
+  # the option host_power_status doesn't exist in foreman 3.8, I think due to #9462, but that's a bit of a guess
+  # host_power_status: {value: false}
   idle_timeout: {value: 7200}  # session timeout in minutes
   ignore_puppet_facts_for_provisioning: {value: true}
   matchers_inheritance: {value: false}
diff --git a/hieradata/site/test.yaml b/hieradata/site/test.yaml
new file mode 100644
index 0000000000..9c8d587959
--- /dev/null
+++ b/hieradata/site/test.yaml
@@ -0,0 +1,38 @@
+---
+resolv_conf::nameservers:
+  - "185.12.64.2"
+  - "185.12.64.1"
+  - "2a01:4ff:ff00::add:2"
+  - "2a01:4ff:ff00::add:1"
+profile::core::foreman::manage_smee: false
+foreman_proxy::plugin::dns::route53::aws_access_key: "foo"
+foreman_proxy::plugin::dns::route53::aws_secret_key: "foo"
+puppet::server::puppetdb::server: "%{trusted.certname}"
+r10k::sources:
+  control:
+    remote: "https://github.com/bastelfreak/lsst-control"
+    basedir: "/etc/puppetlabs/code/environments"
+    invalid_branches: "correct"
+lookup_options:
+  r10k::sources:
+    merge:
+      strategy: "first"
+
+puppet::server_puppetserver_version: &server_version '7.15.0'
+puppet::server_version: '7.15.0'
+puppet_agent::package_version: '7.28.0'
+profile::core::yum::versionlock:
+  puppetdb-termini:
+    ensure: "present"
+    version: "7.16.0"
+    release: "1.el8"
+    before: "Package[puppetdb-termini]"
+
+foreman::repo::repo: "3.8"
+foreman::version: "3.8.0"
+puppetdb::globals::version: '7.16.0'
+
+profile::core::common::manage_sssd: false
+profile::core::common::manage_network_manager: false
+profile::core::common::manage_krb5: false
+profile::core::common::manage_ipa: false
diff --git a/manifests/site.pp b/manifests/site.pp
index 40bf4c1e49..67906efaa8 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -1,4 +1,4 @@
-lookup('classes', Array[String], 'unique').include
+lookup('classes', Array[String], 'unique', []).include
 
 $files = lookup(
   name          => 'files',
diff --git a/site/profile/manifests/core/foreman.pp b/site/profile/manifests/core/foreman.pp
index 1b90c7e45b..72dd93e1d7 100644
--- a/site/profile/manifests/core/foreman.pp
+++ b/site/profile/manifests/core/foreman.pp
@@ -39,7 +39,10 @@
   include foreman::compute::libvirt
   include foreman::compute::vmware
   include foreman_envsync
-  include foreman::plugin::column_view
+  # the plugin isn't supported in foreman 3.8 and newer
+  # https://github.com/theforeman/foreman_column_view
+  # it's now integrated into foreman
+  # include foreman::plugin::column_view
   include foreman::plugin::discovery
   include foreman::plugin::puppet
   include foreman::plugin::remote_execution