test: SSH forwarding test

ic-os/guestos/context/Dockerfile

@@ -87,6 +87,7 @@ RUN rm /etc/ssh/ssh*key*
 # Actually, prohibit-password is the default config, so would not be
 # strictly necessary to be explicit here.
 RUN sed -e "s/.*PermitRootLogin.*/PermitRootLogin prohibit-password/" -i /etc/ssh/sshd_config
+RUN sed -e "s/.*AllowAgentForwarding.*/AllowAgentForwarding yes/" -i /etc/ssh/sshd_config
 
 RUN for SERVICE in /etc/systemd/system/*; do \
         if [ -f "$SERVICE" ] && [ ! -L "$SERVICE" ] && ! echo "$SERVICE" | grep -Eq "@\.service$"; then \

ic-os/hostos/context/Dockerfile

@@ -81,6 +81,7 @@ RUN rm /etc/ssh/ssh*key*
 # Actually, prohibit-password is the default config, so would not be
 # strictly necessary to be explicit here.
 RUN sed -e "s/.*PermitRootLogin.*/PermitRootLogin prohibit-password/" -i /etc/ssh/sshd_config
+RUN sed -e "s/.*AllowAgentForwarding.*/AllowAgentForwarding yes/" -i /etc/ssh/sshd_config
 
 RUN for SERVICE in /etc/systemd/system/*; do \
         if [ -f "$SERVICE" ] && [ ! -L "$SERVICE" ] && ! echo "$SERVICE" | grep -Eq "@\.service$"; then \

rs/tests/consensus/BUILD.bazel

@@ -371,3 +371,22 @@ system_test_nns(
         "@crate_index//:slog",
     ],
 )
+
+system_test_nns(
+    name = "ssh_agent_forwarding_test",
+    extra_head_nns_tags = [],  # don't run the head_nns variant on nightly since it aleady runs on long_test.
+    tags = [
+        "k8s",
+    ],
+    target_compatible_with = ["@platforms//os:linux"],  # requires libssh that does not build on Mac OS
+    runtime_deps = GUESTOS_RUNTIME_DEPS,
+    deps = [
+        # Keep sorted.
+        "//rs/registry/subnet_type",
+        "//rs/tests/consensus/utils",
+        "//rs/tests/driver:ic-system-test-driver",
+        "@crate_index//:anyhow",
+        "@crate_index//:candid",
+        "@crate_index//:slog",
+    ],
+)

rs/tests/consensus/Cargo.toml

@@ -48,6 +48,10 @@ path = "consensus_performance.rs"
 name = "ic-systest-adding-nodes-to-subnet-test"
 path = "adding_nodes_to_subnet_test.rs"
 
+[[bin]]
+name = "ic-systest-ssh-agent-forwarding-test"
+path = "ssh_agent_forwarding_test.rs"
+
 [[bin]]
 name = "ic-systest-node-graceful-leaving-test"
 path = "node_graceful_leaving_test.rs"

rs/tests/consensus/ssh_agent_forwarding_test.rs

@@ -0,0 +1,46 @@
+use anyhow::Result;
+use ic_consensus_system_test_utils::rw_message::install_nns_and_check_progress;
+use ic_registry_subnet_type::SubnetType;
+use ic_system_test_driver::driver::{
+    group::SystemTestGroup,
+    ic::{InternetComputer, Subnet},
+    test_env::TestEnv,
+    test_env_api::{HasTopologySnapshot, IcNodeContainer, SshSession},
+};
+use ic_system_test_driver::systest;
+use slog::info;
+
+fn setup(env: TestEnv) {
+    InternetComputer::new()
+        .add_subnet(Subnet::new(SubnetType::System).add_nodes(2))
+        .setup_and_start(&env)
+        .expect("Should be able to set up IC under test");
+
+    install_nns_and_check_progress(env.topology_snapshot());
+}
+
+fn test(env: TestEnv) {
+    let logger = env.logger();
+    let mut nns_nodes = env.topology_snapshot().root_subnet().nodes();
+    let (node1, node2) = (nns_nodes.next().unwrap(), nns_nodes.next().unwrap());
+
+    info!(logger, "Running local test");
+    let result = node1.block_on_bash_script("echo test").unwrap();
+    info!(logger, "Result: {result}");
+
+    info!(logger, "Running remote test");
+    let result = node1
+        .block_on_bash_script(&format!(
+            "ssh -vvv -A -o StrictHostKeyChecking=no admin@{} echo test2",
+            node2.get_ip_addr()
+        ))
+        .unwrap();
+    info!(logger, "Results: {result}");
+}
+
+fn main() -> Result<()> {
+    SystemTestGroup::new()
+        .with_setup(setup)
+        .add_test(systest!(test))
+        .execute_from_args()
+}

rs/tests/driver/src/driver/test_env_api.rs

@@ -202,9 +202,10 @@ use std::{
     ffi::OsStr,
     fs,
     future::Future,
-    io::{Read, Write},
+    io::{BufRead, BufReader, Read, Write},
     net::{IpAddr, Ipv4Addr, SocketAddr, TcpStream},
     path::{Path, PathBuf},
+    process::{Command, Stdio},
     str::FromStr,
     sync::Arc,
     time::{Duration, Instant},
@@ -1464,20 +1465,57 @@ pub trait SshSession {
     }
 
     fn block_on_bash_script_from_session(&self, session: &Session, script: &str) -> Result<String> {
+        println!("Opening channel");
         let mut channel = session.channel_session()?;
+        println!("Requesting agent forwarding");
+        channel.request_auth_agent_forwarding().unwrap();
+        println!("Executing command");
         channel.exec("bash").unwrap();
-
+        println!("Sending script: {}", script);
         channel.write_all(script.as_bytes())?;
         channel.flush()?;
         channel.send_eof()?;
-        let mut out = String::new();
-        channel.read_to_string(&mut out)?;
-        let mut err = String::new();
-        channel.stderr().read_to_string(&mut err)?;
+        println!("Reading response");
+
+        let stdout = BufReader::new(channel.stream(0));
+        let stderr = BufReader::new(channel.stream(1));
+
+        let stdout_thread = std::thread::spawn(move || {
+            let mut stdout_lines = Vec::new();
+            for line in stdout.lines() {
+                match line {
+                    Ok(line) => {
+                        println!("stdout: {}", line);
+                        stdout_lines.push(line);
+                    }
+                    Err(e) => eprintln!("Error reading stdout: {}", e),
+                }
+            }
+            stdout_lines.join("\n")
+        });
+
+        let stderr_thread = std::thread::spawn(move || {
+            let mut stderr_lines = Vec::new();
+            for line in stderr.lines() {
+                match line {
+                    Ok(line) => {
+                        eprintln!("stderr: {}", line);
+                        stderr_lines.push(line);
+                    }
+                    Err(e) => eprintln!("Error reading stderr: {}", e),
+                }
+            }
+            stderr_lines.join("\n")
+        });
+
+        let out = stdout_thread.join().expect("Failed to join stdout thread");
+        let err = stderr_thread.join().expect("Failed to join stderr thread");
+
         let exit_status = channel.exit_status()?;
         if exit_status != 0 {
             bail!("block_on_bash_script: exit_status = {exit_status:?}. Output: {out} Err: {err}");
         }
+
         Ok(out)
     }
 }
@@ -1933,8 +1971,81 @@ pub fn get_ssh_session_from_env(env: &TestEnv, ip: IpAddr) -> Result<Session> {
     let priv_key_path = env
         .get_path(SSH_AUTHORIZED_PRIV_KEYS_DIR)
         .join(SSH_USERNAME);
-    sess.userauth_pubkey_file(SSH_USERNAME, None, priv_key_path.as_path(), None)?;
-    Ok(sess)
+
+    println!("Start ssh-agent");
+    let output = Command::new("ssh-agent")
+        .arg("-s")
+        .stdout(Stdio::piped())
+        .output()?;
+
+    if !output.status.success() {
+        return Err(anyhow!(
+            "Failed to start ssh-agent: {}",
+            String::from_utf8_lossy(&output.stderr)
+        ));
+    }
+
+    println!("Parse ssh-agent output");
+    let output_str = String::from_utf8(output.stdout)?;
+    for line in output_str.lines() {
+        if let Some((key, value)) = line.split_once('=') {
+            let value = value
+                .split_once(';')
+                .map(|(before, _)| before)
+                .unwrap_or(value);
+            std::env::set_var(key, value);
+            println!("Set environment variable: {}={}", key, value);
+        }
+    }
+
+    if let (Ok(sock), Ok(pid)) = (
+        std::env::var("SSH_AUTH_SOCK"),
+        std::env::var("SSH_AGENT_PID"),
+    ) {
+        println!("SSH_AUTH_SOCK: {}", sock);
+        println!("SSH_AGENT_PID: {}", pid);
+    } else {
+        println!("Failed to set SSH_AUTH_SOCK or SSH_AGENT_PID.");
+    }
+
+    println!("Adding SSH key");
+    let mut cmd = Command::new("ssh-add");
+    cmd.arg(priv_key_path.clone());
+    let output = cmd.output()?;
+    std::io::stdout().write_all(&output.stdout)?;
+
+    // Authenticate using the SSH agent
+    println!("Creating agent");
+    let mut agent = sess.agent()?;
+    println!("Connecting agent");
+    agent.connect()?;
+    println!("Listing Identities");
+    agent.list_identities()?;
+    let identities = agent.identities()?;
+    for identity in identities {
+        println!("Using identity: {}", identity.comment());
+        match agent.userauth(SSH_USERNAME, &identity) {
+            Ok(()) => {
+                println!(
+                    "Authenticated successfully with identity: {}",
+                    identity.comment()
+                );
+                break;
+            }
+            Err(err) => {
+                println!(
+                    "Failed to authenticate with identity {}: {err:?}",
+                    identity.comment()
+                )
+            }
+        }
+    }
+
+    if sess.authenticated() {
+        Ok(sess)
+    } else {
+        Err(anyhow!("Authentication failed"))
+    }
 }
 
 impl SshSession for IcNodeSnapshot {