Skip to content

Commit

Permalink
Add netperf capability to collect CPU traces on Azure (#4774)
Browse files Browse the repository at this point in the history 
* add logic to collect cpu traces on azure

* better naming

* add some logging on the server side

* fix syntax issue

* fix command mismatch

* use wpr properly

* no such thing as wpr stop cpu

* add linux cpu tracing support

* we need *cpu-trace

* lets not worry about client and see if server is working on linux's side

* figure out are we collecting linux server cpu traces

* write to disk to persist state

* idk hopefully this works

* modify common command

* remove use of sudo here

---------

Co-authored-by: Nick Banks <[email protected]>
  • Loading branch information
@ProjectsByJackHe @nibanks
ProjectsByJackHe and nibanks authored Jan 30, 2025
1 parent 54c572a commit ad4a32e
Show file tree
Hide file tree
Showing 2 changed files with 60 additions and 13 deletions.
37 changes: 35 additions & 2 deletions scripts/quic_callback.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ param (
)

if ($PSVersionTable.PSVersion.Major -lt 7) {
$isWindows = $true
$IsWindows = $true
}

function SetLinuxLibPath {
Expand Down Expand Up @@ -32,6 +32,7 @@ function Wait-DriverStarted {
$mode = "maxtput"
$io = "iocp"
$stats = "0"
$env:linux_perf_prefix = ""

if ($Command.Contains("lowlat")) {
$mode = "lowlat"
Expand Down Expand Up @@ -61,7 +62,17 @@ function Repo-Path {
if ($Command.Contains("/home/secnetperf/_work/quic/artifacts/bin/linux/x64_Release_openssl/secnetperf")) {
Write-Host "Executing command: $(pwd)/artifacts/bin/linux/x64_Release_openssl/secnetperf -exec:$mode -io:$io -stats:$stats"
SetLinuxLibPath
./artifacts/bin/linux/x64_Release_openssl/secnetperf -exec:$mode -io:$io -stats:$stats

# Check and see if a 'perf_command.txt' file exists. If it does, then we need to prepend the command with the contents of the file.
if (Test-Path "perf_command.txt") {
Write-Host "Found 'perf_command.txt' file. Prepending the command with the contents of the file."
$perf_command = Get-Content "perf_command.txt"
Write-Host "Prepending the command with: $perf_command"
$env:linux_perf_prefix = $perf_command
}
Write-Host "About to invoke the expression: $env:linux_perf_prefix./artifacts/bin/linux/x64_Release_openssl/secnetperf -exec:$mode -io:$io -stats:$stats"
Invoke-Expression "$env:linux_perf_prefix./artifacts/bin/linux/x64_Release_openssl/secnetperf -exec:$mode -io:$io -stats:$stats"

} elseif ($Command.Contains("C:/_work/quic/artifacts/bin/windows/x64_Release_schannel/secnetperf")) {
Write-Host "Executing command: $(pwd)/artifacts/bin/windows/x64_Release_schannel/secnetperf -exec:$mode -io:$io -stats:$stats"
./artifacts/bin/windows/x64_Release_schannel/secnetperf -exec:$mode -io:$io -stats:$stats
Expand Down Expand Up @@ -98,6 +109,28 @@ if ($Command.Contains("/home/secnetperf/_work/quic/artifacts/bin/linux/x64_Relea
Write-Host "(SERVER) Installing Kernel driver. Path: $localSysPath"
sc.exe create "msquicpriv" type= kernel binpath= $localSysPath start= demand | Out-Null
net.exe start msquicpriv
} elseif ($Command.Contains("Start_Server_CPU_Tracing")) {
if ($IsWindows) {
Write-Host "Starting CPU tracing with WPR on windows!"
wpr -start CPU
} else {
Write-Host "Preprending the command with 'perf record' to start CPU tracing on linux!"
$filename = $Command.Split(";")[1]
$write_this_string_to_disk = "perf record -o server-cpu-traces-$filename -- "
# now write the string to disk
Set-Content -Path "perf_command.txt" -Value $write_this_string_to_disk
ls
}
} elseif ($Command.Contains("Stop_Server_CPU_Tracing")) {
if ($IsWindows) {
Write-Host "Stopping CPU tracing with WPR on windows!"
$filename = $Command.Split(";")[1]
wpr -stop "server-cpu-traces-$filename"
} else {
Write-Host "Nothing to do on Linux. 'perf' should have stopped recording."
Write-Host "Listing directory: "
ls
}
} else {
throw "Invalid command: $Command"
}
36 changes: 25 additions & 11 deletions scripts/secnetperf-helpers.psm1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ if ($psVersion.Major -lt 7) {

# Path to the WER registry key used for collecting dumps on Windows.
$WerDumpRegPath = "HKLM:\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps\secnetperf.exe"
$env:linux_perf_prefix = ""

# Write a GitHub error message to the console.
function Write-GHError($msg) {
Expand Down Expand Up @@ -307,12 +308,6 @@ function Start-RemoteServer {
throw "Server failed to start!"
}

# Passively starts the server on the remote machine by queuing up a new script to execute.
function Start-RemoteServerPassive {
param ($Command)
NetperfSendCommand $Command
}

# Sends a special UDP packet to tell the remote secnetperf to shutdown, and then
# waits for the job to complete. Finally, it returns the console output of the
# job.
Expand Down Expand Up @@ -342,7 +337,7 @@ function Wait-StartRemoteServerPassive {
for ($i = 0; $i -lt 30; $i++) {
Start-Sleep -Seconds 5 | Out-Null
Write-Host "Attempt $i to start the remote server, command: $FullPath -target:$RemoteName"
$Process = Start-LocalTest $FullPath "-target:$RemoteName" $OutputDir $UseSudo
$Process = Start-LocalTest $FullPath "-target:$RemoteName" $OutputDir $UseSudo ""
$ConsoleOutput = Wait-LocalTest $Process $OutputDir $false 30000 $true
Write-Host "Wait-StartRemoteServerPassive: $ConsoleOutput"
$DidMatch = $ConsoleOutput -match "Completed" # Look for the special string to indicate success.
Expand All @@ -356,15 +351,16 @@ function Wait-StartRemoteServerPassive {

# Creates a new local process to asynchronously run the test.
function Start-LocalTest {
param ($FullPath, $FullArgs, $OutputDir, $UseSudo)
param ($FullPath, $FullArgs, $OutputDir, $UseSudo, $LinuxPerfPrefix)
Write-Host "Starting Localtest with LinuxPerfPrefix: $LinuxPerfPrefix"
$pinfo = New-Object System.Diagnostics.ProcessStartInfo
if ($isWindows) {
$pinfo.FileName = $FullPath
$pinfo.Arguments = $FullArgs
} else {
# We use bash to execute the test so we can collect core dumps.
$NOFILE = Invoke-Expression "bash -c 'ulimit -n'"
$CommonCommand = "ulimit -n $NOFILE && ulimit -c unlimited && LD_LIBRARY_PATH=$(Split-Path $FullPath -Parent) LSAN_OPTIONS=report_objects=1 ASAN_OPTIONS=disable_coredump=0:abort_on_error=1 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 $FullPath $FullArgs && echo ''"
$CommonCommand = "ulimit -n $NOFILE && ulimit -c unlimited && LD_LIBRARY_PATH=$(Split-Path $FullPath -Parent) LSAN_OPTIONS=report_objects=1 ASAN_OPTIONS=disable_coredump=0:abort_on_error=1 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 $LinuxPerfPrefix$FullPath $FullArgs && echo ''"
if ($UseSudo) {
$pinfo.FileName = "/usr/bin/sudo"
$pinfo.Arguments = "/usr/bin/bash -c `"$CommonCommand`""
Expand Down Expand Up @@ -622,8 +618,18 @@ function Invoke-Secnetperf {
$StateDir = "/etc/_state"
}
if ($Session -eq "NOT_SUPPORTED") {
Start-RemoteServerPassive "$RemoteDir/$SecNetPerfPath $serverArgs"
if ($env:collect_cpu_traces) {
if ($IsWindows) {
wpr -start CPU
} else {
$env:linux_perf_prefix = "perf record -o cpu-traces-$scenario-$io-istcp-$tcp.data -- "
}
NetperfSendCommand "Start_Server_CPU_Tracing;$scenario-$io-istcp-$tcp.data"
NetperfWaitServerFinishExecution
}
NetperfSendCommand "$RemoteDir/$SecNetPerfPath $serverArgs"
Wait-StartRemoteServerPassive "$clientPath" $RemoteName $artifactDir $useSudo

} else {
$job = Start-RemoteServer $Session "$RemoteDir/$SecNetPerfPath" $serverArgs $useSudo
}
Expand All @@ -636,7 +642,8 @@ function Invoke-Secnetperf {
Write-Host "==============================`nRUN $($try+1):"
"> secnetperf $clientArgs" | Add-Content $clientOut
try {
$process = Start-LocalTest "$clientPath" $clientArgs $artifactDir $useSudo
Write-Host "About to start localtest with linux_perf_prefix: $env:linux_perf_prefix"
$process = Start-LocalTest "$clientPath" $clientArgs $artifactDir $useSudo $env:linux_perf_prefix
$rawOutput = Wait-LocalTest $process $artifactDir ($io -eq "wsk") 30000
Write-Host $rawOutput
$values[$tcp] += Get-TestOutput $rawOutput $metric
Expand Down Expand Up @@ -675,6 +682,13 @@ function Invoke-Secnetperf {
$Socket.Send($BytesToSend, $BytesToSend.Length, $RemoteName, 9999) | Out-Null
Write-Host "Sent special UDP packet to tell the server to die."
}
if ($env:collect_cpu_traces) {
if ($IsWindows) {
wpr -stop "cpu-traces-$scenario-$io-istcp-$tcp.etl"
}
NetperfSendCommand "Stop_Server_CPU_Tracing;$scenario-$io-istcp-$tcp.etl"
NetperfWaitServerFinishExecution
}
} else {
try { Stop-RemoteServer $job $RemoteName | Add-Content $serverOut } catch { }
}
Expand Down

0 comments on commit ad4a32e

Please sign in to comment.