Skip to content

Commit

Permalink
FMO-77: Modify microvm qemu into fmo-qemu (#90)
Browse files Browse the repository at this point in the history
- Add functionality to read pci passthrough information from a config file during runtime instead of prebuilt nix config
- Comments with "FMO" is one that modified compared to upstream microvm
- Remove microvm.devices as it is now unused

Signed-off-by: Anh Huy Bui <[email protected]>
  • Loading branch information
buianhhuy96 authored Dec 5, 2024
1 parent 7b7ff58 commit 85aa13e
Show file tree
Hide file tree
Showing 6 changed files with 548 additions and 58 deletions.
58 changes: 21 additions & 37 deletions hardware/fmo-os-rugged-laptop-7330.nix
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,26 @@
"vim"
"tcpdump"
]; # systemPackages
pciDevices = [
{
path = "0000:72:00.0";
}
{
path = "0000:00:1f.0";
}
{
path = "0000:00:1f.3";
}
{
path = "0000:00:1f.4";
}
{
path = "0000:00:1f.5";
}
{
path = "0000:00:1f.6";
}
]; # pciDevices
extraModules = [
{
users.users."ghaf".extraGroups = ["networkmanager"];
Expand Down Expand Up @@ -178,37 +198,6 @@
}; # services

microvm = {
devices = [
{
bus = "pci";
path = "0000:72:00.0";
}
{
bus = "pci";
path = "0000:00:1f.0";
}
{
bus = "pci";
path = "0000:00:1f.3";
}
{
bus = "pci";
path = "0000:00:1f.4";
}
{
bus = "pci";
path = "0000:00:1f.5";
}
{
bus = "pci";
path = "0000:00:1f.6";
}
{
bus = "usb";
path = "vendorid=0x0525,productid=0xa4a2";
}
]; # microvm.devices

volumes = [
{
image = "/var/tmp/netvm_internal.img";
Expand Down Expand Up @@ -266,19 +255,14 @@
"tcpdump"
"gpsd"
]; # systemPackages
pciDevices = [];
extraModules = [
{
users.users."ghaf".extraGroups = ["docker" "dialout"];
systemd.network.links."10-ethint0".extraConfig = "MTUBytes=1460";
microvm = {
mem = 4096;
vcpu = 2;
devices = [
{
bus = "usb";
path = "vendorid=0x1546,productid=0x01a9";
}
]; # microvm.devices
volumes = [
{
image = "/var/tmp/dockervm_internal.img";
Expand Down
23 changes: 6 additions & 17 deletions hardware/fmo-os-rugged-tablet-7230.nix
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,11 @@
"vim"
"tcpdump"
]; # systemPackages
pciDevices = [
{
path = "0000:00:14.3";
}
]; # pciDevices
extraModules = [
{
users.users."ghaf".extraGroups = ["networkmanager"];
Expand Down Expand Up @@ -178,17 +183,6 @@
}; # services

microvm = {
devices = [
{
bus = "pci";
path = "0000:00:14.3";
}
{
bus = "usb";
path = "vendorid=0x0525,productid=0xa4a2";
}
]; # microvm.devices

volumes = [
{
image = "/var/tmp/netvm_internal.img";
Expand Down Expand Up @@ -246,19 +240,14 @@
"tcpdump"
"gpsd"
]; # systemPackages
pciDevices = [];
extraModules = [
{
users.users."ghaf".extraGroups = ["docker" "dialout"];
systemd.network.links."10-ethint0".extraConfig = "MTUBytes=1460";
microvm = {
mem = 4096;
vcpu = 2;
devices = [
{
bus = "usb";
path = "vendorid=0x1546,productid=0x01a9";
}
]; # microvm.devices
volumes = [
{
image = "/var/tmp/dockervm_internal.img";
Expand Down
111 changes: 111 additions & 0 deletions modules/packages/fmo-qemu/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
{ pkgs
, microvmConfig
, toplevel
, ghafOS
}:

let
inherit (pkgs) lib;

inherit (import "${ghafOS.inputs.microvm}/lib" { nixpkgs-lib = lib; }) createVolumesScript makeMacvtap;
inherit (makeMacvtap {
inherit microvmConfig hypervisorConfig;
}) openMacvtapFds macvtapFds;

hypervisorConfig = import (./qemu.nix) {
inherit pkgs microvmConfig macvtapFds;
};

inherit (hypervisorConfig) command canShutdown shutdownCommand;
supportsNotifySocket = hypervisorConfig.supportsNotifySocket or false;
preStart = hypervisorConfig.preStart or microvmConfig.preStart;
tapMultiQueue = hypervisorConfig.tapMultiQueue or false;

execArg = lib.optionalString microvmConfig.prettyProcnames
''-a "microvm@${microvmConfig.hostName}"'';

# FMO: Use command in an array format
runScriptBin = pkgs.writeShellScriptBin "microvm-run" ''
${preStart}
${createVolumesScript pkgs.buildPackages microvmConfig.volumes}
${lib.optionalString (hypervisorConfig.requiresMacvtapAsFds or false) openMacvtapFds}
CMD=(${command})
exec ${execArg} "''${CMD[@]}"
'';

shutdownScriptBin = pkgs.writeShellScriptBin "microvm-shutdown" ''
${shutdownCommand}
'';

balloonScriptBin = pkgs.writeShellScriptBin "microvm-balloon" ''
set -e
if [ -z "$1" ]; then
echo "Usage: $0 <balloon-size-mb>"
exit 1
fi
SIZE=$1
${hypervisorConfig.setBalloonScript}
'';
in

pkgs.buildPackages.runCommand "microvm-${microvmConfig.hypervisor}-${microvmConfig.hostName}"
{
# for `nix run`
meta.mainProgram = "microvm-run";
passthru = {
inherit canShutdown supportsNotifySocket;
inherit (microvmConfig) hypervisor;
};
} ''
mkdir -p $out/bin
ln -s ${runScriptBin}/bin/microvm-run $out/bin/microvm-run
${if canShutdown
then "ln -s ${shutdownScriptBin}/bin/microvm-shutdown $out/bin/microvm-shutdown"
else ""}
${lib.optionalString ((hypervisorConfig.setBalloonScript or null) != null) ''
ln -s ${balloonScriptBin}/bin/microvm-balloon $out/bin/microvm-balloon
''}
mkdir -p $out/share/microvm
ln -s ${toplevel} $out/share/microvm/system
echo vnet_hdr > $out/share/microvm/tap-flags
${lib.optionalString tapMultiQueue ''
echo multi_queue >> $out/share/microvm/tap-flags
''}
${lib.concatMapStringsSep " " (interface:
lib.optionalString (interface.type == "tap" && interface ? id) ''
echo "${interface.id}" >> $out/share/microvm/tap-interfaces
'') microvmConfig.interfaces}
${lib.concatMapStringsSep " " (interface:
lib.optionalString (
interface.type == "macvtap" &&
interface ? id &&
(interface.macvtap.link or null) != null &&
(interface.macvtap.mode or null) != null
) ''
echo "${builtins.concatStringsSep " " [
interface.id
interface.mac
interface.macvtap.link
(builtins.toString interface.macvtap.mode)
]}" >> $out/share/microvm/macvtap-interfaces
'') microvmConfig.interfaces}
${lib.concatMapStrings ({ tag, socket, source, proto, ... }:
lib.optionalString (proto == "virtiofs") ''
mkdir -p $out/share/microvm/virtiofs/${tag}
echo "${socket}" > $out/share/microvm/virtiofs/${tag}/socket
echo "${source}" > $out/share/microvm/virtiofs/${tag}/source
''
) microvmConfig.shares}
${pkgs.coreutils}/bin/ln -s ${microvmConfig.pciConfigPath} $out/share/microvm/pci-devices
''
10 changes: 10 additions & 0 deletions modules/packages/fmo-qemu/pci-passthrough-options.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# Copyright 2022-2024 TII (SSRC) and the Ghaf contributors
# SPDX-License-Identifier: Apache-2.0
{pkgs, microvmConfig}:
# FMO: A helper function to generate pci passthrough options in the qemu command
pkgs.writeShellScriptBin "pci-passthrough-options" ''
pciDevices=$(cat ${microvmConfig.pciConfigPath})
for device in $pciDevices; do
echo -n "-device vfio-pci,host=$device,multifunction=on "
done
''
Loading

0 comments on commit 85aa13e

Please sign in to comment.