From a585c51ec6efc53f58765bf59688c25303f19eb7 Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Mon, 7 Feb 2022 18:32:18 +0200 Subject: [PATCH 01/28] HYPR-161 Build seL4 accelerator --- accel/Kconfig | 3 +++ accel/meson.build | 2 ++ accel/sel4/meson.build | 1 + meson.build | 10 ++++++++++ meson_options.txt | 2 ++ qemu-options.hx | 8 ++++---- scripts/meson-buildoptions.sh | 2 ++ 7 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 accel/sel4/meson.build diff --git a/accel/Kconfig b/accel/Kconfig index 8bdedb7d15f..c891d66a610 100644 --- a/accel/Kconfig +++ b/accel/Kconfig @@ -16,6 +16,9 @@ config TCG config KVM bool +config SEL4 + bool + config XEN bool select FSDEV_9P if VIRTFS diff --git a/accel/meson.build b/accel/meson.build index b9a963cf80c..5f00d5633e4 100644 --- a/accel/meson.build +++ b/accel/meson.build @@ -8,6 +8,7 @@ if have_system subdir('qtest') subdir('kvm') subdir('xen') + subdir('sel4') subdir('stubs') endif @@ -18,3 +19,4 @@ dummy_ss.add(files( specific_ss.add_all(when: ['CONFIG_SOFTMMU', 'CONFIG_POSIX'], if_true: dummy_ss) specific_ss.add_all(when: ['CONFIG_XEN'], if_true: dummy_ss) +specific_ss.add_all(when: ['CONFIG_SEL4'], if_true: dummy_ss) diff --git a/accel/sel4/meson.build b/accel/sel4/meson.build new file mode 100644 index 00000000000..ce532d4fa34 --- /dev/null +++ b/accel/sel4/meson.build @@ -0,0 +1 @@ +specific_ss.add(when: 'CONFIG_SEL4', if_true: files('sel4-all.c')) diff --git a/meson.build b/meson.build index 20fddbd707c..23445e62773 100644 --- a/meson.build +++ b/meson.build @@ -137,6 +137,12 @@ if cpu in ['x86', 'x86_64'] } endif +if cpu in ['aarch64'] + accelerator_targets += { + 'CONFIG_SEL4': ['aarch64-softmmu'], + } +endif + modular_tcg = [] # Darwin does not support references to thread-local variables in modules if targetos != 'darwin' @@ -396,6 +402,9 @@ accelerators = [] if get_option('kvm').allowed() and targetos == 'linux' accelerators += 'CONFIG_KVM' endif +if not get_option('sel4').disabled() + accelerators += 'CONFIG_SEL4' +endif if get_option('whpx').allowed() and targetos == 'windows' if get_option('whpx').enabled() and host_machine.cpu() != 'x86_64' error('WHPX requires 64-bit host') @@ -3835,6 +3844,7 @@ if have_system summary_info += {'HVF support': config_all.has_key('CONFIG_HVF')} summary_info += {'WHPX support': config_all.has_key('CONFIG_WHPX')} summary_info += {'NVMM support': config_all.has_key('CONFIG_NVMM')} + summary_info += {'seL4 support': config_all.has_key('CONFIG_SEL4')} summary_info += {'Xen support': xen.found()} if xen.found() summary_info += {'xen ctrl version': xen.version()} diff --git a/meson_options.txt b/meson_options.txt index e58e158396b..2e767c526b4 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -74,6 +74,8 @@ option('hvf', type: 'feature', value: 'auto', description: 'HVF acceleration support') option('nvmm', type: 'feature', value: 'auto', description: 'NVMM acceleration support') +option('sel4', type: 'feature', value: 'auto', + description: 'seL4 backend support') option('xen', type: 'feature', value: 'auto', description: 'Xen backend support') option('xen_pci_passthrough', type: 'feature', value: 'auto', diff --git a/qemu-options.hx b/qemu-options.hx index 31c04f7eea0..93370187630 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -26,7 +26,7 @@ DEF("machine", HAS_ARG, QEMU_OPTION_machine, \ "-machine [type=]name[,prop[=value][,...]]\n" " selects emulated machine ('-machine help' for list)\n" " property accel=accel1[:accel2[:...]] selects accelerator\n" - " supported accelerators are kvm, xen, hax, hvf, nvmm, whpx or tcg (default: tcg)\n" + " supported accelerators are kvm, xen, sel4, hax, hvf, nvmm, whpx or tcg (default: tcg)\n" " vmport=on|off|auto controls emulation of vmport (default: auto)\n" " dump-guest-core=on|off include guest memory in a core dump (default=on)\n" " mem-merge=on|off controls memory merge support (default: on)\n" @@ -59,7 +59,7 @@ SRST ``accel=accels1[:accels2[:...]]`` This is used to enable an accelerator. Depending on the target - architecture, kvm, xen, hax, hvf, nvmm, whpx or tcg can be available. + architecture, kvm, xen, sel4, hax, hvf, nvmm, whpx or tcg can be available. By default, tcg is used. If there is more than one accelerator specified, the next one is used if the previous one fails to initialize. @@ -178,7 +178,7 @@ ERST DEF("accel", HAS_ARG, QEMU_OPTION_accel, "-accel [accel=]accelerator[,prop[=value][,...]]\n" - " select accelerator (kvm, xen, hax, hvf, nvmm, whpx or tcg; use 'help' for a list)\n" + " select accelerator (kvm, xen, sel4, hax, hvf, nvmm, whpx or tcg; use 'help' for a list)\n" " igd-passthru=on|off (enable Xen integrated Intel graphics passthrough, default=off)\n" " kernel-irqchip=on|off|split controls accelerated irqchip support (default=on)\n" " kvm-shadow-mem=size of KVM shadow MMU in bytes\n" @@ -189,7 +189,7 @@ DEF("accel", HAS_ARG, QEMU_OPTION_accel, SRST ``-accel name[,prop=value[,...]]`` This is used to enable an accelerator. Depending on the target - architecture, kvm, xen, hax, hvf, nvmm, whpx or tcg can be available. By + architecture, kvm, xen, sel4, hax, hvf, nvmm, whpx or tcg can be available. By default, tcg is used. If there is more than one accelerator specified, the next one is used if the previous one fails to initialize. diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh index 359b04e0e6d..ada2cf31f8c 100644 --- a/scripts/meson-buildoptions.sh +++ b/scripts/meson-buildoptions.sh @@ -458,6 +458,8 @@ _meson_option_parse() { --disable-vvfat) printf "%s" -Dvvfat=disabled ;; --enable-whpx) printf "%s" -Dwhpx=enabled ;; --disable-whpx) printf "%s" -Dwhpx=disabled ;; + --enable-sel4) printf "%s" -Dsel4=enabled ;; + --disable-sel4) printf "%s" -Dsel4=disabled ;; --enable-xen) printf "%s" -Dxen=enabled ;; --disable-xen) printf "%s" -Dxen=disabled ;; --enable-xen-pci-passthrough) printf "%s" -Dxen_pci_passthrough=enabled ;; From 4fc4d2eef755f26facb4cbb93019493b9a6c47e9 Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Wed, 1 Dec 2021 13:26:35 +0200 Subject: [PATCH 02/28] Add seL4 accelerator Signed-off-by: Hannu Lyytinen --- accel/sel4/sel4-all.c | 487 +++++++++++++++++++++++++++++++++++++++++ hw/arm/virt.c | 6 +- hw/pci-host/gpex.c | 4 + hw/virtio/virtio-pci.c | 17 ++ 4 files changed, 511 insertions(+), 3 deletions(-) create mode 100644 accel/sel4/sel4-all.c diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c new file mode 100644 index 00000000000..8b82b580607 --- /dev/null +++ b/accel/sel4/sel4-all.c @@ -0,0 +1,487 @@ +#include "qemu/osdep.h" +#include "qemu/error-report.h" +#include "qemu/module.h" +#include "qapi/error.h" +#include "qemu/accel.h" +#include "sysemu/cpus.h" +#include "hw/boards.h" +#include "hw/pci/pci.h" +#include "hw/pci/pci_bus.h" + +#include + +void tii_printf(const char *fmt, ...); + +bool sel4_allowed; + +void sel4_register_pci_device(PCIDevice *d); +static void send_qemu_op(uint32_t op, uint32_t param); + +static QemuThread sel4_virtio_thread; +static QemuMutex vmm_send_mutex; + +static void *do_sel4_virtio(void *opaque); + +static void qemu_pci_read(void *out, uint32_t address, int len, unsigned int pcidev); +static void qemu_pci_write(void *out, uint32_t address, int len, unsigned int pcidev); + +static void vmm_doorbell_ring(void); +static void vmm_doorbell_wait(void); + +void sel4_set_irq(unsigned int irq, bool); + +#define UIO_INDEX_DATAPORT 0 +#define UIO_INDEX_EVENT_BAR 1 + +#define DP_CTRL 0 +#define DP_MEM 1 + +typedef struct { + void *data; + void *event; + int fd; + size_t size; + const char *filename; +} dataport_t; + +static dataport_t dataports[] = { + { + .fd = -1, + .size = 0x100000, + .filename = "/dev/uio0", + }, + { + .fd = -1, + .size = 128ULL << 20, + .filename = "/dev/uio1", + }, +}; + +void seL4_Yield(void){} + +#define QEMU +#include "sel4-qemu.h" + +static void qemu_send_msg(const rpcmsg_t *msg); + + +static int dataport_open(dataport_t *dp) +{ + if (dp->fd != -1) { + return 0; + } + + dp->fd = open(dp->filename, O_RDWR); + if (dp->fd < 0) { + return -1; + } + + dp->data = mmap(NULL, dp->size, PROT_READ | PROT_WRITE, MAP_SHARED, dp->fd, UIO_INDEX_DATAPORT * 0x1000); + if (dp->data == (void *) -1) { + return -1; + } + + dp->event = mmap(NULL, 0x1000, PROT_READ | PROT_WRITE, MAP_SHARED, dp->fd, UIO_INDEX_EVENT_BAR * 0x1000); + if (dp->event == (void *) -1) { + return -1; + } + + tii_printf("dataport \"%s\" opened\n", dp->filename); + + return 0; +} + +static void open_dataports(void) +{ + static int already_opened = 0; + + if (already_opened) { + return; + } + already_opened = 1; + + if (dataport_open(&dataports[DP_CTRL])) { + perror("cannot open dataport\n"); + exit(1); + } + if (dataport_open(&dataports[DP_MEM])) { + perror("cannot open dataport\n"); + exit(1); + } + + qemu_mutex_init(&vmm_send_mutex); + + qemu_thread_create(&sel4_virtio_thread, "seL4 virtio", + do_sel4_virtio, NULL, QEMU_THREAD_JOINABLE); +} + +static void sel4_setup_post(MachineState *ms, AccelState *accel) +{ + printf("running sel4_setup_post\n"); +} + +static int dataport_wait(dataport_t *dp) +{ + uint32_t val; + return read(dp->fd, &val, sizeof val); +} + +static int dataport_emit(dataport_t *dp) +{ + ((uint32_t *) dp->event)[0] = 1; + return 0; +} + +static int is_raw_address(uint32_t address) +{ + if (address >= (1ULL << 20)) + return 1; + + return 0; +} + +void qemu_virtio_address_space_read(PCIDevice *pci_dev, hwaddr addr, uint8_t *buf, int len); +void qemu_virtio_address_space_write(PCIDevice *pci_dev, hwaddr addr, uint8_t *buf, int len); + +void qmp_ringbuf_write(const char *, const char *, bool, int, Error **); +char *qmp_ringbuf_read(const char *device, int64_t size, bool has_format, int format, Error **errp); + +void tii_printf(const char *fmt, ...) +{ + char tmp[512]; + va_list ap; + va_start(ap, fmt); + vsprintf(tmp, fmt, ap); + va_end(ap); + Error *err; + qmp_ringbuf_write("debug", tmp, false, 0, &err); +} + +static unsigned int pci_dev_count; +static PCIDevice *pci_devs[16]; +static uintptr_t pci_base[16]; +static unsigned int pci_base_count; + +static void qemu_pci_read(void *out, uint32_t address, int len, unsigned int pcidev) +{ + qemu_mutex_lock_iothread(); + if (is_raw_address(address)) { + address_space_read(&address_space_memory, address, MEMTXATTRS_UNSPECIFIED, out, len); + } else { + qemu_virtio_address_space_read(pci_devs[pcidev], address, out, len); + } + qemu_mutex_unlock_iothread(); +} + +static void qemu_pci_write(void *out, uint32_t address, int len, unsigned int pcidev) +{ + qemu_mutex_lock_iothread(); + if (is_raw_address(address)) { + address_space_write(&address_space_memory, address, MEMTXATTRS_UNSPECIFIED, out, len); + } else { + qemu_virtio_address_space_write(pci_devs[pcidev], address, out, len); + } + qemu_mutex_unlock_iothread(); +} + +static int pci_resolve_irq(PCIDevice *pci_dev, int irq_num) +{ + PCIBus *bus; + for (;;) { + bus = pci_get_bus(pci_dev); + irq_num = bus->map_irq(pci_dev, irq_num); + if (bus->set_irq) + break; + pci_dev = bus->parent_dev; + } + return irq_num; +} + +void sel4_register_pci_device(PCIDevice *d) +{ + pci_devs[pci_dev_count] = d; + printf("Registering PCI device to VMM\n"); + send_qemu_op(QEMU_OP_REGISTER_PCI_DEV | (pci_dev_count << QEMU_PCIDEV_SHIFT), 0); + pci_dev_count++; + + // INTX = 1 -> IRQ 0 + printf("IRQ for this device is %d\n", pci_resolve_irq(d, 0)); +} + +typedef struct { + Int128 size; + hwaddr offset_within_address_space; + EventNotifier *n; +} sel4_listener_region_t; + +#define LR_MAX 512 + +static sel4_listener_region_t lrs[LR_MAX]; +static unsigned int nlrs; + +static int lr_match(sel4_listener_region_t *lr, hwaddr addr, Int128 size) +{ + if (addr >= lr->offset_within_address_space + int128_get64(lr->size)) { + return 0; + } + if (addr + int128_get64(size) < lr->offset_within_address_space) { + return 0; + } + return 1; +} + +static sel4_listener_region_t *lr_find_match(hwaddr addr, Int128 size) +{ + int i; + + for (i = 0; i < nlrs; i++) { + if (lr_match(&lrs[i], addr, size)) { + return &lrs[i]; + } + } + + return NULL; +} + +static void vmm_doorbell_ring(void) +{ + dataport_emit(&dataports[DP_CTRL]); +} + +static void vmm_doorbell_wait(void) +{ + int err = dataport_wait(&dataports[DP_CTRL]); + if (err < 0) { + printf("Error: %s\n", strerror(errno)); + abort(); + } +} + +static void send_qemu_op(uint32_t op, uint32_t param) +{ + rpcmsg_t msg = { + .mr0 = op, + .mr1 = param, + }; + + qemu_send_msg(&msg); +} + +void sel4_set_irq(unsigned int irq, bool state) +{ + send_qemu_op(state ? QEMU_OP_SET_IRQ : QEMU_OP_CLR_IRQ, irq); +} + +static void sel4_change_state_handler(void *opaque, bool running, RunState state) +{ + if (running) { + open_dataports(); + printf("Starting user VM\n"); + send_qemu_op(QEMU_OP_START_VM, 0); + } +} + +static void qemu_send_msg(const rpcmsg_t *msg) +{ + qemu_mutex_lock(&vmm_send_mutex); + rpcmsg_t *reply = rpcmsg_queue_tail(tx_queue); + if (!reply) { + printf("QEMU TX queue full\n"); + abort(); + } + *reply = *msg; + smp_mb(); + rpcmsg_queue_advance_tail(tx_queue); + vmm_doorbell_ring(); + qemu_mutex_unlock(&vmm_send_mutex); +} + +static void qemu_request_read(rpcmsg_t *msg) +{ + rpcmsg_t reply = *msg; + qemu_pci_read(&reply.mr3, reply.mr1, reply.mr2, QEMU_PCIDEV(reply.mr0)); + qemu_send_msg(&reply); +} + +static void qemu_request_write(rpcmsg_t *msg) +{ + rpcmsg_t reply = *msg; + /* Writes are synchronous, hence we need to reply. We can do it before + * doing the actual write, since any new fault cannot be served before + * we finish this function anyway. */ + qemu_send_msg(&reply); + qemu_pci_write(&reply.mr3, reply.mr1, reply.mr2, QEMU_PCIDEV(reply.mr0)); + uintptr_t addr = reply.mr1; + if (!is_raw_address(addr) && pci_base[QEMU_PCIDEV(reply.mr0)]) { + addr += pci_base[QEMU_PCIDEV(reply.mr0)]; + } + sel4_listener_region_t *lr = lr_find_match(addr, reply.mr2); + if (lr) { + event_notifier_set(lr->n); + } +} + +static void qemu_putc_log(rpcmsg_t *msg) +{ + rpcmsg_t reply = *msg; + + for (size_t i = 0; i < logbuffer->sz; i++) { + if (logbuffer->data[i] != 0x0d) { + tii_printf("%c", logbuffer->data[i]); + } + } + + qemu_send_msg(&reply); +} + +static void handle_qemu_request(rpcmsg_t *msg) +{ + switch (QEMU_OP(msg->mr0)) { + case QEMU_OP_READ: + qemu_request_read(msg); + break; + case QEMU_OP_WRITE: + qemu_request_write(msg); + break; + case QEMU_OP_PUTC_LOG: + qemu_putc_log(msg); + break; + default: + printf("Invalid operation %"PRIu32"\n", QEMU_OP(msg->mr0)); + abort(); + } +} + +static void *do_sel4_virtio(void *opaque) +{ + for (;;) { + vmm_doorbell_wait(); + + rpcmsg_t *msg = rpcmsg_queue_head(rx_queue); + if (msg) { + handle_qemu_request(msg); + rpcmsg_queue_advance_head(rx_queue); + } + } + + return NULL; +} + +static bool ioeventfd_exists(MemoryRegionSection *section, EventNotifier *e) +{ + for (int i = 0; i < nlrs; i++) { + if (lrs[i].offset_within_address_space == section->offset_within_address_space && + lrs[i].size == section->size && + lrs[i].n == e) { + return true; + } + } + return false; +} + +static void sel4_io_ioeventfd_add(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, + EventNotifier *e) +{ + if (ioeventfd_exists(section, e)) { + return; + } + + sel4_listener_region_t *lr = &lrs[nlrs++]; + + tii_printf("%s: mr=%p offset=0x%"PRIx64" size=0x%"PRIx64" notifier=%p\n", __func__, section->mr, + section->offset_within_address_space, int128_get64(section->size), e); + + lr->offset_within_address_space = section->offset_within_address_space; + lr->size = section->size; + lr->n = e; +} + +static void sel4_io_ioeventfd_del(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, + EventNotifier *e) + +{ + tii_printf("WARNING: %s UNIMPLEMENTED\n", __func__); +} + +static void sel4_region_add(MemoryListener *listener, MemoryRegionSection *section) +{ + tii_printf("%s entered, region name %s, offset within region 0x%lx, size 0x%lx\n", __func__, + memory_region_name(section->mr), (uint64_t) section->offset_within_address_space, + (uint64_t) section->size); + + if (!strcmp(memory_region_name(section->mr), "virtio-pci")) { + pci_base[pci_base_count] = section->offset_within_address_space; + tii_printf("translating accesses to PCI device %d to offset %"PRIxPTR"\n", pci_base_count, + pci_base[pci_base_count]); + pci_base_count++; + } +} + +static void sel4_region_del(MemoryListener *listener, MemoryRegionSection *section) +{ + tii_printf("WARNING: %s entered, but no real implementation\n", __func__); +} + +static MemoryListener sel4_io_listener = { + .eventfd_add = sel4_io_ioeventfd_add, + .eventfd_del = sel4_io_ioeventfd_del, + .region_add = sel4_region_add, + .region_del = sel4_region_del, +}; + +static int sel4_init(MachineState *ms) +{ + open_dataports(); + + MachineClass *mc = MACHINE_GET_CLASS(ms); + + memory_listener_register(&sel4_io_listener, &address_space_memory); + + qemu_add_vm_change_state_handler(sel4_change_state_handler, NULL); + + return 0; +} + +static void sel4_accel_class_init(ObjectClass *oc, void *data) +{ + AccelClass *ac = ACCEL_CLASS(oc); + + ac->name = "seL4"; + ac->init_machine = sel4_init; + ac->setup_post = sel4_setup_post; + ac->allowed = &sel4_allowed; +} + +#define TYPE_SEL4_ACCEL ACCEL_CLASS_NAME("sel4") + +static const TypeInfo sel4_accel_type = { + .name = TYPE_SEL4_ACCEL, + .parent = TYPE_ACCEL, + .class_init = sel4_accel_class_init, +}; + +static void sel4_accel_ops_class_init(ObjectClass *oc, void *data) +{ + AccelOpsClass *ops = ACCEL_OPS_CLASS(oc); + + ops->create_vcpu_thread = dummy_start_vcpu_thread; +} + +static const TypeInfo sel4_accel_ops_type = { + .name = ACCEL_OPS_NAME("sel4"), + + .parent = TYPE_ACCEL_OPS, + .class_init = sel4_accel_ops_class_init, + .abstract = true, +}; + +static void sel4_type_init(void) +{ + type_register_static(&sel4_accel_type); + type_register_static(&sel4_accel_ops_type); +} +type_init(sel4_type_init); diff --git a/hw/arm/virt.c b/hw/arm/virt.c index 9633f822f36..9de2fb7c1ea 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -158,11 +158,11 @@ static const MemMapEntry base_memmap[] = { /* ...repeating for a total of NUM_VIRTIO_TRANSPORTS, each of that size */ [VIRT_PLATFORM_BUS] = { 0x0c000000, 0x02000000 }, [VIRT_SECURE_MEM] = { 0x0e000000, 0x01000000 }, - [VIRT_PCIE_MMIO] = { 0x10000000, 0x2eff0000 }, + [VIRT_PCIE_MMIO] = { 0x50000000, 0x6eff0000 }, [VIRT_PCIE_PIO] = { 0x3eff0000, 0x00010000 }, [VIRT_PCIE_ECAM] = { 0x3f000000, 0x01000000 }, /* Actual RAM size depends on initial RAM and device memory settings */ - [VIRT_MEM] = { GiB, LEGACY_RAMLIMIT_BYTES }, + [VIRT_MEM] = { 0x48000000, 0x08000000 }, }; /* @@ -1438,7 +1438,7 @@ static void create_pcie(VirtMachineState *vms) mmio_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 1); memory_region_init_alias(mmio_alias, OBJECT(dev), "pcie-mmio", mmio_reg, base_mmio, size_mmio); - memory_region_add_subregion(get_system_memory(), base_mmio, mmio_alias); + memory_region_add_subregion_overlap(get_system_memory(), base_mmio, mmio_alias, 1); if (vms->highmem_mmio) { /* Map high MMIO space */ diff --git a/hw/pci-host/gpex.c b/hw/pci-host/gpex.c index a6752fac5e8..cea57ddb51d 100644 --- a/hw/pci-host/gpex.c +++ b/hw/pci-host/gpex.c @@ -41,10 +41,14 @@ * GPEX host */ +void sel4_set_irq(unsigned int irq, bool); + static void gpex_set_irq(void *opaque, int irq_num, int level) { GPEXHost *s = opaque; + sel4_set_irq(irq_num, !!level); + qemu_set_irq(s->irq[irq_num], level); } diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c index a50c5a57d7e..6932c7a072b 100644 --- a/hw/virtio/virtio-pci.c +++ b/hw/virtio/virtio-pci.c @@ -676,6 +676,19 @@ static uint32_t virtio_read_config(PCIDevice *pci_dev, return pci_default_read_config(pci_dev, address, len); } +void qemu_virtio_address_space_read(PCIDevice *pci_dev, hwaddr addr, uint8_t *buf, int len) +{ + uint32_t data; + + data = virtio_read_config(pci_dev, addr, len); + memcpy(buf, &data, len); +} + +void qemu_virtio_address_space_write(PCIDevice *pci_dev, hwaddr addr, uint8_t *buf, int len) +{ + virtio_write_config(pci_dev, addr, *(uint32_t *)buf, len); +} + static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy, unsigned int queue_no, unsigned int vector) @@ -1621,6 +1634,8 @@ static void virtio_pci_pre_plugged(DeviceState *d, Error **errp) virtio_add_feature(&vdev->host_features, VIRTIO_F_BAD_FEATURE); } +void sel4_register_pci_device(PCIDevice *d); + /* This is called by virtio-bus just after the device is plugged. */ static void virtio_pci_device_plugged(DeviceState *d, Error **errp) { @@ -1762,6 +1777,8 @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp) proxy->pci_dev.config_write = virtio_write_config; proxy->pci_dev.config_read = virtio_read_config; + sel4_register_pci_device(&proxy->pci_dev); + if (legacy) { size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev) + virtio_bus_get_vdev_config_len(bus); From 1e19c7d52e9f0882f72e5bbfbdd94a761a2a91b4 Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Mon, 28 Mar 2022 12:42:05 +0300 Subject: [PATCH 03/28] TEMPORARY: copy common header from VMM Signed-off-by: Hannu Lyytinen --- accel/sel4/sel4-qemu.h | 138 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 accel/sel4/sel4-qemu.h diff --git a/accel/sel4/sel4-qemu.h b/accel/sel4/sel4-qemu.h new file mode 100644 index 00000000000..5e74b1bb98d --- /dev/null +++ b/accel/sel4/sel4-qemu.h @@ -0,0 +1,138 @@ +#pragma once + +#ifdef QEMU +typedef unsigned long seL4_Word; +#define ORIGIN "" +#define debug_printf(fmt, ...) qemu_printf(fmt "\n", ## __VA_ARGS__) +#else +#define ORIGIN "VMM " +#define debug_printf(fmt, ...) printf(fmt "\n", ## __VA_ARGS__) +#endif + +typedef struct { + size_t sz; + char data[1024]; +} logbuffer_t; + + +#ifndef QEMU +/* to QEMU */ +#define tx_queue (((rpcmsg_queue_t *) ctrl) + 0) + +/* from QEMU */ +#define rx_queue (((rpcmsg_queue_t *) ctrl) + 1) + +#define logbuffer ((logbuffer_t *)(rx_queue + 1)) +#else +/* from VMM */ +#define rx_queue (((rpcmsg_queue_t *) dataports[DP_CTRL].data) + 0) + +/* to VMM */ +#define tx_queue (((rpcmsg_queue_t *) dataports[DP_CTRL].data) + 1) + +#define logbuffer ((logbuffer_t *)(tx_queue + 1)) +#endif + +/* from VMM to QEMU */ +#define QEMU_OP_READ 0 +#define QEMU_OP_WRITE 1 +#define QEMU_OP_PUTC_LOG 2 + +/* from QEMU to VMM */ +#define QEMU_OP_SET_IRQ 16 +#define QEMU_OP_CLR_IRQ 17 +#define QEMU_OP_START_VM 18 +#define QEMU_OP_REGISTER_PCI_DEV 19 + +#define QEMU_OP_MASK 0xffULL +#define QEMU_OP_SHIFT 0 +#define QEMU_OP(__x__) (((__x__) & QEMU_OP_MASK) >> QEMU_OP_SHIFT) + +#define QEMU_PCIDEV_MASK 0xff00ULL +#define QEMU_PCIDEV_SHIFT 8 +#define QEMU_PCIDEV(__x__) (((__x__) & QEMU_PCIDEV_MASK) >> QEMU_PCIDEV_SHIFT) + +#define QEMU_ID_MASK 0xffff0000ULL +#define QEMU_ID_SHIFT 16 +#define QEMU_ID(__x__) (((__x__) & QEMU_ID_MASK) >> QEMU_ID_SHIFT) + +#define RPCMSG_BUFFER_SIZE 32 + +typedef struct { + seL4_Word mr0; + seL4_Word mr1; + seL4_Word mr2; + seL4_Word mr3; +} rpcmsg_t; + +typedef struct { + uint32_t head; + uint32_t tail; + uint32_t rsvd[2]; + rpcmsg_t data[RPCMSG_BUFFER_SIZE]; +} rpcmsg_queue_t; + +#define QUEUE_PREV(_i) ((_i) ? ((_i) - 1) : (RPCMSG_BUFFER_SIZE - 1)) +#define QUEUE_NEXT(_i) (((_i) + 1) & (RPCMSG_BUFFER_SIZE - 1)) + +static void rpcmsg_queue_init(rpcmsg_queue_t *q) +{ + memset(q, 0, sizeof(*q)); +} + +static void rpcmsg_queue_dump(const char *name, rpcmsg_queue_t *q, unsigned int idx) +{ + unsigned int start_idx = idx; + char tmp[128]; + + sprintf(tmp, ORIGIN "name = %s head = %02d tail = %02d", name, q->head, q->tail); + debug_printf("%s", tmp); + + do { + rpcmsg_t *msg = &q->data[idx]; + sprintf(tmp, ORIGIN "%02d: %08"PRIx64" %08"PRIx64" %08"PRIx64" %08"PRIx64, + idx, msg->mr0, msg->mr1, msg->mr2, msg->mr3); + debug_printf("%s", tmp); + idx = QUEUE_PREV(idx); + } while (idx != start_idx); +} + +static inline bool rpcmsg_queue_full(rpcmsg_queue_t *q) +{ + return QUEUE_NEXT(q->tail) == q->head; +} + +static inline bool rpcmsg_queue_empty(rpcmsg_queue_t *q) +{ + return q->tail == q->head; +} + +static inline rpcmsg_t *rpcmsg_queue_head(rpcmsg_queue_t *q) +{ + return rpcmsg_queue_empty(q) ? NULL : (q->data + q->head); +} + +static inline rpcmsg_t *rpcmsg_queue_tail(rpcmsg_queue_t *q) +{ + + return rpcmsg_queue_full(q) ? NULL : (q->data + q->tail); +} + +static inline void rpcmsg_queue_advance_head(rpcmsg_queue_t *q) +{ + assert(!rpcmsg_queue_empty(q)); + q->head = QUEUE_NEXT(q->head); +} + +static inline void rpcmsg_queue_advance_tail(rpcmsg_queue_t *q) +{ + assert(!rpcmsg_queue_full(q)); + q->tail = QUEUE_NEXT(q->tail); +} + +static inline void rpcmsg_queue_enqueue(rpcmsg_queue_t *q, rpcmsg_t *msg) +{ + assert(!rpcmsg_queue_full(q)); + memcpy(q->data + q->tail, msg, sizeof(*msg)); + q->tail = QUEUE_NEXT(q->tail); +} From 6895de01407a07f02ed7b3bef993cb462c0cc542 Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Thu, 31 Mar 2022 21:24:47 +0300 Subject: [PATCH 04/28] Generalize PCI device support Signed-off-by: Hannu Lyytinen --- accel/sel4/sel4-all.c | 19 ++++++++++++------- hw/misc/ivshmem.c | 4 ++++ hw/virtio/virtio-pci.c | 13 ------------- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c index 8b82b580607..e3c4af3776d 100644 --- a/accel/sel4/sel4-all.c +++ b/accel/sel4/sel4-all.c @@ -140,9 +140,6 @@ static int is_raw_address(uint32_t address) return 0; } -void qemu_virtio_address_space_read(PCIDevice *pci_dev, hwaddr addr, uint8_t *buf, int len); -void qemu_virtio_address_space_write(PCIDevice *pci_dev, hwaddr addr, uint8_t *buf, int len); - void qmp_ringbuf_write(const char *, const char *, bool, int, Error **); char *qmp_ringbuf_read(const char *device, int64_t size, bool has_format, int format, Error **errp); @@ -165,10 +162,13 @@ static unsigned int pci_base_count; static void qemu_pci_read(void *out, uint32_t address, int len, unsigned int pcidev) { qemu_mutex_lock_iothread(); - if (is_raw_address(address)) { + if (pcidev > 16) { address_space_read(&address_space_memory, address, MEMTXATTRS_UNSPECIFIED, out, len); } else { - qemu_virtio_address_space_read(pci_devs[pcidev], address, out, len); + PCIDevice *dev = pci_devs[pcidev]; + + uint32_t val = dev->config_read(dev, address, len); + memcpy(out, &val, len); } qemu_mutex_unlock_iothread(); } @@ -176,10 +176,15 @@ static void qemu_pci_read(void *out, uint32_t address, int len, unsigned int pci static void qemu_pci_write(void *out, uint32_t address, int len, unsigned int pcidev) { qemu_mutex_lock_iothread(); - if (is_raw_address(address)) { + if (pcidev > 16) { address_space_write(&address_space_memory, address, MEMTXATTRS_UNSPECIFIED, out, len); } else { - qemu_virtio_address_space_write(pci_devs[pcidev], address, out, len); + PCIDevice *dev = pci_devs[pcidev]; + + uint32_t val = 0; + memcpy(&val, out, len); + + dev->config_write(dev, address, val, len); } qemu_mutex_unlock_iothread(); } diff --git a/hw/misc/ivshmem.c b/hw/misc/ivshmem.c index 8270db53cda..a7a547daad8 100644 --- a/hw/misc/ivshmem.c +++ b/hw/misc/ivshmem.c @@ -832,6 +832,8 @@ static void ivshmem_write_config(PCIDevice *pdev, uint32_t address, } } +void sel4_register_pci_device(PCIDevice *d); + static void ivshmem_common_realize(PCIDevice *dev, Error **errp) { IVShmemState *s = IVSHMEM_COMMON(dev); @@ -915,6 +917,8 @@ static void ivshmem_common_realize(PCIDevice *dev, Error **errp) PCI_BASE_ADDRESS_MEM_PREFETCH | PCI_BASE_ADDRESS_MEM_TYPE_64, s->ivshmem_bar2); + + sel4_register_pci_device(PCI_DEVICE(s)); } static void ivshmem_exit(PCIDevice *dev) diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c index 6932c7a072b..fa998857268 100644 --- a/hw/virtio/virtio-pci.c +++ b/hw/virtio/virtio-pci.c @@ -676,19 +676,6 @@ static uint32_t virtio_read_config(PCIDevice *pci_dev, return pci_default_read_config(pci_dev, address, len); } -void qemu_virtio_address_space_read(PCIDevice *pci_dev, hwaddr addr, uint8_t *buf, int len) -{ - uint32_t data; - - data = virtio_read_config(pci_dev, addr, len); - memcpy(buf, &data, len); -} - -void qemu_virtio_address_space_write(PCIDevice *pci_dev, hwaddr addr, uint8_t *buf, int len) -{ - virtio_write_config(pci_dev, addr, *(uint32_t *)buf, len); -} - static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy, unsigned int queue_no, unsigned int vector) From f0aefe69610532ff2536e2e756afacd271db739a Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Fri, 1 Apr 2022 11:09:22 +0300 Subject: [PATCH 05/28] Support offset when mapping RAM from file Signed-off-by: Hannu Lyytinen --- backends/hostmem-file.c | 38 +++++++++++++++++++++++++++++++++++++- include/exec/memory.h | 2 ++ include/exec/ram_addr.h | 2 +- qapi/qom.json | 1 + qemu-options.hx | 5 ++++- softmmu/memory.c | 3 ++- softmmu/physmem.c | 5 +++-- 7 files changed, 50 insertions(+), 6 deletions(-) diff --git a/backends/hostmem-file.c b/backends/hostmem-file.c index 25141283c4a..1b1ac272e34 100644 --- a/backends/hostmem-file.c +++ b/backends/hostmem-file.c @@ -29,6 +29,7 @@ struct HostMemoryBackendFile { uint64_t align; bool discard_data; bool is_pmem; + off_t offset; bool readonly; }; @@ -58,7 +59,8 @@ file_backend_memory_alloc(HostMemoryBackend *backend, Error **errp) ram_flags |= fb->is_pmem ? RAM_PMEM : 0; memory_region_init_ram_from_file(&backend->mr, OBJECT(backend), name, backend->size, fb->align, ram_flags, - fb->mem_path, fb->readonly, errp); + fb->mem_path, fb->offset, fb->readonly, + errp); g_free(name); #endif } @@ -125,6 +127,36 @@ static void file_memory_backend_set_align(Object *o, Visitor *v, fb->align = val; } +static void file_memory_backend_get_offset(Object *o, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + HostMemoryBackendFile *fb = MEMORY_BACKEND_FILE(o); + uint64_t val = fb->offset; + + visit_type_size(v, name, &val, errp); +} + +static void file_memory_backend_set_offset(Object *o, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + HostMemoryBackend *backend = MEMORY_BACKEND(o); + HostMemoryBackendFile *fb = MEMORY_BACKEND_FILE(o); + uint64_t val; + + if (host_memory_backend_mr_inited(backend)) { + error_setg(errp, "cannot change property '%s' of %s", name, + object_get_typename(o)); + return; + } + + if (!visit_type_size(v, name, &val, errp)) { + return; + } + fb->offset = val; +} + #ifdef CONFIG_LIBPMEM static bool file_memory_backend_get_pmem(Object *o, Error **errp) { @@ -201,6 +233,10 @@ file_backend_class_init(ObjectClass *oc, void *data) object_class_property_add_bool(oc, "pmem", file_memory_backend_get_pmem, file_memory_backend_set_pmem); #endif + object_class_property_add(oc, "offset", "int", + file_memory_backend_get_offset, + file_memory_backend_set_offset, + NULL, NULL); object_class_property_add_bool(oc, "readonly", file_memory_backend_get_readonly, file_memory_backend_set_readonly); diff --git a/include/exec/memory.h b/include/exec/memory.h index bfb1de8eeae..16ee9723393 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -1288,6 +1288,7 @@ void memory_region_init_resizeable_ram(MemoryRegion *mr, * @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_PMEM, * RAM_NORESERVE, * @path: the path in which to allocate the RAM. + * @off: offset within the file referenced by path * @readonly: true to open @path for reading, false for read/write. * @errp: pointer to Error*, to store an error if it happens. * @@ -1301,6 +1302,7 @@ void memory_region_init_ram_from_file(MemoryRegion *mr, uint64_t align, uint32_t ram_flags, const char *path, + off_t off, bool readonly, Error **errp); diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index f3e0c78161d..0fe6c3bbf26 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -118,7 +118,7 @@ long qemu_maxrampagesize(void); */ RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, uint32_t ram_flags, const char *mem_path, - bool readonly, Error **errp); + off_t offset, bool readonly, Error **errp); RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, uint32_t ram_flags, int fd, off_t offset, bool readonly, Error **errp); diff --git a/qapi/qom.json b/qapi/qom.json index 80dd419b392..d742cc77b22 100644 --- a/qapi/qom.json +++ b/qapi/qom.json @@ -648,6 +648,7 @@ '*discard-data': 'bool', 'mem-path': 'str', '*pmem': { 'type': 'bool', 'if': 'CONFIG_LIBPMEM' }, + '*offset': 'size', '*readonly': 'bool' } } ## diff --git a/qemu-options.hx b/qemu-options.hx index 93370187630..b6188e4b699 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -4814,7 +4814,7 @@ SRST they are specified. Note that the 'id' property must be set. These objects are placed in the '/objects' path. - ``-object memory-backend-file,id=id,size=size,mem-path=dir,share=on|off,discard-data=on|off,merge=on|off,dump=on|off,prealloc=on|off,host-nodes=host-nodes,policy=default|preferred|bind|interleave,align=align,readonly=on|off`` + ``-object memory-backend-file,id=id,size=size,mem-path=dir,share=on|off,discard-data=on|off,merge=on|off,dump=on|off,prealloc=on|off,host-nodes=host-nodes,policy=default|preferred|bind|interleave,align=align,offset=offset,readonly=on|off`` Creates a memory file backend object, which can be used to back the guest RAM with huge pages. @@ -4884,6 +4884,9 @@ SRST such cases, users can specify the required alignment via this option. + The ``offset`` option specifies the offset when QEMU mmap(2) + ``mem-path``. + The ``pmem`` option specifies whether the backing file specified by ``mem-path`` is in host persistent memory that can be accessed using the SNIA NVM programming model (e.g. Intel diff --git a/softmmu/memory.c b/softmmu/memory.c index 7ba2048836b..a5e49e5740a 100644 --- a/softmmu/memory.c +++ b/softmmu/memory.c @@ -1600,6 +1600,7 @@ void memory_region_init_ram_from_file(MemoryRegion *mr, uint64_t align, uint32_t ram_flags, const char *path, + off_t offset, bool readonly, Error **errp) { @@ -1611,7 +1612,7 @@ void memory_region_init_ram_from_file(MemoryRegion *mr, mr->destructor = memory_region_destructor_ram; mr->align = align; mr->ram_block = qemu_ram_alloc_from_file(size, mr, ram_flags, path, - readonly, &err); + offset, readonly, &err); if (err) { mr->size = int128_zero(); object_unparent(OBJECT(mr)); diff --git a/softmmu/physmem.c b/softmmu/physmem.c index dc3c3e5f2e7..4170cd3f5ca 100644 --- a/softmmu/physmem.c +++ b/softmmu/physmem.c @@ -2125,7 +2125,7 @@ RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, uint32_t ram_flags, const char *mem_path, - bool readonly, Error **errp) + off_t offset, bool readonly, Error **errp) { int fd; bool created; @@ -2137,7 +2137,8 @@ RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, return NULL; } - block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, 0, readonly, errp); + block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, + offset, readonly, errp); if (!block) { if (created) { unlink(mem_path); From 3eb693e8f187501206a7ddc8012e137075e595df Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Fri, 1 Apr 2022 19:27:10 +0300 Subject: [PATCH 06/28] Select UIO page in QEMU Previously the seL4 cross-connector had the BARs swapped so that the dataport would be the first mapping (at offset 0), but turned out patching QEMU to have the offset option is not a big job. This saves us from pushing an API-breaking change to seL4 source trees. Signed-off-by: Hannu Lyytinen --- accel/sel4/sel4-all.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c index e3c4af3776d..cf76e279409 100644 --- a/accel/sel4/sel4-all.c +++ b/accel/sel4/sel4-all.c @@ -30,8 +30,8 @@ static void vmm_doorbell_wait(void); void sel4_set_irq(unsigned int irq, bool); -#define UIO_INDEX_DATAPORT 0 -#define UIO_INDEX_EVENT_BAR 1 +#define UIO_INDEX_DATAPORT 1 +#define UIO_INDEX_EVENT_BAR 0 #define DP_CTRL 0 #define DP_MEM 1 From 5cce1314291d65716c3813edf87287d1a7cdb043 Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Sun, 3 Apr 2022 13:57:25 +0300 Subject: [PATCH 07/28] Process multiple async requests at once Signed-off-by: Hannu Lyytinen --- accel/sel4/sel4-all.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c index cf76e279409..ad30fbc169f 100644 --- a/accel/sel4/sel4-all.c +++ b/accel/sel4/sel4-all.c @@ -360,13 +360,14 @@ static void handle_qemu_request(rpcmsg_t *msg) static void *do_sel4_virtio(void *opaque) { for (;;) { - vmm_doorbell_wait(); - rpcmsg_t *msg = rpcmsg_queue_head(rx_queue); if (msg) { handle_qemu_request(msg); rpcmsg_queue_advance_head(rx_queue); + continue; } + + vmm_doorbell_wait(); } return NULL; From 5057eb524e7aed26f7da92a759ffde837855e5c3 Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Tue, 5 Apr 2022 16:20:46 +0300 Subject: [PATCH 08/28] Change PCIe address Signed-off-by: Hannu Lyytinen --- hw/arm/virt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hw/arm/virt.c b/hw/arm/virt.c index 9de2fb7c1ea..06fa2fe0a29 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -158,7 +158,7 @@ static const MemMapEntry base_memmap[] = { /* ...repeating for a total of NUM_VIRTIO_TRANSPORTS, each of that size */ [VIRT_PLATFORM_BUS] = { 0x0c000000, 0x02000000 }, [VIRT_SECURE_MEM] = { 0x0e000000, 0x01000000 }, - [VIRT_PCIE_MMIO] = { 0x50000000, 0x6eff0000 }, + [VIRT_PCIE_MMIO] = { 0x60000000, 0x7eff0000 }, [VIRT_PCIE_PIO] = { 0x3eff0000, 0x00010000 }, [VIRT_PCIE_ECAM] = { 0x3f000000, 0x01000000 }, /* Actual RAM size depends on initial RAM and device memory settings */ From c5c66f5a309d97cf43b790f6aaa3f6e24c489c69 Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Fri, 15 Apr 2022 21:40:35 +0300 Subject: [PATCH 09/28] Update header Signed-off-by: Hannu Lyytinen --- accel/sel4/sel4-qemu.h | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/accel/sel4/sel4-qemu.h b/accel/sel4/sel4-qemu.h index 5e74b1bb98d..8898ba59841 100644 --- a/accel/sel4/sel4-qemu.h +++ b/accel/sel4/sel4-qemu.h @@ -46,15 +46,21 @@ typedef struct { #define QEMU_OP_MASK 0xffULL #define QEMU_OP_SHIFT 0 -#define QEMU_OP(__x__) (((__x__) & QEMU_OP_MASK) >> QEMU_OP_SHIFT) +#define QEMU_OP(__x__) ((unsigned int)(((__x__) & QEMU_OP_MASK) >> QEMU_OP_SHIFT)) #define QEMU_PCIDEV_MASK 0xff00ULL #define QEMU_PCIDEV_SHIFT 8 -#define QEMU_PCIDEV(__x__) (((__x__) & QEMU_PCIDEV_MASK) >> QEMU_PCIDEV_SHIFT) +#define QEMU_PCIDEV(__x__) ((unsigned int)(((__x__) & QEMU_PCIDEV_MASK) >> QEMU_PCIDEV_SHIFT)) -#define QEMU_ID_MASK 0xffff0000ULL +#define QEMU_ID_MASK 0xff0000ULL #define QEMU_ID_SHIFT 16 -#define QEMU_ID(__x__) (((__x__) & QEMU_ID_MASK) >> QEMU_ID_SHIFT) +#define QEMU_ID(__x__) ((unsigned int)(((__x__) & QEMU_ID_MASK) >> QEMU_ID_SHIFT)) +#define QEMU_ID_FROM(__x__) (((__x__) << QEMU_ID_SHIFT) & QEMU_ID_MASK) + +#define QEMU_VCPU_MASK 0xff000000ULL +#define QEMU_VCPU_SHIFT 24 +#define QEMU_VCPU(__x__) ((unsigned int)(((__x__) & QEMU_VCPU_MASK) >> QEMU_VCPU_SHIFT)) +#define QEMU_VCPU_NONE 0xff #define RPCMSG_BUFFER_SIZE 32 From d7a4c05733f50ffd4a8f56200b8a72fb287cd03d Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Thu, 28 Apr 2022 00:43:25 +0300 Subject: [PATCH 10/28] Specify licenses Signed-off-by: Hannu Lyytinen --- accel/sel4/sel4-all.c | 6 ++++++ accel/sel4/sel4-qemu.h | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c index ad30fbc169f..0774d394c9a 100644 --- a/accel/sel4/sel4-all.c +++ b/accel/sel4/sel4-all.c @@ -1,3 +1,9 @@ +/* + * Copyright 2022, Technology Innovation Institute + * + * SPDX-License-Identifier: Apache-2.0 + */ + #include "qemu/osdep.h" #include "qemu/error-report.h" #include "qemu/module.h" diff --git a/accel/sel4/sel4-qemu.h b/accel/sel4/sel4-qemu.h index 8898ba59841..931582b3c7d 100644 --- a/accel/sel4/sel4-qemu.h +++ b/accel/sel4/sel4-qemu.h @@ -1,3 +1,9 @@ +/* + * Copyright 2022, Technology Innovation Institute + * + * SPDX-License-Identifier: Apache-2.0 + */ + #pragma once #ifdef QEMU From 036667a0eddcfee360f731f5af5569379d371cf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markku=20Ahvenj=C3=A4rvi?= Date: Tue, 6 Dec 2022 09:04:01 +0200 Subject: [PATCH 11/28] sel4: use sel4-virt driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Markku Ahvenjärvi --- accel/sel4/sel4-all.c | 430 +++++++++++++++++++++--------------------- 1 file changed, 217 insertions(+), 213 deletions(-) diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c index 0774d394c9a..bc5c5094641 100644 --- a/accel/sel4/sel4-all.c +++ b/accel/sel4/sel4-all.c @@ -9,133 +9,89 @@ #include "qemu/module.h" #include "qapi/error.h" #include "qemu/accel.h" +#include "qemu/atomic.h" #include "sysemu/cpus.h" +#include "sysemu/runstate.h" #include "hw/boards.h" #include "hw/pci/pci.h" #include "hw/pci/pci_bus.h" +#include "migration/vmstate.h" #include +#include +#include void tii_printf(const char *fmt, ...); +static MemoryRegion ram_mr; bool sel4_allowed; void sel4_register_pci_device(PCIDevice *d); -static void send_qemu_op(uint32_t op, uint32_t param); static QemuThread sel4_virtio_thread; -static QemuMutex vmm_send_mutex; static void *do_sel4_virtio(void *opaque); -static void qemu_pci_read(void *out, uint32_t address, int len, unsigned int pcidev); -static void qemu_pci_write(void *out, uint32_t address, int len, unsigned int pcidev); - -static void vmm_doorbell_ring(void); -static void vmm_doorbell_wait(void); - +/* FIXME: we might need to create interrupt controller for seL4 for + * qemu_set_irq to call. */ void sel4_set_irq(unsigned int irq, bool); -#define UIO_INDEX_DATAPORT 1 -#define UIO_INDEX_EVENT_BAR 0 - -#define DP_CTRL 0 -#define DP_MEM 1 +typedef struct SeL4State +{ + AccelState parent_obj; -typedef struct { - void *data; - void *event; int fd; - size_t size; - const char *filename; -} dataport_t; - -static dataport_t dataports[] = { - { - .fd = -1, - .size = 0x100000, - .filename = "/dev/uio0", - }, - { - .fd = -1, - .size = 128ULL << 20, - .filename = "/dev/uio1", - }, -}; - -void seL4_Yield(void){} - -#define QEMU -#include "sel4-qemu.h" + int vmfd; + int ioreqfd; + struct sel4_iohandler_buffer *ioreq_buffer; +} SeL4State; -static void qemu_send_msg(const rpcmsg_t *msg); +#define TYPE_SEL4_ACCEL ACCEL_CLASS_NAME("sel4") +DECLARE_INSTANCE_CHECKER(SeL4State, SEL4_STATE, TYPE_SEL4_ACCEL) -static int dataport_open(dataport_t *dp) +static int sel4_ioctl(SeL4State *s, int type, ...) { - if (dp->fd != -1) { - return 0; - } - - dp->fd = open(dp->filename, O_RDWR); - if (dp->fd < 0) { - return -1; - } + int ret; + void *arg; + va_list ap; - dp->data = mmap(NULL, dp->size, PROT_READ | PROT_WRITE, MAP_SHARED, dp->fd, UIO_INDEX_DATAPORT * 0x1000); - if (dp->data == (void *) -1) { - return -1; - } + va_start(ap, type); + arg = va_arg(ap, void *); + va_end(ap); - dp->event = mmap(NULL, 0x1000, PROT_READ | PROT_WRITE, MAP_SHARED, dp->fd, UIO_INDEX_EVENT_BAR * 0x1000); - if (dp->event == (void *) -1) { - return -1; + ret = ioctl(s->fd, type, arg); + if (ret == -1) { + ret = -errno; } - - tii_printf("dataport \"%s\" opened\n", dp->filename); - - return 0; + return ret; } -static void open_dataports(void) +static int sel4_vm_ioctl(SeL4State *s, int type, ...) { - static int already_opened = 0; + int ret; + void *arg; + va_list ap; - if (already_opened) { - return; - } - already_opened = 1; + va_start(ap, type); + arg = va_arg(ap, void *); + va_end(ap); - if (dataport_open(&dataports[DP_CTRL])) { - perror("cannot open dataport\n"); - exit(1); + ret = ioctl(s->vmfd, type, arg); + if (ret == -1) { + ret = -errno; } - if (dataport_open(&dataports[DP_MEM])) { - perror("cannot open dataport\n"); - exit(1); - } - - qemu_mutex_init(&vmm_send_mutex); - - qemu_thread_create(&sel4_virtio_thread, "seL4 virtio", - do_sel4_virtio, NULL, QEMU_THREAD_JOINABLE); + return ret; } +void seL4_Yield(void){} + static void sel4_setup_post(MachineState *ms, AccelState *accel) { - printf("running sel4_setup_post\n"); -} + SeL4State *s = SEL4_STATE(ms->accelerator); -static int dataport_wait(dataport_t *dp) -{ - uint32_t val; - return read(dp->fd, &val, sizeof val); -} - -static int dataport_emit(dataport_t *dp) -{ - ((uint32_t *) dp->event)[0] = 1; - return 0; + qemu_thread_create(&sel4_virtio_thread, "seL4 virtio", + do_sel4_virtio, s, QEMU_THREAD_JOINABLE); } static int is_raw_address(uint32_t address) @@ -165,36 +121,6 @@ static PCIDevice *pci_devs[16]; static uintptr_t pci_base[16]; static unsigned int pci_base_count; -static void qemu_pci_read(void *out, uint32_t address, int len, unsigned int pcidev) -{ - qemu_mutex_lock_iothread(); - if (pcidev > 16) { - address_space_read(&address_space_memory, address, MEMTXATTRS_UNSPECIFIED, out, len); - } else { - PCIDevice *dev = pci_devs[pcidev]; - - uint32_t val = dev->config_read(dev, address, len); - memcpy(out, &val, len); - } - qemu_mutex_unlock_iothread(); -} - -static void qemu_pci_write(void *out, uint32_t address, int len, unsigned int pcidev) -{ - qemu_mutex_lock_iothread(); - if (pcidev > 16) { - address_space_write(&address_space_memory, address, MEMTXATTRS_UNSPECIFIED, out, len); - } else { - PCIDevice *dev = pci_devs[pcidev]; - - uint32_t val = 0; - memcpy(&val, out, len); - - dev->config_write(dev, address, val, len); - } - qemu_mutex_unlock_iothread(); -} - static int pci_resolve_irq(PCIDevice *pci_dev, int irq_num) { PCIBus *bus; @@ -210,9 +136,18 @@ static int pci_resolve_irq(PCIDevice *pci_dev, int irq_num) void sel4_register_pci_device(PCIDevice *d) { + SeL4State *s = SEL4_STATE(current_accel()); + struct sel4_vpci_device vpcidev = { + .pcidev = pci_dev_count, + }; + pci_devs[pci_dev_count] = d; printf("Registering PCI device to VMM\n"); - send_qemu_op(QEMU_OP_REGISTER_PCI_DEV | (pci_dev_count << QEMU_PCIDEV_SHIFT), 0); + + if (sel4_vm_ioctl(s, SEL4_CREATE_VPCI_DEVICE, &vpcidev)) { + fprintf(stderr, "Failed to register PCI device: %m\n"); + } + pci_dev_count++; // INTX = 1 -> IRQ 0 @@ -254,126 +189,117 @@ static sel4_listener_region_t *lr_find_match(hwaddr addr, Int128 size) return NULL; } -static void vmm_doorbell_ring(void) -{ - dataport_emit(&dataports[DP_CTRL]); -} - -static void vmm_doorbell_wait(void) -{ - int err = dataport_wait(&dataports[DP_CTRL]); - if (err < 0) { - printf("Error: %s\n", strerror(errno)); - abort(); - } -} - -static void send_qemu_op(uint32_t op, uint32_t param) +void sel4_set_irq(unsigned int irq, bool state) { - rpcmsg_t msg = { - .mr0 = op, - .mr1 = param, + SeL4State *s = SEL4_STATE(current_accel()); + struct sel4_irqline irqline = { + .irq = irq, + .op = state ? SEL4_IRQ_OP_SET : SEL4_IRQ_OP_CLR, }; - qemu_send_msg(&msg); -} - -void sel4_set_irq(unsigned int irq, bool state) -{ - send_qemu_op(state ? QEMU_OP_SET_IRQ : QEMU_OP_CLR_IRQ, irq); + if (sel4_vm_ioctl(s, SEL4_SET_IRQLINE, &irqline)) + fprintf(stderr, "Failed to set irq: %m\n"); } static void sel4_change_state_handler(void *opaque, bool running, RunState state) { + SeL4State *s = opaque; if (running) { - open_dataports(); printf("Starting user VM\n"); - send_qemu_op(QEMU_OP_START_VM, 0); + if (sel4_vm_ioctl(s, SEL4_START_VM, 0)) + fprintf(stderr, "Failed to start user VM: %m\n"); } } -static void qemu_send_msg(const rpcmsg_t *msg) +static void sel4_mmio_do_io(struct sel4_ioreq_mmio *mmio) { - qemu_mutex_lock(&vmm_send_mutex); - rpcmsg_t *reply = rpcmsg_queue_tail(tx_queue); - if (!reply) { - printf("QEMU TX queue full\n"); - abort(); - } - *reply = *msg; - smp_mb(); - rpcmsg_queue_advance_tail(tx_queue); - vmm_doorbell_ring(); - qemu_mutex_unlock(&vmm_send_mutex); -} + sel4_listener_region_t *lr = NULL; -static void qemu_request_read(rpcmsg_t *msg) -{ - rpcmsg_t reply = *msg; - qemu_pci_read(&reply.mr3, reply.mr1, reply.mr2, QEMU_PCIDEV(reply.mr0)); - qemu_send_msg(&reply); -} - -static void qemu_request_write(rpcmsg_t *msg) -{ - rpcmsg_t reply = *msg; - /* Writes are synchronous, hence we need to reply. We can do it before - * doing the actual write, since any new fault cannot be served before - * we finish this function anyway. */ - qemu_send_msg(&reply); - qemu_pci_write(&reply.mr3, reply.mr1, reply.mr2, QEMU_PCIDEV(reply.mr0)); - uintptr_t addr = reply.mr1; - if (!is_raw_address(addr) && pci_base[QEMU_PCIDEV(reply.mr0)]) { - addr += pci_base[QEMU_PCIDEV(reply.mr0)]; + qemu_mutex_lock_iothread(); + switch (mmio->direction) { + case SEL4_IO_DIR_WRITE: + address_space_write(&address_space_memory, mmio->addr, MEMTXATTRS_UNSPECIFIED, &mmio->data, mmio->len); + lr = lr_find_match(mmio->addr, mmio->len); + break; + case SEL4_IO_DIR_READ: + address_space_read(&address_space_memory, mmio->addr, MEMTXATTRS_UNSPECIFIED, &mmio->data, mmio->len); + break; + default: + error_report("sel4: invalid mmio direction (%d)", mmio->direction); + break; } - sel4_listener_region_t *lr = lr_find_match(addr, reply.mr2); + qemu_mutex_unlock_iothread(); + if (lr) { event_notifier_set(lr->n); } } -static void qemu_putc_log(rpcmsg_t *msg) +static void sel4_pci_do_io(struct sel4_ioreq_pci *pci) { - rpcmsg_t reply = *msg; + PCIDevice *dev = pci_devs[pci->pcidev]; + uint32_t val; + sel4_listener_region_t *lr = NULL; - for (size_t i = 0; i < logbuffer->sz; i++) { - if (logbuffer->data[i] != 0x0d) { - tii_printf("%c", logbuffer->data[i]); - } + qemu_mutex_lock_iothread(); + switch (pci->direction) { + case SEL4_IO_DIR_WRITE: + val = 0; + memcpy(&val, &pci->data, pci->len); + dev->config_write(dev, pci->addr, val, pci->len); + lr = lr_find_match(pci->addr + pci_base[pci->pcidev], pci->len); + break; + case SEL4_IO_DIR_READ: + val = dev->config_read(dev, pci->addr, pci->len); + memcpy(&pci->data, &val, pci->len); + break; + default: + error_report("sel4: invalid pci direction (%d)", pci->direction); + break; } + qemu_mutex_unlock_iothread(); - qemu_send_msg(&reply); + if (lr) { + event_notifier_set(lr->n); + } } -static void handle_qemu_request(rpcmsg_t *msg) +static inline handle_ioreq(SeL4State *s) { - switch (QEMU_OP(msg->mr0)) { - case QEMU_OP_READ: - qemu_request_read(msg); - break; - case QEMU_OP_WRITE: - qemu_request_write(msg); - break; - case QEMU_OP_PUTC_LOG: - qemu_putc_log(msg); - break; - default: - printf("Invalid operation %"PRIu32"\n", QEMU_OP(msg->mr0)); - abort(); + struct sel4_ioreq *ioreq; + int slot; + + for (slot = 0; slot < SEL4_MAX_IOREQS; slot++) { + ioreq = &s->ioreq_buffer->request_slots[slot]; + if (qatomic_load_acquire(&ioreq->state) == SEL4_IOREQ_STATE_PROCESSING) { + switch (ioreq->type) { + case SEL4_IOREQ_TYPE_MMIO: + sel4_mmio_do_io(&ioreq->req.mmio); + break; + case SEL4_IOREQ_TYPE_PCI: + sel4_pci_do_io(&ioreq->req.pci); + break; + default: + fprintf(stderr, "sel4: unknown ioreq type (%"PRIu32")", ioreq->type); + break; + } + + sel4_vm_ioctl(s, SEL4_NOTIFY_IO_HANDLED, slot); + } } } static void *do_sel4_virtio(void *opaque) { + SeL4State *s = opaque; + int rc; + for (;;) { - rpcmsg_t *msg = rpcmsg_queue_head(rx_queue); - if (msg) { - handle_qemu_request(msg); - rpcmsg_queue_advance_head(rx_queue); - continue; - } + rc = sel4_vm_ioctl(s, SEL4_WAIT_IO, 0); + if (rc) + continue; - vmm_doorbell_wait(); + handle_ioreq(s); } return NULL; @@ -392,9 +318,9 @@ static bool ioeventfd_exists(MemoryRegionSection *section, EventNotifier *e) } static void sel4_io_ioeventfd_add(MemoryListener *listener, - MemoryRegionSection *section, - bool match_data, uint64_t data, - EventNotifier *e) + MemoryRegionSection *section, + bool match_data, uint64_t data, + EventNotifier *e) { if (ioeventfd_exists(section, e)) { return; @@ -447,15 +373,84 @@ static MemoryListener sel4_io_listener = { static int sel4_init(MachineState *ms) { - open_dataports(); - MachineClass *mc = MACHINE_GET_CLASS(ms); + SeL4State *s = SEL4_STATE(ms->accelerator); + int rc; + struct sel4_vm_params params = { + .ram_size = ms->ram_size, + }; + void *ram = NULL; + + s->fd = open("/dev/sel4", O_RDWR); + if (s->fd == -1) { + fprintf(stderr, "sel4: Failed to open kernel module: %m\n"); + return -errno; + } + + do { + rc = sel4_ioctl(s, SEL4_CREATE_VM, ¶ms); + } while (rc == -EINTR); + + if (rc < 0) { + fprintf(stderr, "sel4: create VM failed: %d %s\n", -rc, + strerror(-rc)); + goto err; + } + + s->vmfd = rc; + + /* setup ram */ + ram = mmap(NULL, ms->ram_size, PROT_READ | PROT_WRITE, + MAP_SHARED, s->vmfd, 0); + if (!ram) { + fprintf(stderr, "sel4: ram mmap failed: %m\n"); + goto err; + } + + memory_region_init_ram_ptr(&ram_mr, OBJECT(ms), "virt.ram", + ms->ram_size, ram); + vmstate_register_ram_global(&ram_mr); + ms->ram = &ram_mr; + + /* do not allocate RAM from generic code */ + mc->default_ram_id = NULL; + + rc = sel4_vm_ioctl(s, SEL4_CREATE_IO_HANDLER, 0); + if (rc < 0) { + fprintf(stderr, "sel4: create IO handler failed: %d %s\n", -rc, + strerror(-rc)); + goto err; + } + s->ioreqfd = rc; + + s->ioreq_buffer = mmap(NULL, sizeof(*s->ioreq_buffer), + PROT_READ | PROT_WRITE, MAP_SHARED, s->ioreqfd, 0); + if (!s->ioreq_buffer) { + fprintf(stderr, "sel4: iohandler mmap failed %m\n"); + goto err; + } + memory_listener_register(&sel4_io_listener, &address_space_memory); - qemu_add_vm_change_state_handler(sel4_change_state_handler, NULL); + qemu_add_vm_change_state_handler(sel4_change_state_handler, s); return 0; + +err: + if (s->ioreq_buffer) + munmap(s->ioreq_buffer, sizeof(*s->ioreq_buffer)); + + if (ram) + munmap(ram, ms->ram_size); + + if (s->vmfd >= 0) + close(s->vmfd); + + if (s->fd >= 0) + close(s->fd); + + return rc; } static void sel4_accel_class_init(ObjectClass *oc, void *data) @@ -468,12 +463,22 @@ static void sel4_accel_class_init(ObjectClass *oc, void *data) ac->allowed = &sel4_allowed; } -#define TYPE_SEL4_ACCEL ACCEL_CLASS_NAME("sel4") +static void sel4_accel_instance_init(Object *obj) +{ + SeL4State *s = SEL4_STATE(obj); + + s->fd = -1; + s->vmfd = -1; + s->ioreqfd = -1; + s->ioreq_buffer = NULL; +} static const TypeInfo sel4_accel_type = { .name = TYPE_SEL4_ACCEL, .parent = TYPE_ACCEL, + .instance_init = sel4_accel_instance_init, .class_init = sel4_accel_class_init, + .instance_size = sizeof(SeL4State), }; static void sel4_accel_ops_class_init(ObjectClass *oc, void *data) @@ -485,7 +490,6 @@ static void sel4_accel_ops_class_init(ObjectClass *oc, void *data) static const TypeInfo sel4_accel_ops_type = { .name = ACCEL_OPS_NAME("sel4"), - .parent = TYPE_ACCEL_OPS, .class_init = sel4_accel_ops_class_init, .abstract = true, From 9541d5e77269fdaf7aa36dc19861543a08bb4828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markku=20Ahvenj=C3=A4rvi?= Date: Fri, 3 Feb 2023 14:31:14 +0200 Subject: [PATCH 12/28] Fix compiler warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Markku Ahvenjärvi --- accel/sel4/sel4-all.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c index bc5c5094641..c9962fe2d1b 100644 --- a/accel/sel4/sel4-all.c +++ b/accel/sel4/sel4-all.c @@ -94,14 +94,6 @@ static void sel4_setup_post(MachineState *ms, AccelState *accel) do_sel4_virtio, s, QEMU_THREAD_JOINABLE); } -static int is_raw_address(uint32_t address) -{ - if (address >= (1ULL << 20)) - return 1; - - return 0; -} - void qmp_ringbuf_write(const char *, const char *, bool, int, Error **); char *qmp_ringbuf_read(const char *device, int64_t size, bool has_format, int format, Error **errp); @@ -264,7 +256,7 @@ static void sel4_pci_do_io(struct sel4_ioreq_pci *pci) } } -static inline handle_ioreq(SeL4State *s) +static inline void handle_ioreq(SeL4State *s) { struct sel4_ioreq *ioreq; int slot; @@ -430,7 +422,6 @@ static int sel4_init(MachineState *ms) goto err; } - memory_listener_register(&sel4_io_listener, &address_space_memory); qemu_add_vm_change_state_handler(sel4_change_state_handler, s); From c570cf96783200fbb50987335d1a4b0b39121520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markku=20Ahvenj=C3=A4rvi?= Date: Fri, 17 Mar 2023 09:32:48 +0200 Subject: [PATCH 13/28] Remove GH lockdown workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Markku Ahvenjärvi --- .github/workflows/lockdown.yml | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 .github/workflows/lockdown.yml diff --git a/.github/workflows/lockdown.yml b/.github/workflows/lockdown.yml deleted file mode 100644 index d5e1265cffb..00000000000 --- a/.github/workflows/lockdown.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Configuration for Repo Lockdown - https://github.com/dessant/repo-lockdown - -name: 'Repo Lockdown' - -on: - pull_request_target: - types: opened - -permissions: - pull-requests: write - -jobs: - action: - runs-on: ubuntu-latest - steps: - - uses: dessant/repo-lockdown@v2 - with: - pr-comment: | - Thank you for your interest in the QEMU project. - - This repository is a read-only mirror of the project's repostories hosted - on https://gitlab.com/qemu-project/qemu.git. - The project does not process merge requests filed on GitHub. - - QEMU welcomes contributions of code (either fixing bugs or adding new - functionality). However, we get a lot of patches, and so we have some - guidelines about contributing on the project website: - https://www.qemu.org/contribute/ - lock-pr: true - close-pr: true From 0e82e9b2eb69eeadf228cb36eaf6204d480ee8bb Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Thu, 23 Mar 2023 19:23:49 +0200 Subject: [PATCH 14/28] Enlarge VM1 memory size to 256 MB This change aligns the memory size with VM1 ram_size. Signed-off-by: Hannu Lyytinen --- hw/arm/virt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hw/arm/virt.c b/hw/arm/virt.c index 06fa2fe0a29..342208f129a 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -162,7 +162,7 @@ static const MemMapEntry base_memmap[] = { [VIRT_PCIE_PIO] = { 0x3eff0000, 0x00010000 }, [VIRT_PCIE_ECAM] = { 0x3f000000, 0x01000000 }, /* Actual RAM size depends on initial RAM and device memory settings */ - [VIRT_MEM] = { 0x48000000, 0x08000000 }, + [VIRT_MEM] = { 0x48000000, 0x10000000 }, }; /* From f522f885e303a54df3e00e294cacf29d88e8a0e2 Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Thu, 16 Mar 2023 23:40:55 +0200 Subject: [PATCH 15/28] Revert memory map changes Signed-off-by: Hannu Lyytinen --- hw/arm/virt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hw/arm/virt.c b/hw/arm/virt.c index 342208f129a..242857405c9 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -158,11 +158,11 @@ static const MemMapEntry base_memmap[] = { /* ...repeating for a total of NUM_VIRTIO_TRANSPORTS, each of that size */ [VIRT_PLATFORM_BUS] = { 0x0c000000, 0x02000000 }, [VIRT_SECURE_MEM] = { 0x0e000000, 0x01000000 }, - [VIRT_PCIE_MMIO] = { 0x60000000, 0x7eff0000 }, + [VIRT_PCIE_MMIO] = { 0x10000000, 0x2eff0000 }, [VIRT_PCIE_PIO] = { 0x3eff0000, 0x00010000 }, [VIRT_PCIE_ECAM] = { 0x3f000000, 0x01000000 }, /* Actual RAM size depends on initial RAM and device memory settings */ - [VIRT_MEM] = { 0x48000000, 0x10000000 }, + [VIRT_MEM] = { GiB, LEGACY_RAMLIMIT_BYTES }, }; /* From 8bb7654be1d399ca39cdf5103e26e61ab4b8a5cc Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Thu, 16 Mar 2023 23:38:47 +0200 Subject: [PATCH 16/28] sel4: Use uservm configuration from /proc/cmdline Signed-off-by: Hannu Lyytinen --- accel/sel4/sel4-all.c | 90 +++++++++++++++++++++++++++++++++++++++++++ hw/arm/virt.c | 10 ++++- 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c index c9962fe2d1b..cba8c6bb328 100644 --- a/accel/sel4/sel4-all.c +++ b/accel/sel4/sel4-all.c @@ -17,10 +17,14 @@ #include "hw/pci/pci_bus.h" #include "migration/vmstate.h" +#include "hw/arm/virt.h" + #include #include #include +extern MemMapEntry (*virt_memmap_customize)(const MemMapEntry *base_memmap, int i); + void tii_printf(const char *fmt, ...); static MemoryRegion ram_mr; @@ -444,6 +448,90 @@ static int sel4_init(MachineState *ms) return rc; } +static const int vmid = 1; +static unsigned long uservm_ram_base; +static unsigned long uservm_ram_size; +static unsigned long uservm_pcie_mmio_base; +static unsigned long uservm_pcie_mmio_size; + +static int parse_kernel_bootargs(void) +{ + FILE *fp; + char buffer[4096]; + char *s, *arg, *sp; + int ret = -1; + int id; + + if ((fp = fopen("/proc/cmdline", "r")) == NULL) { + goto out; + } + if (fgets(buffer, sizeof buffer, fp) == NULL) { + goto close_fp; + } + + for (s = buffer; (arg = strtok_r(s, " \t\n", &sp)) != NULL; s = NULL) { + if (strncmp(arg, "uservm=", 7)) { + continue; + } + if (sscanf(arg + 7, "%d,%lx,%lx,%lx,%lx", + &id, + &uservm_ram_base, + &uservm_ram_size, + &uservm_pcie_mmio_base, + &uservm_pcie_mmio_size) != 5) { + fprintf(stderr, "Improper %s in bootargs\n", arg); + goto close_fp; + } + if (id == vmid) { + ret = 0; + break; + } + } + +close_fp: + fclose(fp); + +out: + return ret; +} + +static MemMapEntry sel4_memmap_customize(const MemMapEntry *base_memmap, int i) +{ + static bool init = false; + MemMapEntry e; + + if (!init) { + if (parse_kernel_bootargs()) { + fprintf(stderr, "No uservm details given in kernel bootargs\n"); + exit(1); + } + init = true; + } + + switch (i) { + case VIRT_MEM: + e.base = uservm_ram_base; + e.size = uservm_ram_size; + break; + case VIRT_PCIE_MMIO: + e.base = uservm_pcie_mmio_base; + e.size = uservm_pcie_mmio_size; + break; + case VIRT_PCIE_PIO: + e.base = uservm_pcie_mmio_base + uservm_pcie_mmio_size; + e.size = 0x10000; + break; + case VIRT_PCIE_ECAM: + e.base = uservm_pcie_mmio_base + uservm_pcie_mmio_size + 0x10000; + e.size = 0x1000000; + break; + default: + return base_memmap[i]; + }; + + return e; +} + static void sel4_accel_class_init(ObjectClass *oc, void *data) { AccelClass *ac = ACCEL_CLASS(oc); @@ -462,6 +550,8 @@ static void sel4_accel_instance_init(Object *obj) s->vmfd = -1; s->ioreqfd = -1; s->ioreq_buffer = NULL; + + virt_memmap_customize = sel4_memmap_customize; } static const TypeInfo sel4_accel_type = { diff --git a/hw/arm/virt.c b/hw/arm/virt.c index 242857405c9..653fd0ae0d4 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -1688,6 +1688,14 @@ static uint64_t virt_cpu_mp_affinity(VirtMachineState *vms, int idx) return arm_cpu_mp_affinity(idx, clustersz); } +static MemMapEntry noop_memmap_customize(const MemMapEntry *base_memmap, int i) +{ + return base_memmap[i]; +} + +MemMapEntry (*virt_memmap_customize)(const MemMapEntry *base_memmap, int i) = + noop_memmap_customize; + static void virt_set_memmap(VirtMachineState *vms, int pa_bits) { MachineState *ms = MACHINE(vms); @@ -1697,7 +1705,7 @@ static void virt_set_memmap(VirtMachineState *vms, int pa_bits) vms->memmap = extended_memmap; for (i = 0; i < ARRAY_SIZE(base_memmap); i++) { - vms->memmap[i] = base_memmap[i]; + vms->memmap[i] = virt_memmap_customize(base_memmap, i); } if (ms->ram_slots > ACPI_MAX_RAM_SLOTS) { From f9b568a6dd9db8e10f7d5dbeb9816dd491293c32 Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Wed, 19 Apr 2023 10:33:36 +0300 Subject: [PATCH 17/28] Make seL4 hooks no-op on other accels Signed-off-by: Hannu Lyytinen --- accel/sel4/sel4-all.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c index cba8c6bb328..77a9ffee5ae 100644 --- a/accel/sel4/sel4-all.c +++ b/accel/sel4/sel4-all.c @@ -25,6 +25,8 @@ extern MemMapEntry (*virt_memmap_customize)(const MemMapEntry *base_memmap, int i); +static MemMapEntry sel4_memmap_customize(const MemMapEntry *base_memmap, int i); + void tii_printf(const char *fmt, ...); static MemoryRegion ram_mr; @@ -130,8 +132,17 @@ static int pci_resolve_irq(PCIDevice *pci_dev, int irq_num) return irq_num; } +static inline bool using_sel4(void) +{ + return virt_memmap_customize == sel4_memmap_customize; +} + void sel4_register_pci_device(PCIDevice *d) { + if (!using_sel4()) { + return; + } + SeL4State *s = SEL4_STATE(current_accel()); struct sel4_vpci_device vpcidev = { .pcidev = pci_dev_count, @@ -187,6 +198,10 @@ static sel4_listener_region_t *lr_find_match(hwaddr addr, Int128 size) void sel4_set_irq(unsigned int irq, bool state) { + if (!using_sel4()) { + return; + } + SeL4State *s = SEL4_STATE(current_accel()); struct sel4_irqline irqline = { .irq = irq, From 770ccbd10d59ac81518b8599dbfe60435fafe672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markku=20Ahvenj=C3=A4rvi?= Date: Wed, 24 May 2023 15:46:31 +0300 Subject: [PATCH 18/28] sel4: use ioeventfd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Markku Ahvenjärvi --- accel/sel4/sel4-all.c | 130 +++++++++++++++++------------------------- 1 file changed, 53 insertions(+), 77 deletions(-) diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c index 77a9ffee5ae..a66ad7877c4 100644 --- a/accel/sel4/sel4-all.c +++ b/accel/sel4/sel4-all.c @@ -50,7 +50,8 @@ typedef struct SeL4State int vmfd; int ioreqfd; struct sel4_iohandler_buffer *ioreq_buffer; -} SeL4State; + MemoryListener mem_listener; +} SeL4State ; #define TYPE_SEL4_ACCEL ACCEL_CLASS_NAME("sel4") @@ -161,41 +162,6 @@ void sel4_register_pci_device(PCIDevice *d) printf("IRQ for this device is %d\n", pci_resolve_irq(d, 0)); } -typedef struct { - Int128 size; - hwaddr offset_within_address_space; - EventNotifier *n; -} sel4_listener_region_t; - -#define LR_MAX 512 - -static sel4_listener_region_t lrs[LR_MAX]; -static unsigned int nlrs; - -static int lr_match(sel4_listener_region_t *lr, hwaddr addr, Int128 size) -{ - if (addr >= lr->offset_within_address_space + int128_get64(lr->size)) { - return 0; - } - if (addr + int128_get64(size) < lr->offset_within_address_space) { - return 0; - } - return 1; -} - -static sel4_listener_region_t *lr_find_match(hwaddr addr, Int128 size) -{ - int i; - - for (i = 0; i < nlrs; i++) { - if (lr_match(&lrs[i], addr, size)) { - return &lrs[i]; - } - } - - return NULL; -} - void sel4_set_irq(unsigned int irq, bool state) { if (!using_sel4()) { @@ -224,13 +190,10 @@ static void sel4_change_state_handler(void *opaque, bool running, RunState state static void sel4_mmio_do_io(struct sel4_ioreq_mmio *mmio) { - sel4_listener_region_t *lr = NULL; - qemu_mutex_lock_iothread(); switch (mmio->direction) { case SEL4_IO_DIR_WRITE: address_space_write(&address_space_memory, mmio->addr, MEMTXATTRS_UNSPECIFIED, &mmio->data, mmio->len); - lr = lr_find_match(mmio->addr, mmio->len); break; case SEL4_IO_DIR_READ: address_space_read(&address_space_memory, mmio->addr, MEMTXATTRS_UNSPECIFIED, &mmio->data, mmio->len); @@ -240,17 +203,12 @@ static void sel4_mmio_do_io(struct sel4_ioreq_mmio *mmio) break; } qemu_mutex_unlock_iothread(); - - if (lr) { - event_notifier_set(lr->n); - } } static void sel4_pci_do_io(struct sel4_ioreq_pci *pci) { PCIDevice *dev = pci_devs[pci->pcidev]; uint32_t val; - sel4_listener_region_t *lr = NULL; qemu_mutex_lock_iothread(); switch (pci->direction) { @@ -258,7 +216,6 @@ static void sel4_pci_do_io(struct sel4_ioreq_pci *pci) val = 0; memcpy(&val, &pci->data, pci->len); dev->config_write(dev, pci->addr, val, pci->len); - lr = lr_find_match(pci->addr + pci_base[pci->pcidev], pci->len); break; case SEL4_IO_DIR_READ: val = dev->config_read(dev, pci->addr, pci->len); @@ -269,10 +226,6 @@ static void sel4_pci_do_io(struct sel4_ioreq_pci *pci) break; } qemu_mutex_unlock_iothread(); - - if (lr) { - event_notifier_set(lr->n); - } } static inline void handle_ioreq(SeL4State *s) @@ -316,16 +269,30 @@ static void *do_sel4_virtio(void *opaque) return NULL; } -static bool ioeventfd_exists(MemoryRegionSection *section, EventNotifier *e) +static int sel4_ioeventfd_set(SeL4State *s, int fd, hwaddr addr, uint32_t val, + bool assign, uint32_t size, bool datamatch) { - for (int i = 0; i < nlrs; i++) { - if (lrs[i].offset_within_address_space == section->offset_within_address_space && - lrs[i].size == section->size && - lrs[i].n == e) { - return true; - } + struct sel4_ioeventfd_config config = { + .fd = fd, + .addr = addr, + .len = size, + .data = datamatch ? val : 0, + .flags = 0, + }; + + if (datamatch) { + config.flags |= SEL4_IOEVENTFD_FLAG_DATAMATCH; + } + + if (!assign) { + config.flags |= SEL4_IOEVENTFD_FLAG_DEASSIGN; + } + + if (sel4_vm_ioctl(s, SEL4_IOEVENTFD, &config) < 0) { + return -errno; } - return false; + + return 0; } static void sel4_io_ioeventfd_add(MemoryListener *listener, @@ -333,18 +300,18 @@ static void sel4_io_ioeventfd_add(MemoryListener *listener, bool match_data, uint64_t data, EventNotifier *e) { - if (ioeventfd_exists(section, e)) { - return; - } - - sel4_listener_region_t *lr = &lrs[nlrs++]; - - tii_printf("%s: mr=%p offset=0x%"PRIx64" size=0x%"PRIx64" notifier=%p\n", __func__, section->mr, - section->offset_within_address_space, int128_get64(section->size), e); + SeL4State *s = container_of(listener, SeL4State, mem_listener); + int fd = event_notifier_get_fd(e); + int rc; - lr->offset_within_address_space = section->offset_within_address_space; - lr->size = section->size; - lr->n = e; + rc = sel4_ioeventfd_set(s, fd, section->offset_within_address_space, + data, true, int128_get64(section->size), + match_data); + if (rc < 0) { + fprintf(stderr, "%s: error adding ioeventfd: %s (%d)\n", + __func__, strerror(-rc), -rc); + abort(); + } } static void sel4_io_ioeventfd_del(MemoryListener *listener, @@ -353,7 +320,18 @@ static void sel4_io_ioeventfd_del(MemoryListener *listener, EventNotifier *e) { - tii_printf("WARNING: %s UNIMPLEMENTED\n", __func__); + SeL4State *s = container_of(listener, SeL4State, mem_listener); + int fd = event_notifier_get_fd(e); + int rc; + + rc = sel4_ioeventfd_set(s, fd, section->offset_within_address_space, + data, false, int128_get64(section->size), + match_data); + if (rc < 0) { + fprintf(stderr, "%s: error deleting ioeventfd: %s (%d)\n", + __func__, strerror(-rc), -rc); + abort(); + } } static void sel4_region_add(MemoryListener *listener, MemoryRegionSection *section) @@ -375,13 +353,6 @@ static void sel4_region_del(MemoryListener *listener, MemoryRegionSection *secti tii_printf("WARNING: %s entered, but no real implementation\n", __func__); } -static MemoryListener sel4_io_listener = { - .eventfd_add = sel4_io_ioeventfd_add, - .eventfd_del = sel4_io_ioeventfd_del, - .region_add = sel4_region_add, - .region_del = sel4_region_del, -}; - static int sel4_init(MachineState *ms) { MachineClass *mc = MACHINE_GET_CLASS(ms); @@ -441,7 +412,12 @@ static int sel4_init(MachineState *ms) goto err; } - memory_listener_register(&sel4_io_listener, &address_space_memory); + s->mem_listener.eventfd_add = sel4_io_ioeventfd_add; + s->mem_listener.eventfd_del = sel4_io_ioeventfd_del; + s->mem_listener.region_add = sel4_region_add; + s->mem_listener.region_del = sel4_region_del; + + memory_listener_register(&s->mem_listener, &address_space_memory); qemu_add_vm_change_state_handler(sel4_change_state_handler, s); From 5610658ae25d1743ee512d81a76d25d3ce260521 Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Fri, 14 Jul 2023 10:02:36 +0300 Subject: [PATCH 19/28] Unify IO request types Signed-off-by: Hannu Lyytinen --- accel/sel4/sel4-all.c | 42 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c index a66ad7877c4..df340471333 100644 --- a/accel/sel4/sel4-all.c +++ b/accel/sel4/sel4-all.c @@ -188,41 +188,41 @@ static void sel4_change_state_handler(void *opaque, bool running, RunState state } } -static void sel4_mmio_do_io(struct sel4_ioreq_mmio *mmio) +static void sel4_mmio_do_io(struct sel4_ioreq *ioreq) { qemu_mutex_lock_iothread(); - switch (mmio->direction) { + switch (ioreq->direction) { case SEL4_IO_DIR_WRITE: - address_space_write(&address_space_memory, mmio->addr, MEMTXATTRS_UNSPECIFIED, &mmio->data, mmio->len); + address_space_write(&address_space_memory, ioreq->addr, MEMTXATTRS_UNSPECIFIED, &ioreq->data, ioreq->len); break; case SEL4_IO_DIR_READ: - address_space_read(&address_space_memory, mmio->addr, MEMTXATTRS_UNSPECIFIED, &mmio->data, mmio->len); + address_space_read(&address_space_memory, ioreq->addr, MEMTXATTRS_UNSPECIFIED, &ioreq->data, ioreq->len); break; default: - error_report("sel4: invalid mmio direction (%d)", mmio->direction); + error_report("sel4: invalid ioreq direction (%d)", ioreq->direction); break; } qemu_mutex_unlock_iothread(); } -static void sel4_pci_do_io(struct sel4_ioreq_pci *pci) +static void sel4_pci_do_io(struct sel4_ioreq *ioreq) { - PCIDevice *dev = pci_devs[pci->pcidev]; - uint32_t val; + PCIDevice *dev = pci_devs[ioreq->addr_space]; + uint64_t val; qemu_mutex_lock_iothread(); - switch (pci->direction) { + switch (ioreq->direction) { case SEL4_IO_DIR_WRITE: val = 0; - memcpy(&val, &pci->data, pci->len); - dev->config_write(dev, pci->addr, val, pci->len); + memcpy(&val, &ioreq->data, ioreq->len); + dev->config_write(dev, ioreq->addr, val, ioreq->len); break; case SEL4_IO_DIR_READ: - val = dev->config_read(dev, pci->addr, pci->len); - memcpy(&pci->data, &val, pci->len); + val = dev->config_read(dev, ioreq->addr, ioreq->len); + memcpy(&ioreq->data, &val, ioreq->len); break; default: - error_report("sel4: invalid pci direction (%d)", pci->direction); + error_report("sel4: invalid ioreq direction (%d)", ioreq->direction); break; } qemu_mutex_unlock_iothread(); @@ -236,16 +236,10 @@ static inline void handle_ioreq(SeL4State *s) for (slot = 0; slot < SEL4_MAX_IOREQS; slot++) { ioreq = &s->ioreq_buffer->request_slots[slot]; if (qatomic_load_acquire(&ioreq->state) == SEL4_IOREQ_STATE_PROCESSING) { - switch (ioreq->type) { - case SEL4_IOREQ_TYPE_MMIO: - sel4_mmio_do_io(&ioreq->req.mmio); - break; - case SEL4_IOREQ_TYPE_PCI: - sel4_pci_do_io(&ioreq->req.pci); - break; - default: - fprintf(stderr, "sel4: unknown ioreq type (%"PRIu32")", ioreq->type); - break; + if (ioreq->addr_space == AS_GLOBAL) { + sel4_mmio_do_io(ioreq); + } else { + sel4_pci_do_io(ioreq); } sel4_vm_ioctl(s, SEL4_NOTIFY_IO_HANDLED, slot); From 626d4ac127d155c333373b263db893c89db9aa2f Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Sun, 27 Aug 2023 10:01:54 +0300 Subject: [PATCH 20/28] Support multiple guest VMs Signed-off-by: Hannu Lyytinen --- accel/sel4/sel4-all.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c index df340471333..6b727e29cd1 100644 --- a/accel/sel4/sel4-all.c +++ b/accel/sel4/sel4-all.c @@ -29,6 +29,7 @@ static MemMapEntry sel4_memmap_customize(const MemMapEntry *base_memmap, int i); void tii_printf(const char *fmt, ...); +static int vmid = 1; static MemoryRegion ram_mr; bool sel4_allowed; @@ -353,10 +354,17 @@ static int sel4_init(MachineState *ms) SeL4State *s = SEL4_STATE(ms->accelerator); int rc; struct sel4_vm_params params = { + .id = 1, .ram_size = ms->ram_size, }; void *ram = NULL; + char *p = getenv("VMID"); + if (p) { + vmid = atoi(p); + } + params.id = vmid; + s->fd = open("/dev/sel4", O_RDWR); if (s->fd == -1) { fprintf(stderr, "sel4: Failed to open kernel module: %m\n"); @@ -433,7 +441,7 @@ static int sel4_init(MachineState *ms) return rc; } -static const int vmid = 1; + static unsigned long uservm_ram_base; static unsigned long uservm_ram_size; static unsigned long uservm_pcie_mmio_base; @@ -473,6 +481,10 @@ static int parse_kernel_bootargs(void) } } + fprintf(stderr, "User VM %d, shared RAM %zu bytes at 0x%"PRIxPTR", PCI MMIO %zu bytes at 0x%"PRIxPTR"\n", + id, uservm_ram_size, uservm_ram_base, + uservm_pcie_mmio_size, uservm_pcie_mmio_base); + close_fp: fclose(fp); From 39fb7c97c3289fcacb4be7e18e37aec41bc0219b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markku=20Ahvenj=C3=A4rvi?= Date: Wed, 1 Nov 2023 10:36:58 +0200 Subject: [PATCH 21/28] Add seL4 header Signed-off-by: Hannu Lyytinen --- include/sysemu/sel4.h | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 include/sysemu/sel4.h diff --git a/include/sysemu/sel4.h b/include/sysemu/sel4.h new file mode 100644 index 00000000000..3949314d124 --- /dev/null +++ b/include/sysemu/sel4.h @@ -0,0 +1,33 @@ +/* + * QEMU seL4 support + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#ifndef SYSEMU_SEL4_H +#define SYSEMU_SEL4_H + +#ifdef NEED_CPU_H +# ifdef CONFIG_SEL4 +# define CONFIG_SEL4_IS_POSSIBLE +# endif +#else +# define CONFIG_SEL4_IS_POSSIBLE +#endif + +#ifdef CONFIG_SEL4_IS_POSSIBLE + +extern bool sel4_allowed; + +#define sel4_enabled() (sel4_allowed) + +void sel4_set_irq(unsigned int irq, bool); + +#else /* !CONFIG_SEL4_IS_POSSIBLE */ + +#define sel4_enabled() 0 + +#endif /* CONFIG_SEL4_IS_POSSIBLE */ + +#endif From b696331cc1ee147a809a38afaacb3eee6833c513 Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Wed, 1 Nov 2023 10:38:26 +0200 Subject: [PATCH 22/28] Make using_sel4() public Signed-off-by: Hannu Lyytinen --- accel/sel4/sel4-all.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c index 6b727e29cd1..4ab2fdd5578 100644 --- a/accel/sel4/sel4-all.c +++ b/accel/sel4/sel4-all.c @@ -12,6 +12,7 @@ #include "qemu/atomic.h" #include "sysemu/cpus.h" #include "sysemu/runstate.h" +#include "sysemu/sel4.h" #include "hw/boards.h" #include "hw/pci/pci.h" #include "hw/pci/pci_bus.h" @@ -134,7 +135,7 @@ static int pci_resolve_irq(PCIDevice *pci_dev, int irq_num) return irq_num; } -static inline bool using_sel4(void) +bool using_sel4(void) { return virt_memmap_customize == sel4_memmap_customize; } From 8a22b085c5105b9b75d848a1e8a1d14757e8fefc Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Wed, 1 Nov 2023 10:07:56 +0200 Subject: [PATCH 23/28] Improve PCI slot index book keeping Signed-off-by: Hannu Lyytinen --- accel/sel4/sel4-all.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c index 4ab2fdd5578..3b8b8e6337f 100644 --- a/accel/sel4/sel4-all.c +++ b/accel/sel4/sel4-all.c @@ -24,6 +24,8 @@ #include #include +#define PCI_NUM_SLOTS (32) + extern MemMapEntry (*virt_memmap_customize)(const MemMapEntry *base_memmap, int i); static MemMapEntry sel4_memmap_customize(const MemMapEntry *base_memmap, int i); @@ -117,8 +119,7 @@ void tii_printf(const char *fmt, ...) qmp_ringbuf_write("debug", tmp, false, 0, &err); } -static unsigned int pci_dev_count; -static PCIDevice *pci_devs[16]; +static PCIDevice *pci_devs[PCI_NUM_SLOTS]; static uintptr_t pci_base[16]; static unsigned int pci_base_count; @@ -146,20 +147,20 @@ void sel4_register_pci_device(PCIDevice *d) return; } + unsigned int slot = PCI_SLOT(d->devfn); + SeL4State *s = SEL4_STATE(current_accel()); struct sel4_vpci_device vpcidev = { - .pcidev = pci_dev_count, + .pcidev = slot, }; - pci_devs[pci_dev_count] = d; - printf("Registering PCI device to VMM\n"); + pci_devs[slot] = d; + printf("Registering PCI device %u to VMM\n", slot); if (sel4_vm_ioctl(s, SEL4_CREATE_VPCI_DEVICE, &vpcidev)) { fprintf(stderr, "Failed to register PCI device: %m\n"); } - pci_dev_count++; - // INTX = 1 -> IRQ 0 printf("IRQ for this device is %d\n", pci_resolve_irq(d, 0)); } From 790e2501fe97ce3c536e74b8d5715065ff8bdfb8 Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Tue, 19 Sep 2023 10:17:18 +0300 Subject: [PATCH 24/28] Refactor code to bypass kernel Signed-off-by: Hannu Lyytinen --- accel/sel4/sel4-all.c | 199 ++++++++++++++++++++++++++++++----------- accel/sel4/sel4-qemu.h | 150 ------------------------------- 2 files changed, 146 insertions(+), 203 deletions(-) delete mode 100644 accel/sel4/sel4-qemu.h diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c index 3b8b8e6337f..5b4c04fe3b8 100644 --- a/accel/sel4/sel4-all.c +++ b/accel/sel4/sel4-all.c @@ -26,6 +26,8 @@ #define PCI_NUM_SLOTS (32) +#define EVENT_BAR_EMIT_REGISTER 0x0 + extern MemMapEntry (*virt_memmap_customize)(const MemMapEntry *base_memmap, int i); static MemMapEntry sel4_memmap_customize(const MemMapEntry *base_memmap, int i); @@ -53,7 +55,10 @@ typedef struct SeL4State int fd; int vmfd; int ioreqfd; - struct sel4_iohandler_buffer *ioreq_buffer; + int event_bar_fd; + void *iobuf; + void *event_bar; + sel4_rpc_t rpc; MemoryListener mem_listener; } SeL4State ; @@ -95,8 +100,6 @@ static int sel4_vm_ioctl(SeL4State *s, int type, ...) return ret; } -void seL4_Yield(void){} - static void sel4_setup_post(MachineState *ms, AccelState *accel) { SeL4State *s = SEL4_STATE(ms->accelerator); @@ -191,76 +194,137 @@ static void sel4_change_state_handler(void *opaque, bool running, RunState state } } -static void sel4_mmio_do_io(struct sel4_ioreq *ioreq) +static int sel4_mmio_do_io(unsigned int dir, seL4_Word addr, seL4_Word *data, + unsigned int len) { - qemu_mutex_lock_iothread(); - switch (ioreq->direction) { - case SEL4_IO_DIR_WRITE: - address_space_write(&address_space_memory, ioreq->addr, MEMTXATTRS_UNSPECIFIED, &ioreq->data, ioreq->len); - break; - case SEL4_IO_DIR_READ: - address_space_read(&address_space_memory, ioreq->addr, MEMTXATTRS_UNSPECIFIED, &ioreq->data, ioreq->len); - break; - default: - error_report("sel4: invalid ioreq direction (%d)", ioreq->direction); - break; + MemTxResult result; + + if (dir == SEL4_IO_DIR_READ) { + result = address_space_read(&address_space_memory, addr, + MEMTXATTRS_UNSPECIFIED, data, len); + } else { + result = address_space_write(&address_space_memory, addr, + MEMTXATTRS_UNSPECIFIED, data, len); } - qemu_mutex_unlock_iothread(); + + return (result == MEMTX_OK) ? 0 : -1; } -static void sel4_pci_do_io(struct sel4_ioreq *ioreq) +static int sel4_pci_do_io(unsigned int addr_space, unsigned int dir, + seL4_Word addr, seL4_Word *data, + unsigned int len) { - PCIDevice *dev = pci_devs[ioreq->addr_space]; + if (addr_space >= ARRAY_SIZE(pci_devs)) { + return -1; + } + + PCIDevice *dev = pci_devs[addr_space]; + if (!dev) { + return -1; + } + uint64_t val; - qemu_mutex_lock_iothread(); - switch (ioreq->direction) { - case SEL4_IO_DIR_WRITE: + if (dir == SEL4_IO_DIR_READ) { + val = dev->config_read(dev, addr, len); + memcpy(data, &val, len); + } else { val = 0; - memcpy(&val, &ioreq->data, ioreq->len); - dev->config_write(dev, ioreq->addr, val, ioreq->len); - break; - case SEL4_IO_DIR_READ: - val = dev->config_read(dev, ioreq->addr, ioreq->len); - memcpy(&ioreq->data, &val, ioreq->len); - break; - default: - error_report("sel4: invalid ioreq direction (%d)", ioreq->direction); - break; + memcpy(&val, data, len); + dev->config_write(dev, addr, val, len); } + + return 0; +} + +static inline int handle_mmio(SeL4State *s, rpcmsg_t *req) +{ + int err; + + seL4_Word dir = BIT_FIELD_GET(req->mr0, RPC_MR0_MMIO_DIRECTION); + seL4_Word as = BIT_FIELD_GET(req->mr0, RPC_MR0_MMIO_ADDR_SPACE); + seL4_Word len = BIT_FIELD_GET(req->mr0, RPC_MR0_MMIO_LENGTH); + seL4_Word slot = BIT_FIELD_GET(req->mr0, RPC_MR0_MMIO_SLOT); + seL4_Word addr = req->mr1; + seL4_Word data = req->mr2; + + qemu_mutex_lock_iothread(); + + if (as == AS_GLOBAL) { + err = sel4_mmio_do_io(dir, addr, &data, len); + } else { + err = sel4_pci_do_io(as, dir, addr, &data, len); + } + qemu_mutex_unlock_iothread(); + + if (err) { + fprintf(stderr, "%s failed, addr=0x%lx, dir=%u\n", __func__, addr, dir); + exit(1); + } + + return driver_ack_mmio_finish(&s->rpc, slot, data) ? 0 : -1; } -static inline void handle_ioreq(SeL4State *s) +static int handle_msg(SeL4State *s, rpcmsg_t *msg) { - struct sel4_ioreq *ioreq; - int slot; - - for (slot = 0; slot < SEL4_MAX_IOREQS; slot++) { - ioreq = &s->ioreq_buffer->request_slots[slot]; - if (qatomic_load_acquire(&ioreq->state) == SEL4_IOREQ_STATE_PROCESSING) { - if (ioreq->addr_space == AS_GLOBAL) { - sel4_mmio_do_io(ioreq); - } else { - sel4_pci_do_io(ioreq); - } + unsigned int op = BIT_FIELD_GET(msg->mr0, RPC_MR0_OP); - sel4_vm_ioctl(s, SEL4_NOTIFY_IO_HANDLED, slot); - } + switch (op) { + case RPC_MR0_OP_MMIO: + return handle_mmio(s, msg); + default: + /* no idea what to do */ + return -1; } + + return 0; +} + +static inline int sel4_vm_wait_for_io(SeL4State *s) +{ + return sel4_vm_ioctl(s, SEL4_WAIT_IO, 0); } static void *do_sel4_virtio(void *opaque) { SeL4State *s = opaque; + rpcmsg_t *msg; int rc; for (;;) { - rc = sel4_vm_ioctl(s, SEL4_WAIT_IO, 0); + sel4_rpc_doorbell(&s->rpc); + + rc = sel4_vm_wait_for_io(s); if (rc) continue; - handle_ioreq(s); + while ((msg = rpcmsg_queue_head(s->rpc.rx_queue)) != NULL) { + seL4_Word state = BIT_FIELD_GET(msg->mr0, RPC_MR0_STATE); + + if (state == RPC_MR0_STATE_RESERVED) { + /* VMM side is still crafting the message, try again later */ + break; + } else if (state == RPC_MR0_STATE_PENDING) { + /* kernel has not examined this yet, try again later */ + break; + } else if (state != RPC_MR0_STATE_PROCESSING) { + /* kernel is supposed to discard completed ioeventfds */ + error_report("corrupted msgqueue"); + exit(1); + } + + rc = handle_msg(s, msg); + if (rc) { + error_report("handle_msg() failed (%d)", rc); + exit(1); + } + + msg->mr0 = BIT_FIELD_SET(msg->mr0, RPC_MR0_STATE, RPC_MR0_STATE_COMPLETE); + + /* smp_wmb()? */ + rpcmsg_queue_advance_head(s->rpc.rx_queue); + } } return NULL; @@ -350,6 +414,14 @@ static void sel4_region_del(MemoryListener *listener, MemoryRegionSection *secti tii_printf("WARNING: %s entered, but no real implementation\n", __func__); } +static void s2_fault_doorbell(void *cookie) +{ + uint32_t *event_bar = cookie; + + /* data does not matter, just the write fault */ + event_bar[0] = 1; +} + static int sel4_init(MachineState *ms) { MachineClass *mc = MACHINE_GET_CLASS(ms); @@ -409,13 +481,31 @@ static int sel4_init(MachineState *ms) } s->ioreqfd = rc; - s->ioreq_buffer = mmap(NULL, sizeof(*s->ioreq_buffer), - PROT_READ | PROT_WRITE, MAP_SHARED, s->ioreqfd, 0); - if (!s->ioreq_buffer) { + s->iobuf = mmap(NULL, 2 * 4096, PROT_READ | PROT_WRITE, MAP_SHARED, + s->ioreqfd, 0); + if (!s->iobuf) { fprintf(stderr, "sel4: iohandler mmap failed %m\n"); goto err; } + rc = sel4_vm_ioctl(s, SEL4_CREATE_EVENT_BAR, 0); + if (rc < 0) { + fprintf(stderr, "sel4: create event handler failed: %d %s\n", -rc, + strerror(-rc)); + goto err; + } + s->event_bar_fd = rc; + s->event_bar = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, + s->event_bar_fd, 0); + if (!s->iobuf) { + fprintf(stderr, "sel4: event mmap failed %m\n"); + goto err; + } + + s->rpc.rx_queue = device_rx_queue(s->iobuf); + s->rpc.tx_queue = device_tx_queue(s->iobuf); + s->rpc.doorbell = s2_fault_doorbell; + s->rpc.doorbell_cookie = (void *)(((uintptr_t)s->event_bar) + EVENT_BAR_EMIT_REGISTER); s->mem_listener.eventfd_add = sel4_io_ioeventfd_add; s->mem_listener.eventfd_del = sel4_io_ioeventfd_del; s->mem_listener.region_add = sel4_region_add; @@ -428,8 +518,11 @@ static int sel4_init(MachineState *ms) return 0; err: - if (s->ioreq_buffer) - munmap(s->ioreq_buffer, sizeof(*s->ioreq_buffer)); + if (s->event_bar) + munmap(s->event_bar, 4096); + + if (s->iobuf) + munmap(s->iobuf, 2 * 4096); if (ram) munmap(ram, ms->ram_size); @@ -548,7 +641,7 @@ static void sel4_accel_instance_init(Object *obj) s->fd = -1; s->vmfd = -1; s->ioreqfd = -1; - s->ioreq_buffer = NULL; + memset(&s->rpc, 0, sizeof(s->rpc)); virt_memmap_customize = sel4_memmap_customize; } diff --git a/accel/sel4/sel4-qemu.h b/accel/sel4/sel4-qemu.h deleted file mode 100644 index 931582b3c7d..00000000000 --- a/accel/sel4/sel4-qemu.h +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright 2022, Technology Innovation Institute - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#ifdef QEMU -typedef unsigned long seL4_Word; -#define ORIGIN "" -#define debug_printf(fmt, ...) qemu_printf(fmt "\n", ## __VA_ARGS__) -#else -#define ORIGIN "VMM " -#define debug_printf(fmt, ...) printf(fmt "\n", ## __VA_ARGS__) -#endif - -typedef struct { - size_t sz; - char data[1024]; -} logbuffer_t; - - -#ifndef QEMU -/* to QEMU */ -#define tx_queue (((rpcmsg_queue_t *) ctrl) + 0) - -/* from QEMU */ -#define rx_queue (((rpcmsg_queue_t *) ctrl) + 1) - -#define logbuffer ((logbuffer_t *)(rx_queue + 1)) -#else -/* from VMM */ -#define rx_queue (((rpcmsg_queue_t *) dataports[DP_CTRL].data) + 0) - -/* to VMM */ -#define tx_queue (((rpcmsg_queue_t *) dataports[DP_CTRL].data) + 1) - -#define logbuffer ((logbuffer_t *)(tx_queue + 1)) -#endif - -/* from VMM to QEMU */ -#define QEMU_OP_READ 0 -#define QEMU_OP_WRITE 1 -#define QEMU_OP_PUTC_LOG 2 - -/* from QEMU to VMM */ -#define QEMU_OP_SET_IRQ 16 -#define QEMU_OP_CLR_IRQ 17 -#define QEMU_OP_START_VM 18 -#define QEMU_OP_REGISTER_PCI_DEV 19 - -#define QEMU_OP_MASK 0xffULL -#define QEMU_OP_SHIFT 0 -#define QEMU_OP(__x__) ((unsigned int)(((__x__) & QEMU_OP_MASK) >> QEMU_OP_SHIFT)) - -#define QEMU_PCIDEV_MASK 0xff00ULL -#define QEMU_PCIDEV_SHIFT 8 -#define QEMU_PCIDEV(__x__) ((unsigned int)(((__x__) & QEMU_PCIDEV_MASK) >> QEMU_PCIDEV_SHIFT)) - -#define QEMU_ID_MASK 0xff0000ULL -#define QEMU_ID_SHIFT 16 -#define QEMU_ID(__x__) ((unsigned int)(((__x__) & QEMU_ID_MASK) >> QEMU_ID_SHIFT)) -#define QEMU_ID_FROM(__x__) (((__x__) << QEMU_ID_SHIFT) & QEMU_ID_MASK) - -#define QEMU_VCPU_MASK 0xff000000ULL -#define QEMU_VCPU_SHIFT 24 -#define QEMU_VCPU(__x__) ((unsigned int)(((__x__) & QEMU_VCPU_MASK) >> QEMU_VCPU_SHIFT)) -#define QEMU_VCPU_NONE 0xff - -#define RPCMSG_BUFFER_SIZE 32 - -typedef struct { - seL4_Word mr0; - seL4_Word mr1; - seL4_Word mr2; - seL4_Word mr3; -} rpcmsg_t; - -typedef struct { - uint32_t head; - uint32_t tail; - uint32_t rsvd[2]; - rpcmsg_t data[RPCMSG_BUFFER_SIZE]; -} rpcmsg_queue_t; - -#define QUEUE_PREV(_i) ((_i) ? ((_i) - 1) : (RPCMSG_BUFFER_SIZE - 1)) -#define QUEUE_NEXT(_i) (((_i) + 1) & (RPCMSG_BUFFER_SIZE - 1)) - -static void rpcmsg_queue_init(rpcmsg_queue_t *q) -{ - memset(q, 0, sizeof(*q)); -} - -static void rpcmsg_queue_dump(const char *name, rpcmsg_queue_t *q, unsigned int idx) -{ - unsigned int start_idx = idx; - char tmp[128]; - - sprintf(tmp, ORIGIN "name = %s head = %02d tail = %02d", name, q->head, q->tail); - debug_printf("%s", tmp); - - do { - rpcmsg_t *msg = &q->data[idx]; - sprintf(tmp, ORIGIN "%02d: %08"PRIx64" %08"PRIx64" %08"PRIx64" %08"PRIx64, - idx, msg->mr0, msg->mr1, msg->mr2, msg->mr3); - debug_printf("%s", tmp); - idx = QUEUE_PREV(idx); - } while (idx != start_idx); -} - -static inline bool rpcmsg_queue_full(rpcmsg_queue_t *q) -{ - return QUEUE_NEXT(q->tail) == q->head; -} - -static inline bool rpcmsg_queue_empty(rpcmsg_queue_t *q) -{ - return q->tail == q->head; -} - -static inline rpcmsg_t *rpcmsg_queue_head(rpcmsg_queue_t *q) -{ - return rpcmsg_queue_empty(q) ? NULL : (q->data + q->head); -} - -static inline rpcmsg_t *rpcmsg_queue_tail(rpcmsg_queue_t *q) -{ - - return rpcmsg_queue_full(q) ? NULL : (q->data + q->tail); -} - -static inline void rpcmsg_queue_advance_head(rpcmsg_queue_t *q) -{ - assert(!rpcmsg_queue_empty(q)); - q->head = QUEUE_NEXT(q->head); -} - -static inline void rpcmsg_queue_advance_tail(rpcmsg_queue_t *q) -{ - assert(!rpcmsg_queue_full(q)); - q->tail = QUEUE_NEXT(q->tail); -} - -static inline void rpcmsg_queue_enqueue(rpcmsg_queue_t *q, rpcmsg_t *msg) -{ - assert(!rpcmsg_queue_full(q)); - memcpy(q->data + q->tail, msg, sizeof(*msg)); - q->tail = QUEUE_NEXT(q->tail); -} From e09a104de6c2e578d63a975be404c4e33ac33458 Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Fri, 27 Oct 2023 14:33:50 +0300 Subject: [PATCH 25/28] Remove context switch for emulated IRQs Signed-off-by: Hannu Lyytinen --- accel/sel4/sel4-all.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c index 5b4c04fe3b8..39c11b05638 100644 --- a/accel/sel4/sel4-all.c +++ b/accel/sel4/sel4-all.c @@ -170,18 +170,24 @@ void sel4_register_pci_device(PCIDevice *d) void sel4_set_irq(unsigned int irq, bool state) { + int err; + if (!using_sel4()) { return; } SeL4State *s = SEL4_STATE(current_accel()); - struct sel4_irqline irqline = { - .irq = irq, - .op = state ? SEL4_IRQ_OP_SET : SEL4_IRQ_OP_CLR, - }; - if (sel4_vm_ioctl(s, SEL4_SET_IRQLINE, &irqline)) - fprintf(stderr, "Failed to set irq: %m\n"); + if (state) { + err = sel4_rpc_doorbell(driver_req_set_irqline(&s->rpc, irq)); + } else { + err = sel4_rpc_doorbell(driver_req_clear_irqline(&s->rpc, irq)); + } + + if (err) { + fprintf(stderr, "Failed to set IRQ %u\n", irq); + exit(1); + } } static void sel4_change_state_handler(void *opaque, bool running, RunState state) From e5f4b538bcd667fca75fbca03e2d609a2ffa6f98 Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Tue, 31 Oct 2023 12:18:12 +0200 Subject: [PATCH 26/28] irq 1/3: Prettier virtio-pci hook Signed-off-by: Hannu Lyytinen --- hw/virtio/virtio-pci.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c index fa998857268..73bf5b13f05 100644 --- a/hw/virtio/virtio-pci.c +++ b/hw/virtio/virtio-pci.c @@ -1621,8 +1621,6 @@ static void virtio_pci_pre_plugged(DeviceState *d, Error **errp) virtio_add_feature(&vdev->host_features, VIRTIO_F_BAD_FEATURE); } -void sel4_register_pci_device(PCIDevice *d); - /* This is called by virtio-bus just after the device is plugged. */ static void virtio_pci_device_plugged(DeviceState *d, Error **errp) { @@ -1764,8 +1762,6 @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp) proxy->pci_dev.config_write = virtio_write_config; proxy->pci_dev.config_read = virtio_read_config; - sel4_register_pci_device(&proxy->pci_dev); - if (legacy) { size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev) + virtio_bus_get_vdev_config_len(bus); @@ -1780,6 +1776,17 @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp) } } +void sel4_register_pci_device(PCIDevice *d); + +static void sel4_virtio_pci_device_plugged(DeviceState *d, Error **errp) +{ + VirtIOPCIProxy *proxy = VIRTIO_PCI(d); + + virtio_pci_device_plugged(d, errp); + + sel4_register_pci_device(&proxy->pci_dev); +} + static void virtio_pci_device_unplugged(DeviceState *d) { VirtIOPCIProxy *proxy = VIRTIO_PCI(d); @@ -2226,7 +2233,7 @@ static void virtio_pci_bus_class_init(ObjectClass *klass, void *data) k->set_host_notifier_mr = virtio_pci_set_host_notifier_mr; k->vmstate_change = virtio_pci_vmstate_change; k->pre_plugged = virtio_pci_pre_plugged; - k->device_plugged = virtio_pci_device_plugged; + k->device_plugged = sel4_virtio_pci_device_plugged; k->device_unplugged = virtio_pci_device_unplugged; k->query_nvectors = virtio_pci_query_nvectors; k->ioeventfd_enabled = virtio_pci_ioeventfd_enabled; From 04ac9c316b60946f4476b7a03e0a9c1ead2d624e Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Fri, 27 Oct 2023 14:36:34 +0300 Subject: [PATCH 27/28] irq 2/3: Improve accelerator code Signed-off-by: Hannu Lyytinen --- accel/sel4/sel4-all.c | 16 ---------------- hw/pci-host/gpex.c | 18 +++++++++++++++--- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/accel/sel4/sel4-all.c b/accel/sel4/sel4-all.c index 39c11b05638..2269e2caa72 100644 --- a/accel/sel4/sel4-all.c +++ b/accel/sel4/sel4-all.c @@ -126,19 +126,6 @@ static PCIDevice *pci_devs[PCI_NUM_SLOTS]; static uintptr_t pci_base[16]; static unsigned int pci_base_count; -static int pci_resolve_irq(PCIDevice *pci_dev, int irq_num) -{ - PCIBus *bus; - for (;;) { - bus = pci_get_bus(pci_dev); - irq_num = bus->map_irq(pci_dev, irq_num); - if (bus->set_irq) - break; - pci_dev = bus->parent_dev; - } - return irq_num; -} - bool using_sel4(void) { return virt_memmap_customize == sel4_memmap_customize; @@ -163,9 +150,6 @@ void sel4_register_pci_device(PCIDevice *d) if (sel4_vm_ioctl(s, SEL4_CREATE_VPCI_DEVICE, &vpcidev)) { fprintf(stderr, "Failed to register PCI device: %m\n"); } - - // INTX = 1 -> IRQ 0 - printf("IRQ for this device is %d\n", pci_resolve_irq(d, 0)); } void sel4_set_irq(unsigned int irq, bool state) diff --git a/hw/pci-host/gpex.c b/hw/pci-host/gpex.c index cea57ddb51d..2a93e583d20 100644 --- a/hw/pci-host/gpex.c +++ b/hw/pci-host/gpex.c @@ -36,13 +36,12 @@ #include "hw/qdev-properties.h" #include "migration/vmstate.h" #include "qemu/module.h" +#include "sysemu/sel4.h" /**************************************************************************** * GPEX host */ -void sel4_set_irq(unsigned int irq, bool); - static void gpex_set_irq(void *opaque, int irq_num, int level) { GPEXHost *s = opaque; @@ -78,6 +77,19 @@ static PCIINTxRoute gpex_route_intx_pin_to_irq(void *opaque, int pin) return route; } +/* + * 0 <= pin <= 3 0 = INTA, 1 = INTB, 2 = INTC, 3 = INTD + * 0-origin unlike PCI interrupt pin register. + */ +int sel4_pci_swizzle_map_irq_fn(PCIDevice *pci_dev, int pin) +{ + if (!using_sel4()) { + return pci_swizzle(PCI_SLOT(pci_dev->devfn), pin); + } + + return 4 * PCI_SLOT(pci_dev->devfn) + pin; +} + static void gpex_host_realize(DeviceState *dev, Error **errp) { PCIHostState *pci = PCI_HOST_BRIDGE(dev); @@ -138,7 +150,7 @@ static void gpex_host_realize(DeviceState *dev, Error **errp) } pci->bus = pci_register_root_bus(dev, "pcie.0", gpex_set_irq, - pci_swizzle_map_irq_fn, s, &s->io_mmio, + sel4_pci_swizzle_map_irq_fn, s, &s->io_mmio, &s->io_ioport, 0, 4, TYPE_PCIE_BUS); pci_bus_set_route_irq_fn(pci->bus, gpex_route_intx_pin_to_irq); From 7e82dccf6cbcaf5c1b880acc7a0b16576f4b6f58 Mon Sep 17 00:00:00 2001 From: Hannu Lyytinen Date: Wed, 1 Nov 2023 09:04:32 +0200 Subject: [PATCH 28/28] irq 3/3: Support seL4 accelerator changes Signed-off-by: Hannu Lyytinen --- hw/arm/virt.c | 4 ++-- hw/pci-host/gpex.c | 2 +- include/hw/pci-host/gpex.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hw/arm/virt.c b/hw/arm/virt.c index 653fd0ae0d4..2bfc925fa4c 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -109,7 +109,7 @@ /* Number of external interrupt lines to configure the GIC with */ -#define NUM_IRQS 256 +#define NUM_IRQS 512 #define PLATFORM_BUS_NUM_IRQS 64 @@ -186,7 +186,7 @@ static MemMapEntry extended_memmap[] = { static const int a15irqmap[] = { [VIRT_UART] = 1, [VIRT_RTC] = 2, - [VIRT_PCIE] = 3, /* ... to 6 */ + [VIRT_PCIE] = 176, /* ... to 6 */ [VIRT_GPIO] = 7, [VIRT_SECURE_UART] = 8, [VIRT_ACPI_GED] = 9, diff --git a/hw/pci-host/gpex.c b/hw/pci-host/gpex.c index 2a93e583d20..d8f63277f31 100644 --- a/hw/pci-host/gpex.c +++ b/hw/pci-host/gpex.c @@ -151,7 +151,7 @@ static void gpex_host_realize(DeviceState *dev, Error **errp) pci->bus = pci_register_root_bus(dev, "pcie.0", gpex_set_irq, sel4_pci_swizzle_map_irq_fn, s, &s->io_mmio, - &s->io_ioport, 0, 4, TYPE_PCIE_BUS); + &s->io_ioport, 0, GPEX_NUM_IRQS, TYPE_PCIE_BUS); pci_bus_set_route_irq_fn(pci->bus, gpex_route_intx_pin_to_irq); qdev_realize(DEVICE(&s->gpex_root), BUS(pci->bus), &error_fatal); diff --git a/include/hw/pci-host/gpex.h b/include/hw/pci-host/gpex.h index fcf8b638200..700de4d9759 100644 --- a/include/hw/pci-host/gpex.h +++ b/include/hw/pci-host/gpex.h @@ -32,7 +32,7 @@ OBJECT_DECLARE_SIMPLE_TYPE(GPEXHost, GPEX_HOST) #define TYPE_GPEX_ROOT_DEVICE "gpex-root" OBJECT_DECLARE_SIMPLE_TYPE(GPEXRootState, GPEX_ROOT_DEVICE) -#define GPEX_NUM_IRQS 4 +#define GPEX_NUM_IRQS (32*4) struct GPEXRootState { /*< private >*/