From c0fc034e34065d72d148e675c93ac19418050da1 Mon Sep 17 00:00:00 2001 From: Narayana Murty N Date: Mon, 8 Jan 2024 03:17:17 -0600 Subject: [PATCH 01/12] Adding ppc specific changes to test on powerpc systems This commit will ppc extension to the Alex Williamson vfio testing programs --- vfio-pci-device-open.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/vfio-pci-device-open.c b/vfio-pci-device-open.c index 5a33cdd..afaa2e0 100644 --- a/vfio-pci-device-open.c +++ b/vfio-pci-device-open.c @@ -13,6 +13,7 @@ #include #include +#include #define VFIO_API_VERSION 0 @@ -22,6 +23,8 @@ /* Extensions */ #define VFIO_TYPE1_IOMMU 1 +#define VFIO_TYPE1v2_IOMMU 3 +#define VFIO_SPAPR_TCE_v2_IOMMU 7 #define VFIO_NOIOMMU_IOMMU 8 /* @@ -506,13 +509,15 @@ int main(int argc, char **argv) } printf("pre-SET_CONTAINER:\n"); + +#ifndef __PPC64__ printf("VFIO_CHECK_EXTENSION VFIO_TYPE1_IOMMU: %sPresent\n", ioctl(container, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) ? "" : "Not "); printf("VFIO_CHECK_EXTENSION VFIO_NOIOMMU_IOMMU: %sPresent\n", ioctl(container, VFIO_CHECK_EXTENSION, VFIO_NOIOMMU_IOMMU) ? "" : "Not "); - +#endif ret = ioctl(group, VFIO_GROUP_SET_CONTAINER, &container); if (ret) { printf("Failed to set group container\n"); @@ -520,7 +525,8 @@ int main(int argc, char **argv) } printf("post-SET_CONTAINER:\n"); - printf("VFIO_CHECK_EXTENSION VFIO_TYPE1_IOMMU: %sPresent\n", +#ifndef __PPC64__ + printf("VFIO_CHECK_EXTENSION VFIO_TYPE1_IOMMU: %sPresent\n", ioctl(container, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) ? "" : "Not "); printf("VFIO_CHECK_EXTENSION VFIO_NOIOMMU_IOMMU: %sPresent\n", @@ -538,6 +544,20 @@ int main(int argc, char **argv) printf("Failed to set IOMMU\n"); return ret; } +#else + printf("VFIO_CHECK_EXTENSION VFIO_SPAPR_TCE_v2_IOMMU: %s Present\n", + ioctl(container, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_v2_IOMMU) ? + "" : "Not "); + + + ret = ioctl(container, VFIO_SET_IOMMU, VFIO_SPAPR_TCE_v2_IOMMU); + if (ret) { + printf("Failed to set IOMMU\n"); + return ret; + } + printf("VFIO_SET_IOMMU to VFIO_SPAPR_TCE_v2_IOMMU successfull\n"); + +#endif snprintf(path, sizeof(path), "%04x:%02x:%02x.%d", seg, bus, dev, func); From 9d9512acef8cd756b111e2aab7c38356341f9253 Mon Sep 17 00:00:00 2001 From: Narayana Murty N Date: Mon, 8 Jan 2024 03:35:17 -0600 Subject: [PATCH 02/12] Adding dma map/unmap test specific to ppc powerpc specific implementation to register memory before mapping to pci device. and creating second window for huge memory mapping to io devices. --- vfio-huge-guest-test.c | 180 ++++++++++++++++++++++++++++++++++++++++- vfio-iommu-map-unmap.c | 64 ++++++++++++++- 2 files changed, 239 insertions(+), 5 deletions(-) diff --git a/vfio-huge-guest-test.c b/vfio-huge-guest-test.c index 1ce4edd..d817701 100644 --- a/vfio-huge-guest-test.c +++ b/vfio-huge-guest-test.c @@ -22,6 +22,7 @@ /* Extensions */ #define VFIO_TYPE1_IOMMU 1 +#define VFIO_SPAPR_TCE_v2_IOMMU 7 /* * The IOCTL interface is designed for extensibility by embedding the @@ -378,6 +379,69 @@ struct vfio_iommu_type1_dma_unmap { #define VFIO_IOMMU_UNMAP_DMA _IO(VFIO_TYPE, VFIO_BASE + 14) + +/** + * VFIO_IOMMU_SPAPR_REGISTER_MEMORY - _IOW(VFIO_TYPE, VFIO_BASE + 17, struct vfio_iommu_spapr_register_memory) + * + * Registers user space memory where DMA is allowed. It pins + * user pages and does the locked memory accounting so + * subsequent VFIO_IOMMU_MAP_DMA/VFIO_IOMMU_UNMAP_DMA calls + * get faster. + */ +struct vfio_iommu_spapr_register_memory { + __u32 argsz; + __u32 flags; + __u64 vaddr; /* Process virtual address */ + __u64 size; /* Size of mapping (bytes) */ +}; +#define VFIO_IOMMU_SPAPR_REGISTER_MEMORY _IO(VFIO_TYPE, VFIO_BASE + 17) + +/** + * VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY - _IOW(VFIO_TYPE, VFIO_BASE + 18, struct vfio_iommu_spapr_register_memory) + * + * Unregisters user space memory registered with + * VFIO_IOMMU_SPAPR_REGISTER_MEMORY. + * Uses vfio_iommu_spapr_register_memory for parameters. + */ +#define VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY _IO(VFIO_TYPE, VFIO_BASE + 18) + +/** + * VFIO_IOMMU_SPAPR_TCE_CREATE - _IOWR(VFIO_TYPE, VFIO_BASE + 19, struct vfio_iommu_spapr_tce_create) + * + * Creates an additional TCE table and programs it (sets a new DMA window) + * to every IOMMU group in the container. It receives page shift, window + * size and number of levels in the TCE table being created. + * + * It allocates and returns an offset on a PCI bus of the new DMA window. + */ +struct vfio_iommu_spapr_tce_create { + __u32 argsz; + __u32 flags; + /* in */ + __u32 page_shift; + __u32 __resv1; + __u64 window_size; + __u32 levels; + __u32 __resv2; + /* out */ + __u64 start_addr; +}; +#define VFIO_IOMMU_SPAPR_TCE_CREATE _IO(VFIO_TYPE, VFIO_BASE + 19) + +/** + * VFIO_IOMMU_SPAPR_TCE_REMOVE - _IOW(VFIO_TYPE, VFIO_BASE + 20, struct vfio_iommu_spapr_tce_remove) + * + * Unprograms a TCE table from all groups in the container and destroys it. + * It receives a PCI bus offset as a window id. + */ +struct vfio_iommu_spapr_tce_remove { + __u32 argsz; + __u32 flags; + /* in */ + __u64 start_addr; +}; +#define VFIO_IOMMU_SPAPR_TCE_REMOVE _IO(VFIO_TYPE, VFIO_BASE + 20) + #endif /* _UAPIVFIO_H */ #include @@ -392,8 +456,12 @@ struct vfio_iommu_type1_dma_unmap { #include #include #include +#include #include +#ifdef __PPC64__ +#include +#endif #define MMAP_GB (4UL) #define MMAP_SIZE (MMAP_GB * 1024 * 1024 * 1024) @@ -416,6 +484,23 @@ int main(int argc, char **argv) .argsz = sizeof(dma_map) }; + struct vfio_iommu_type1_dma_map dma_map64 = { + .argsz = sizeof(dma_map64) + }; + + struct vfio_iommu_spapr_register_memory reg64 = { + .argsz = sizeof(reg64), + .flags = 0 + }; + + struct vfio_iommu_type1_dma_unmap dma64_unmap = { + .argsz = sizeof(dma64_unmap) + }; + + struct vfio_iommu_spapr_tce_create tce_create = { + .argsz = sizeof(tce_create) + }; + if (argc < 2) { usage(argv[0]); return -1; @@ -466,13 +551,14 @@ int main(int argc, char **argv) printf("Failed to set group container\n"); return ret; } + dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; +#ifndef __PPC64__ ret = ioctl(container, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU); if (ret) { printf("Failed to set IOMMU\n"); return ret; } - if (strlen(mempath)) { struct statfs fs; @@ -509,7 +595,6 @@ int main(int argc, char **argv) return -1; } - dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; /* 640K@0, enough for anyone */ printf("Mapping 0-640K"); @@ -528,7 +613,7 @@ int main(int argc, char **argv) printf("Mapping low memory"); fflush(stdout); dma_map.size = (3UL * 1024 * 1024 * 1024) - (1024 * 1024); - dma_map.iova = 1024 * 1024; + dma_map.iova = tce_create.start_addr ;//1024 * 1024; dma_map.vaddr = vaddr + dma_map.iova; ret = ioctl(container, VFIO_IOMMU_MAP_DMA, &dma_map); if (ret) { @@ -537,10 +622,84 @@ int main(int argc, char **argv) } printf(".\n"); +#else + ret = ioctl(container, VFIO_SET_IOMMU, VFIO_SPAPR_TCE_v2_IOMMU); + if (ret) { + printf("Failed to set IOMMU\n"); + return ret; + } + + /* create 64 bit second window */ + tce_create.window_size = MMAP_SIZE; //(16UL *1024*1024*1024);//4194306; //2097152 ; // 1 page + tce_create.page_shift = __builtin_ctzll ((2UL*1024*1024)); + tce_create.levels = 1; + tce_create.flags = 0; + + if(ioctl(container, VFIO_IOMMU_SPAPR_TCE_CREATE, &tce_create)){ + printf("Create second window failed %d \n",errno); + perror("VFIO_IOMMU_SPAPR_TCE_CREATE : "); + } else { + printf("second window created successfully \n"); + } + + /* map huge page more than 1GB page */ +/* + reg64.vaddr = (long unsigned int)mmap(0, (2UL*1024*1024*1024 ), PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_HUGE_1GB , 0, 0); + printf("Mapping high memory at 0x%lx\n", tce_create.start_addr); + fflush(stdout); + + reg64.size = 2UL*1024*1024*1024;//MMAP_SIZE; + reg64.flags = 0; + if (ioctl(container, VFIO_IOMMU_SPAPR_REGISTER_MEMORY, ®64)) { + perror("Set iommu register memory failed\n"); + exit(-1); + } + printf("Register success\n"); + getchar(); + + dma_map64.vaddr = reg64.vaddr; + dma_map64.size = reg64.size; + //dma_map64.size = 2UL*1024*1024*1024; + dma_map64.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + dma_map64.iova = tce_create.start_addr; + + ret = ioctl(container, VFIO_IOMMU_MAP_DMA, &dma_map64); + if (ret) { + printf("Failed to map memory (%s)\n", strerror(errno)); + getchar(); + exit(-1); + } + + printf("Mapping done. Enter a key\n"); + + getchar(); + */ +#endif /* (1TB - 4G)@4G "high memory" after the I/O hole */ printf("Mapping high memory"); fflush(stdout); +#ifdef __PPC64__ + /* map huge page more than 1GB page */ + + vaddr = (long unsigned int)mmap(0, MMAP_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_HUGE_1GB , 0, 0); + printf("Mapping high memory at 0x%lx\n", tce_create.start_addr); + fflush(stdout); + + reg64.vaddr = vaddr; + reg64.size = MMAP_SIZE; + reg64.flags = 0; + if (ioctl(container, VFIO_IOMMU_SPAPR_REGISTER_MEMORY, ®64)) { + perror("Set iommu register memory failed\n"); + exit(-1); + } + printf("Register success\n"); + getchar(); +#endif + dma_map.size = MMAP_SIZE; +#ifndef __PPC64__ dma_map.iova = 4UL * 1024 * 1024 * 1024; dma_map.vaddr = vaddr; while (dma_map.iova < GUEST_GB * 1024 * 1024 * 1024) { @@ -553,6 +712,21 @@ int main(int argc, char **argv) fflush(stdout); dma_map.iova += MMAP_SIZE; } +#else + dma_map.iova = tce_create.start_addr; + dma_map.vaddr = vaddr; + + ret = ioctl(container, VFIO_IOMMU_MAP_DMA, &dma_map); + if (ret) { + printf("Failed to map memory (%s)\n", strerror(errno)); + getchar(); + exit(-1); + } + + printf("Mapping done. Enter a key\n"); + + getchar(); +#endif printf("\n"); if (fd >= 0) diff --git a/vfio-iommu-map-unmap.c b/vfio-iommu-map-unmap.c index c7030b2..11153c8 100644 --- a/vfio-iommu-map-unmap.c +++ b/vfio-iommu-map-unmap.c @@ -13,8 +13,13 @@ #include #include +#ifndef __PPC64__ #define MAP_SIZE (1UL * 1024 * 1024 * 1024) -#define MAP_CHUNK (4 * 1024) +#define MAP_CHUNK (4UL * 1024) +#else +#define MAP_SIZE (1UL * 1024 * 1024*1024) +#define MAP_CHUNK (64*1024) +#endif #define REALLOC_INTERVAL 30 void usage(char *name) @@ -45,6 +50,14 @@ int main(int argc, char **argv) struct vfio_iommu_type1_dma_unmap dma_unmap = { .argsz = sizeof(dma_unmap) }; + struct vfio_iommu_spapr_register_memory reg = { + .argsz = sizeof(reg), + .flags = 0 + }; + struct vfio_iommu_spapr_tce_create tce_create = { + .argsz = sizeof(tce_create) + }; + if (argc != 2) { usage(argv[0]); @@ -116,12 +129,34 @@ int main(int argc, char **argv) return ret; } +#ifndef __PPC64__ ret = ioctl(container, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU); if (ret) { printf("Failed to set IOMMU\n"); return ret; } +#else + ret = ioctl(container, VFIO_SET_IOMMU, VFIO_SPAPR_TCE_v2_IOMMU); + if (ret) { + printf("Failed to set IOMMU\n"); + return ret; + } + + /*create window before mapping the dma buffers*/ + long ram_pagesize = sysconf(_SC_PAGESIZE); + tce_create.window_size = MAP_SIZE ; + tce_create.page_shift = __builtin_ctzll(ram_pagesize); + tce_create.levels = 1; + tce_create.flags = 0; + if(ioctl(container, VFIO_IOMMU_SPAPR_TCE_CREATE, &tce_create)){ + printf("Create window failed %d \n",errno); + perror("VFIO_IOMMU_SPAPR_TCE_CREATE : "); + } else { + printf("window created successfully \n"); + } + +#endif /* Test code */ dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; dma_map.size = MAP_CHUNK; @@ -137,6 +172,7 @@ int main(int argc, char **argv) memset(maps, 0, sizeof(void *) * (MAP_SIZE/dma_map.size)); + for (count = 0;; count++) { /* Every REALLOC_INTERVAL, dump our mappings to give THP something to collapse */ @@ -160,7 +196,7 @@ int main(int argc, char **argv) if (!maps[i]) { maps[i] = mmap(NULL, dma_map.size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); if (maps[i] == MAP_FAILED) { printf("Failed to mmap memory (%s)\n", strerror(errno)); return -1; @@ -173,6 +209,16 @@ int main(int argc, char **argv) } dma_map.vaddr = (unsigned long)maps[i]; +#ifdef __PPC64__ + reg.vaddr = dma_map.vaddr; + reg.size = MAP_CHUNK; + reg.flags = 0; + + if (ioctl(container, VFIO_IOMMU_SPAPR_REGISTER_MEMORY, ®)) { + perror("Set iommu register memory failed\n"); + return -1;; + } +#endif ret = ioctl(container, VFIO_IOMMU_MAP_DMA, &dma_map); if (ret) { @@ -180,6 +226,7 @@ int main(int argc, char **argv) strerror(errno)); return ret; } + printf("dma map was successful for MAP_CHUNK %d\n",i); } printf("+"); @@ -191,6 +238,19 @@ int main(int argc, char **argv) printf("Failed to unmap memory (%s)\n", strerror(errno)); return ret; } + printf("dma unmap was successful for MAP_SIZE \n"); +#ifdef __PPC64__ + for (i = dma_map.iova = 0; i < MAP_SIZE/dma_map.size; i++) { + reg.vaddr = (unsigned long)maps[i]; + reg.size = MAP_CHUNK; + reg.flags = 0; + + if (ioctl(container, VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY, ®)) { + perror("Set iommu unregister memory failed\n"); + return -1;; + } + } +#endif printf("-"); fflush(stdout); From 4a4b638fe28465b11740459a59b1a20ef1f4e6bb Mon Sep 17 00:00:00 2001 From: Narayana Murty N Date: Mon, 8 Jan 2024 15:28:04 +0530 Subject: [PATCH 03/12] Create README.md --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..9d8963a --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +summary : +VFIO test cases on ppc uses VFIO_SPAPR_TCE_v2_IOMMU. +read more on https://docs.kernel.org/driver-api/vfio.html. + +build & run : +$gcc -o vfio-pci-device-open vfio-pci-device-open.c +$gcc -o vfio-iommu-map-unmap vfio-iommu-map-unmap.c +$gcc -o vfio-huge-guest-test vfio-huge-guest-test.c + +$ vfio-pci-device-open +$ vfio-iommu-map-unmap ssss:bb:dd.f +$ vfio-huge-guest-test From af761a0a7ef90c810e49f5c079d992326b81a507 Mon Sep 17 00:00:00 2001 From: Narayana Murty N Date: Mon, 8 Jan 2024 15:29:08 +0530 Subject: [PATCH 04/12] Update README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 9d8963a..e22882e 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,15 @@ VFIO test cases on ppc uses VFIO_SPAPR_TCE_v2_IOMMU. read more on https://docs.kernel.org/driver-api/vfio.html. build & run : + $gcc -o vfio-pci-device-open vfio-pci-device-open.c + $gcc -o vfio-iommu-map-unmap vfio-iommu-map-unmap.c + $gcc -o vfio-huge-guest-test vfio-huge-guest-test.c $ vfio-pci-device-open + $ vfio-iommu-map-unmap ssss:bb:dd.f + $ vfio-huge-guest-test From 3c15d139ccf3eed3d21d07cbdf116ee9d8fa6786 Mon Sep 17 00:00:00 2001 From: Narayana Murty N Date: Mon, 8 Jan 2024 15:31:01 +0530 Subject: [PATCH 05/12] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e22882e..df83a9b 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,9 @@ $gcc -o vfio-iommu-map-unmap vfio-iommu-map-unmap.c $gcc -o vfio-huge-guest-test vfio-huge-guest-test.c -$ vfio-pci-device-open +$ vfio-pci-device-open \ \ $ vfio-iommu-map-unmap ssss:bb:dd.f -$ vfio-huge-guest-test +$ vfio-huge-guest-test \ + From 3bb42618b0cd543c92a079f3fac3cfab23a81e11 Mon Sep 17 00:00:00 2001 From: Narayana Murty N Date: Tue, 12 Mar 2024 09:54:29 -0500 Subject: [PATCH 06/12] In order to test the pseries support of vfio userspace driver in legecy mode the Alex Williamson vfio test code modified. --- vfio-huge-guest-test.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vfio-huge-guest-test.c b/vfio-huge-guest-test.c index d817701..0959973 100644 --- a/vfio-huge-guest-test.c +++ b/vfio-huge-guest-test.c @@ -643,7 +643,7 @@ int main(int argc, char **argv) } /* map huge page more than 1GB page */ -/* + reg64.vaddr = (long unsigned int)mmap(0, (2UL*1024*1024*1024 ), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_HUGE_1GB , 0, 0); printf("Mapping high memory at 0x%lx\n", tce_create.start_addr); @@ -673,8 +673,6 @@ int main(int argc, char **argv) printf("Mapping done. Enter a key\n"); - getchar(); - */ #endif /* (1TB - 4G)@4G "high memory" after the I/O hole */ printf("Mapping high memory"); From 0a607c1cd90164a7c8a04c3d1166474dbb55bffe Mon Sep 17 00:00:00 2001 From: Narayana Murty N Date: Wed, 26 Jun 2024 23:19:57 -0500 Subject: [PATCH 07/12] adding eeh test with nvme specific changes --- vfio-spapr-eeh-test.c | 431 ++++++++++++++++++++++++++++++++++++++++ vfio-spapr-eeh-test2.c | 433 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 864 insertions(+) create mode 100644 vfio-spapr-eeh-test.c create mode 100644 vfio-spapr-eeh-test2.c diff --git a/vfio-spapr-eeh-test.c b/vfio-spapr-eeh-test.c new file mode 100644 index 0000000..2e2c208 --- /dev/null +++ b/vfio-spapr-eeh-test.c @@ -0,0 +1,431 @@ + +#include +#include +#include + +#define VFIO_API_VERSION 0 + + +/* Kernel & User level defines for VFIO IOCTLs. */ + +/* Extensions */ + +#define VFIO_TYPE1_IOMMU 1 +#define VFIO_TYPE1v2_IOMMU 3 +#define VFIO_SPAPR_TCE_v2_IOMMU 7 +#define VFIO_NOIOMMU_IOMMU 8 + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#define CAP 0x0 +#define VS 0x8 +#define CC 0x14 +#define CSTS 0x1c +#define AQA 0x24 +#define ASQ 0x28 +#define ACQ 0x30 +#define S_DB 0x1000 +#define C_DB 0x1004 +#define S_DB_Q1 0x1008 +#define C_DB_Q1 0x100c + + +void usage(char *name) +{ + printf("usage: %s \n", name); +} + + + + struct device_t { + struct vfio_region_info regs[VFIO_PCI_NUM_REGIONS]; + struct vfio_irq_info irqs[VFIO_PCI_NUM_IRQS]; + void* mmio_addr; // mmio address (BAR0); + } pcidev; + + struct vfio_region_info* bar_info; + + static inline void write_u32(struct device_t* dev, int offset, uint32_t value) { + __asm__ volatile("" : : : "memory"); + *((volatile uint32_t*)(dev->mmio_addr + offset)) = value; + } + + static inline uint32_t read_u32(struct device_t* dev, int offset) { + __asm__ volatile("" : : : "memory"); + return *((volatile uint32_t*)(dev->mmio_addr + offset)); + } +static void eeh_dev_break_debugfs(char *bdf, int bdf_len) +{ + int fd; + printf("buf out : %s %d\n", bdf, bdf_len); + fd = open("/sys/kernel/debug/powerpc/eeh_dev_break", O_WRONLY); + write(fd, bdf, bdf_len); + close(fd); +} + +int main(int argc, char **argv) +{ + int i, ret, container, group, device, groupid; + char path[PATH_MAX], c; + int seg, bus, dev, func; + uint32_t phys_val =0, val; + __u64 offset; + ssize_t pret; + uint32_t mask= 0b0100; + unsigned int buf[512]; + + struct vfio_group_status group_status = { + .argsz = sizeof(group_status) + }; + + struct vfio_device_info device_info = { + .argsz = sizeof(device_info) + }; + + struct vfio_region_info region_info = { + .argsz = sizeof(region_info) + }; + + struct vfio_eeh_pe_op pe_op = { + .argsz = sizeof(pe_op), + .flags = 0 + }; + + + if (argc < 3) { + usage(argv[0]); + return -1; + } + + ret = sscanf(argv[1], "%d", &groupid); + if (ret != 1) { + usage(argv[0]); + return -1; + } + + ret = sscanf(argv[2], "%04x:%02x:%02x.%d", &seg, &bus, &dev, &func); + if (ret != 4) { + usage(argv[0]); + return -1; + } + + printf("Using PCI device %04x:%02x:%02x.%d in group %d\n", + seg, bus, dev, func, groupid); + + container = open("/dev/vfio/vfio", O_RDWR); + if (container < 0) { + printf("Failed to open /dev/vfio/vfio, %d (%s)\n", + container, strerror(errno)); + return container; + } + + snprintf(path, sizeof(path), "/dev/vfio/%d", groupid); + group = open(path, O_RDWR); + if (group < 0) { + printf("Failed to open %s, %d (%s)\n", + path, group, strerror(errno)); + return group; + } + + ret = ioctl(group, VFIO_GROUP_GET_STATUS, &group_status); + if (ret) { + printf("ioctl(VFIO_GROUP_GET_STATUS) failed\n"); + return ret; + } + + if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) { + printf("Group not viable, are all devices attached to vfio?\n"); + return -1; + } + + printf("pre-SET_CONTAINER:\n"); + +#ifndef __PPC64__ + printf("VFIO_CHECK_EXTENSION VFIO_TYPE1_IOMMU: %sPresent\n", + ioctl(container, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) ? + "" : "Not "); + printf("VFIO_CHECK_EXTENSION VFIO_NOIOMMU_IOMMU: %sPresent\n", + ioctl(container, VFIO_CHECK_EXTENSION, VFIO_NOIOMMU_IOMMU) ? + "" : "Not "); +#endif + ret = ioctl(group, VFIO_GROUP_SET_CONTAINER, &container); + if (ret) { + printf("Failed to set group container\n"); + return ret; + } + + printf("post-SET_CONTAINER:\n"); +#ifndef __PPC64__ + printf("VFIO_CHECK_EXTENSION VFIO_TYPE1_IOMMU: %sPresent\n", + ioctl(container, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) ? + "" : "Not "); + printf("VFIO_CHECK_EXTENSION VFIO_NOIOMMU_IOMMU: %sPresent\n", + ioctl(container, VFIO_CHECK_EXTENSION, VFIO_NOIOMMU_IOMMU) ? + "" : "Not "); + + ret = ioctl(container, VFIO_SET_IOMMU, VFIO_NOIOMMU_IOMMU); + if (!ret) { + printf("Incorrectly allowed no-iommu usage!\n"); + return -1; + } + + ret = ioctl(container, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU); + if (ret) { + printf("Failed to set IOMMU\n"); + return ret; + } +#else + printf("VFIO_CHECK_EXTENSION VFIO_SPAPR_TCE_v2_IOMMU: %s Present\n", + ioctl(container, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_v2_IOMMU) ? + "" : "Not "); + + + ret = ioctl(container, VFIO_SET_IOMMU, VFIO_SPAPR_TCE_v2_IOMMU); + if (ret) { + printf("Failed to set IOMMU\n"); + return ret; + } + printf("VFIO_SET_IOMMU to VFIO_SPAPR_TCE_v2_IOMMU successfull\n"); + +#endif + + snprintf(path, sizeof(path), "%04x:%02x:%02x.%d", seg, bus, dev, func); + + device = ioctl(group, VFIO_GROUP_GET_DEVICE_FD, path); + if (device < 0) { + printf("Failed to get device %s\n", path); + return -1; + } + + if (ioctl(device, VFIO_DEVICE_GET_INFO, &device_info)) { + printf("Failed to get device info\n"); + return -1; + } + + printf("Device supports %d regions, %d irqs\n", + device_info.num_regions, device_info.num_irqs); + + for (i = 0; i < device_info.num_regions; i++) { + printf("Region %d: ", i); + region_info.index = i; + pcidev.regs[i].argsz = sizeof(struct vfio_region_info); + pcidev.regs[i].index = i; + if (ioctl(device, VFIO_DEVICE_GET_REGION_INFO, &pcidev.regs[i])) { + printf("Failed to get info\n"); + continue; + } + + printf("size 0x%lx, offset 0x%lx, flags 0b%b\n", + (unsigned long)pcidev.regs[i].size, + (unsigned long)pcidev.regs[i].offset, region_info.flags); + if (pcidev.regs[i].flags & VFIO_REGION_INFO_FLAG_MMAP) { + void *map = mmap(NULL, (size_t)pcidev.regs[i].size, + PROT_READ, MAP_SHARED, device, + (off_t)pcidev.regs[i].offset); + if (map == MAP_FAILED) { + printf("mmap failed\n"); + continue; + } + + printf("["); + fwrite(map, 1, pcidev.regs[i].size > 16 ? 16 : + pcidev.regs[i].size, stdout); + printf("]\n"); + munmap(map, (size_t)pcidev.regs[i].size); + } + if (i==0) { + offset = pcidev.regs[i].offset+4; + } + } + + /* MMIO mapings */ + bar_info = &pcidev.regs[VFIO_PCI_BAR0_REGION_INDEX]; + pcidev.mmio_addr = mmap(NULL, bar_info->size, PROT_READ | PROT_WRITE, + MAP_SHARED, device, bar_info->offset); + + write_u32(&pcidev, 0x100c, 0x1); + printf("CAP %x \n",read_u32(&pcidev, CAP)); + printf("VS %x \n",read_u32(&pcidev, VS)); + printf("CC %x \n",read_u32(&pcidev, CC)); + printf("CSTS %x \n",read_u32(&pcidev, CSTS)); + printf("AQA %x \n",read_u32(&pcidev, AQA)); + printf("ASQ %x \n",read_u32(&pcidev, ASQ)); + printf("ACQ %x \n",read_u32(&pcidev, ACQ)); + printf("S_DB %x \n",read_u32(&pcidev, S_DB)); + printf("C_DB %x \n",read_u32(&pcidev, C_DB)); + printf("S_DB_Q1 %x \n",read_u32(&pcidev,S_DB_Q1)); + printf("C_DB_Q1 %x \n",read_u32(&pcidev,C_DB_Q1 )); + + /* Make sure EEH is supported */ + ret = ioctl(container, VFIO_CHECK_EXTENSION, VFIO_EEH); + if(ret < 0) { + printf("EEH not supported %d\n",errno); + perror(""); + exit(0); + } + /* Enable the EEH functionality on the device */ + pe_op.op = VFIO_EEH_PE_ENABLE; + if(ioctl(container, VFIO_EEH_PE_OP, &pe_op) ==-1) { + printf("EEH enable on device failed\n"); + perror(""); + } + /* Check the PE's state and make sure it's in functional state */ + pe_op.op = VFIO_EEH_PE_GET_STATE; + if(ioctl(container, VFIO_EEH_PE_OP, &pe_op) ==-1) { + perror(""); + + } + //pci read config + printf("\ndump configuration space registers\n"); + pret = pread(device, buf, 512, pcidev.regs[VFIO_PCI_CONFIG_REGION_INDEX].offset); + if(pret < 0) + { + perror("read onfig: "); + } + else + { + for(i=0; i<10; i++) + printf("%x \n",buf[i]); + //printf("read config offset = %ld physical val 0x%X\n", offset, phys_val); + c = getc(stdin); + } + + + /* Inject EEH error, which is expected to be caused by 32-bits + * config load. + */ + pe_op.op = VFIO_EEH_PE_INJECT_ERR; + pe_op.err.type = EEH_ERR_TYPE_32; + pe_op.err.func = EEH_ERR_FUNC_LD_CFG_ADDR; + pe_op.err.addr = 0ul; + pe_op.err.mask = 0ul; + pret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); + if(pret < 0) { + perror(""); + char buf[]="0028:01:00.0"; + int fd; + int len = sizeof(buf); + printf("buf out : %s %d\n", buf, len); + //fd = open("/sys/kernel/debug/powerpc/eeh_dev_break", O_WRONLY); + //write(fd, &buf, (len-1)); + //close(fd); + eeh_dev_break_debugfs(buf, len); + } + + + pe_op.op = VFIO_EEH_PE_GET_STATE; + ret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); + if(ret < 0) { + perror(""); + } + printf("VFIO_EEH_PE_STATE after injectiong error %x ", ret); + c = getc(stdin); + + /* When 0xFF's returned from reading PCI config space or IO BARs + * of the PCI device. Check the PE's state to see if that has been + * frozen. + */ + //write_u32(&pcidev, 0x100c, 0x1); + //pci read config + printf("\ndump configuration space registers\n"); + pret = pread(device, buf, 512, pcidev.regs[VFIO_PCI_CONFIG_REGION_INDEX].offset); + if(pret < 0) + { + perror("read onfig: "); + } + else + { + for(i=0; i<10; i++) + printf("%x \n",buf[i]); + //printf("read config offset = %ld physical val 0x%X\n", offset, phys_val); + c = getc(stdin); + } + + //pci read config + pret = pread(device, &phys_val, 2,offset); + if(pret < 0) + { + perror("read onfig: "); + } + else + { + printf("read config offset = %ld physical val 0x%X\n", offset, phys_val); + c = getc(stdin); + } + /* read MMIO */ + printf("CAP %x \n",read_u32(&pcidev, CAP)); + printf("VS %x \n",read_u32(&pcidev, VS)); + printf("CC %x \n",read_u32(&pcidev, CC)); + printf("CSTS %x \n",read_u32(&pcidev, CSTS)); + printf("AQA %x \n",read_u32(&pcidev, AQA)); + printf("ASQ %x \n",read_u32(&pcidev, ASQ)); + printf("ACQ %x \n",read_u32(&pcidev, ACQ)); + printf("S_DB %x \n",read_u32(&pcidev, S_DB)); + printf("C_DB %x \n",read_u32(&pcidev, C_DB)); + printf("S_DB_Q1 %x \n",read_u32(&pcidev,S_DB_Q1)); + printf("C_DB_Q1 %x \n",read_u32(&pcidev,C_DB_Q1 )); + + pe_op.op = VFIO_EEH_PE_GET_STATE; + ret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); + if( ret < 0) { + perror(""); + } + printf("VFIO_EEH_PE_GET_STATE %d \n",ret); + /* Waiting for pending PCI transactions to be completed and don't + * produce any more PCI traffic from/to the affected PE until + * recovery is finished. + */ + + /* Enable IO for the affected PE and collect logs. Usually, the + * standard part of PCI config space, AER registers are dumped + * as logs for further analysis. + */ + printf("VFIO_EEH_PE_UNFREEZE_IO\n"); + fgetc(stdin); + pe_op.op = VFIO_EEH_PE_UNFREEZE_IO; + ioctl(container, VFIO_EEH_PE_OP, &pe_op); + + /* + * Issue PE reset: hot or fundamental reset. Usually, hot reset + * is enough. However, the firmware of some PCI adapters would + * require fundamental reset. + */ + printf("Press any key to exit\n"); + fgetc(stdin); + pe_op.op = VFIO_EEH_PE_RESET_HOT; + ioctl(container, VFIO_EEH_PE_OP, &pe_op); + pe_op.op = VFIO_EEH_PE_RESET_DEACTIVATE; + ioctl(container, VFIO_EEH_PE_OP, &pe_op); + + printf("Press any key to exit\n"); + fgetc(stdin); + /* Configure the PCI bridges for the affected PE */ + pe_op.op = VFIO_EEH_PE_CONFIGURE; + ioctl(container, VFIO_EEH_PE_OP, &pe_op); + + + printf("Success\n"); + printf("Press any key to exit\n"); + fgetc(stdin); + + return 0; +} diff --git a/vfio-spapr-eeh-test2.c b/vfio-spapr-eeh-test2.c new file mode 100644 index 0000000..2f2c77e --- /dev/null +++ b/vfio-spapr-eeh-test2.c @@ -0,0 +1,433 @@ + +#include +#include +#include + +#define VFIO_API_VERSION 0 + + +/* Kernel & User level defines for VFIO IOCTLs. */ + +/* Extensions */ + +#define VFIO_TYPE1_IOMMU 1 +#define VFIO_TYPE1v2_IOMMU 3 +#define VFIO_SPAPR_TCE_v2_IOMMU 7 +#define VFIO_NOIOMMU_IOMMU 8 + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#define CAP 0x0 +#define VS 0x8 +#define CC 0x14 +#define CSTS 0x1c +#define AQA 0x24 +#define ASQ 0x28 +#define ACQ 0x30 +#define S_DB 0x1000 +#define C_DB 0x1004 +#define S_DB_Q1 0x1008 +#define C_DB_Q1 0x100c + + +void usage(char *name) +{ + printf("usage: %s \n", name); +} + + + + struct device_t { + struct vfio_region_info regs[VFIO_PCI_NUM_REGIONS]; + struct vfio_irq_info irqs[VFIO_PCI_NUM_IRQS]; + void* mmio_addr; // mmio address (BAR0); + } pcidev; + + struct vfio_region_info* bar_info; + + static inline void write_u32(struct device_t* dev, int offset, uint32_t value) { + __asm__ volatile("" : : : "memory"); + *((volatile uint32_t*)(dev->mmio_addr + offset)) = value; + } + + static inline uint64_t read_u32(struct device_t* dev, int offset) { + __asm__ volatile("" : : : "memory"); + return *((volatile uint64_t*)(dev->mmio_addr + offset)); + } +static void eeh_dev_break_debugfs(char *bdf, int bdf_len) +{ + int fd; + printf("buf out : %s %d\n", bdf, bdf_len); + fd = open("/sys/kernel/debug/powerpc/eeh_dev_break", O_WRONLY); + write(fd, bdf, bdf_len); + close(fd); +} + +int main(int argc, char **argv) +{ + int i, ret, container, group, device, groupid; + char path[PATH_MAX], c; + int seg, bus, dev, func; + uint32_t phys_val =0, val; + __u64 offset; + ssize_t pret; + uint32_t mask= 0b0100; + unsigned int buf[512]; + char bdf_buf[100]; + + struct vfio_group_status group_status = { + .argsz = sizeof(group_status) + }; + + struct vfio_device_info device_info = { + .argsz = sizeof(device_info) + }; + + struct vfio_region_info region_info = { + .argsz = sizeof(region_info) + }; + + struct vfio_eeh_pe_op pe_op = { + .argsz = sizeof(pe_op), + .flags = 0 + }; + + + if (argc < 3) { + usage(argv[0]); + return -1; + } + + ret = sscanf(argv[1], "%d", &groupid); + if (ret != 1) { + usage(argv[0]); + return -1; + } + + ret = sscanf(argv[2], "%04x:%02x:%02x.%d", &seg, &bus, &dev, &func); + if (ret != 4) { + usage(argv[0]); + return -1; + } + + printf("Using PCI device %04x:%02x:%02x.%d in group %d\n", + seg, bus, dev, func, groupid); + sprintf(bdf_buf,"%04x:%02x:%02x.%d",seg, bus, dev, func); + printf (" bdf_buf = %s - %d\n ", bdf_buf, strlen(bdf_buf)); + + container = open("/dev/vfio/vfio", O_RDWR); + if (container < 0) { + printf("Failed to open /dev/vfio/vfio, %d (%s)\n", + container, strerror(errno)); + return container; + } + + snprintf(path, sizeof(path), "/dev/vfio/%d", groupid); + group = open(path, O_RDWR); + if (group < 0) { + printf("Failed to open %s, %d (%s)\n", + path, group, strerror(errno)); + return group; + } + + ret = ioctl(group, VFIO_GROUP_GET_STATUS, &group_status); + if (ret) { + printf("ioctl(VFIO_GROUP_GET_STATUS) failed\n"); + return ret; + } + + if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) { + printf("Group not viable, are all devices attached to vfio?\n"); + return -1; + } + + printf("pre-SET_CONTAINER:\n"); + +#ifndef __PPC64__ + printf("VFIO_CHECK_EXTENSION VFIO_TYPE1_IOMMU: %sPresent\n", + ioctl(container, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) ? + "" : "Not "); + printf("VFIO_CHECK_EXTENSION VFIO_NOIOMMU_IOMMU: %sPresent\n", + ioctl(container, VFIO_CHECK_EXTENSION, VFIO_NOIOMMU_IOMMU) ? + "" : "Not "); +#endif + ret = ioctl(group, VFIO_GROUP_SET_CONTAINER, &container); + if (ret) { + printf("Failed to set group container\n"); + return ret; + } + + printf("post-SET_CONTAINER:\n"); +#ifndef __PPC64__ + printf("VFIO_CHECK_EXTENSION VFIO_TYPE1_IOMMU: %sPresent\n", + ioctl(container, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) ? + "" : "Not "); + printf("VFIO_CHECK_EXTENSION VFIO_NOIOMMU_IOMMU: %sPresent\n", + ioctl(container, VFIO_CHECK_EXTENSION, VFIO_NOIOMMU_IOMMU) ? + "" : "Not "); + + ret = ioctl(container, VFIO_SET_IOMMU, VFIO_NOIOMMU_IOMMU); + if (!ret) { + printf("Incorrectly allowed no-iommu usage!\n"); + return -1; + } + + ret = ioctl(container, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU); + if (ret) { + printf("Failed to set IOMMU\n"); + return ret; + } +#else + printf("VFIO_CHECK_EXTENSION VFIO_SPAPR_TCE_v2_IOMMU: %s Present\n", + ioctl(container, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_v2_IOMMU) ? + "" : "Not "); + + + ret = ioctl(container, VFIO_SET_IOMMU, VFIO_SPAPR_TCE_v2_IOMMU); + if (ret) { + printf("Failed to set IOMMU\n"); + return ret; + } + printf("VFIO_SET_IOMMU to VFIO_SPAPR_TCE_v2_IOMMU successfull\n"); + +#endif + + snprintf(path, sizeof(path), "%04x:%02x:%02x.%d", seg, bus, dev, func); + + device = ioctl(group, VFIO_GROUP_GET_DEVICE_FD, path); + if (device < 0) { + printf("Failed to get device %s\n", path); + return -1; + } + + if (ioctl(device, VFIO_DEVICE_GET_INFO, &device_info)) { + printf("Failed to get device info\n"); + return -1; + } + + printf("Device supports %d regions, %d irqs\n", + device_info.num_regions, device_info.num_irqs); + + for (i = 0; i < device_info.num_regions; i++) { + printf("Region %d: ", i); + region_info.index = i; + pcidev.regs[i].argsz = sizeof(struct vfio_region_info); + pcidev.regs[i].index = i; + if (ioctl(device, VFIO_DEVICE_GET_REGION_INFO, &pcidev.regs[i])) { + printf("Failed to get info\n"); + continue; + } + + printf("size 0x%lx, offset 0x%lx, flags 0x%x , regions.flags 0b%b\n", + (unsigned long)pcidev.regs[i].size, + (unsigned long)pcidev.regs[i].offset, region_info.flags, + pcidev.regs[i].flags); + if (pcidev.regs[i].flags & VFIO_REGION_INFO_FLAG_MMAP) { + void *map = mmap(NULL, (size_t)pcidev.regs[i].size, + PROT_READ, MAP_SHARED, device, + (off_t)pcidev.regs[i].offset); + if (map == MAP_FAILED) { + printf("mmap failed\n"); + continue; + } + + munmap(map, (size_t)pcidev.regs[i].size); + } + if( i==0) { + offset = pcidev.regs[VFIO_PCI_BAR0_REGION_INDEX].offset+4; + } + } + //pci read config + pret = pread(device, &phys_val, 2,pcidev.regs[VFIO_PCI_CONFIG_REGION_INDEX].offset); + if(pret < 0) + { + perror("read onfig: "); + } + else + { + printf("read config offset = %ld physical val 0x%X\n", offset, phys_val); + c = getc(stdin); + } + /* MMIO mapings */ + bar_info = &pcidev.regs[VFIO_PCI_BAR0_REGION_INDEX]; + pcidev.mmio_addr = mmap(NULL, bar_info->size, PROT_READ , + MAP_SHARED, device, bar_info->offset); + + /* printf("CAP %x \n",read_u32(&pcidev, CAP)); + printf("VS %x \n",read_u32(&pcidev, VS)); + printf("CC %x \n",read_u32(&pcidev, CC)); + printf("CSTS %x \n",read_u32(&pcidev, CSTS)); + printf("AQA %x \n",read_u32(&pcidev, AQA)); + printf("ASQ %x \n",read_u32(&pcidev, ASQ)); + printf("ACQ %x \n",read_u32(&pcidev, ACQ)); + printf("S_DB %x \n",read_u32(&pcidev, S_DB)); + printf("C_DB %x \n",read_u32(&pcidev, C_DB)); + printf("S_DB_Q1 %x \n",read_u32(&pcidev,S_DB_Q1)); + printf("C_DB_Q1 %x \n",read_u32(&pcidev,C_DB_Q1 )); +*/ + /* Make sure EEH is supported */ + ret = ioctl(container, VFIO_CHECK_EXTENSION, VFIO_EEH); + if(ret < 0) { + printf("EEH not supported %d\n",errno); + perror(""); + exit(0); + } + /* Enable the EEH functionality on the device */ + pe_op.op = VFIO_EEH_PE_ENABLE; + if(ioctl(container, VFIO_EEH_PE_OP, &pe_op) ==-1) { + printf("EEH enable on device failed\n"); + perror(""); + } + /* Check the PE's state and make sure it's in functional state */ + pe_op.op = VFIO_EEH_PE_GET_STATE; + if(ioctl(container, VFIO_EEH_PE_OP, &pe_op) ==-1) { + perror(""); + + } + //pci read config + printf("\ndump configuration space registers\n"); + pret = pread(device, buf, 512, pcidev.regs[VFIO_PCI_CONFIG_REGION_INDEX].offset); + if(pret < 0) + { + perror("read onfig: "); + } + else + { + for(i=0; i<10; i++) + printf("%x \n",buf[i]); + //printf("read config offset = %ld physical val 0x%X\n", offset, phys_val); + c = getc(stdin); + } + + + /* Inject EEH error, which is expected to be caused by 32-bits + * config load. + */ + pe_op.op = VFIO_EEH_PE_INJECT_ERR; + pe_op.err.type = EEH_ERR_TYPE_32; + pe_op.err.func = EEH_ERR_FUNC_LD_CFG_ADDR; + pe_op.err.addr = 0ul; + pe_op.err.mask = 0ul; + pret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); + if(pret < 0) { + perror(""); + eeh_dev_break_debugfs(bdf_buf, strlen(bdf_buf)); + } + + + pe_op.op = VFIO_EEH_PE_GET_STATE; + ret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); + if(ret < 0) { + perror(""); + } + printf("VFIO_EEH_PE_STATE after injectiong error %x ", ret); + c = getc(stdin); + + /* When 0xFF's returned from reading PCI config space or IO BARs + * of the PCI device. Check the PE's state to see if that has been + * frozen. + */ + //write_u32(&pcidev, 0x100c, 0x1); + //pci read config + printf("\ndump configuration space registers\n"); + pret = pread(device, buf, 512, pcidev.regs[VFIO_PCI_CONFIG_REGION_INDEX].offset); + if(pret < 0) + { + perror("read onfig: "); + } + else + { + for(i=0; i<10; i++) + printf("%x \n",buf[i]); + //printf("read config offset = %ld physical val 0x%X\n", offset, phys_val); + c = getc(stdin); + } + + //pci read config + pret = pread(device, &phys_val, 2,pcidev.regs[7].offset); + if(pret < 0) + { + perror("read onfig: "); + } + else + { + printf("read config offset = %ld physical val 0x%X\n", offset, phys_val); + c = getc(stdin); + } + /* read MMIO */ + /* printf("CAP %x \n",read_u32(&pcidev, CAP)); + printf("VS %x \n",read_u32(&pcidev, VS)); + printf("CC %x \n",read_u32(&pcidev, CC)); + printf("CSTS %x \n",read_u32(&pcidev, CSTS)); + printf("AQA %x \n",read_u32(&pcidev, AQA)); + printf("ASQ %x \n",read_u32(&pcidev, ASQ)); + printf("ACQ %x \n",read_u32(&pcidev, ACQ)); + printf("S_DB %x \n",read_u32(&pcidev, S_DB)); + printf("C_DB %x \n",read_u32(&pcidev, C_DB)); + printf("S_DB_Q1 %x \n",read_u32(&pcidev,S_DB_Q1)); + printf("C_DB_Q1 %x \n",read_u32(&pcidev,C_DB_Q1 )); +*/ + pe_op.op = VFIO_EEH_PE_GET_STATE; + ret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); + if( ret < 0) { + perror(""); + } + printf("VFIO_EEH_PE_GET_STATE %d \n",ret); + /* Waiting for pending PCI transactions to be completed and don't + * produce any more PCI traffic from/to the affected PE until + * recovery is finished. + */ + + /* Enable IO for the affected PE and collect logs. Usually, the + * standard part of PCI config space, AER registers are dumped + * as logs for further analysis. + */ + printf("VFIO_EEH_PE_UNFREEZE_IO\n"); + fgetc(stdin); + pe_op.op = VFIO_EEH_PE_UNFREEZE_IO; + ioctl(container, VFIO_EEH_PE_OP, &pe_op); + + /* + * Issue PE reset: hot or fundamental reset. Usually, hot reset + * is enough. However, the firmware of some PCI adapters would + * require fundamental reset. + */ + printf("Press any key to exit\n"); + fgetc(stdin); + pe_op.op = VFIO_EEH_PE_RESET_HOT; + ioctl(container, VFIO_EEH_PE_OP, &pe_op); + pe_op.op = VFIO_EEH_PE_RESET_DEACTIVATE; + ioctl(container, VFIO_EEH_PE_OP, &pe_op); + + printf("Press any key to exit\n"); + fgetc(stdin); + /* Configure the PCI bridges for the affected PE */ + pe_op.op = VFIO_EEH_PE_CONFIGURE; + ioctl(container, VFIO_EEH_PE_OP, &pe_op); + + + printf("Success\n"); + printf("Press any key to exit\n"); + fgetc(stdin); + + return 0; +} From e81a8b57fae2b1785f170737a7748fb368c3589e Mon Sep 17 00:00:00 2001 From: Narayana Murty N Date: Wed, 3 Jul 2024 09:30:09 -0500 Subject: [PATCH 08/12] mmio generic read --- vfio-spapr-eeh-test.c | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/vfio-spapr-eeh-test.c b/vfio-spapr-eeh-test.c index 2e2c208..5f50060 100644 --- a/vfio-spapr-eeh-test.c +++ b/vfio-spapr-eeh-test.c @@ -104,10 +104,10 @@ int main(int argc, char **argv) struct vfio_region_info region_info = { .argsz = sizeof(region_info) }; - - struct vfio_eeh_pe_op pe_op = { - .argsz = sizeof(pe_op), - .flags = 0 + + struct vfio_eeh_pe_op pe_op = { + .argsz = sizeof(pe_op), + .flags = 0 }; @@ -262,7 +262,10 @@ int main(int argc, char **argv) pcidev.mmio_addr = mmap(NULL, bar_info->size, PROT_READ | PROT_WRITE, MAP_SHARED, device, bar_info->offset); - write_u32(&pcidev, 0x100c, 0x1); + /* read MMIO */ + for(i = 0; i < 0x10; i+=0x4 ) + printf("MMIO register offset 0x%X value 0x%X \n",i, read_u32(&pcidev, i)); + /*write_u32(&pcidev, 0x100c, 0x1); printf("CAP %x \n",read_u32(&pcidev, CAP)); printf("VS %x \n",read_u32(&pcidev, VS)); printf("CC %x \n",read_u32(&pcidev, CC)); @@ -273,7 +276,7 @@ int main(int argc, char **argv) printf("S_DB %x \n",read_u32(&pcidev, S_DB)); printf("C_DB %x \n",read_u32(&pcidev, C_DB)); printf("S_DB_Q1 %x \n",read_u32(&pcidev,S_DB_Q1)); - printf("C_DB_Q1 %x \n",read_u32(&pcidev,C_DB_Q1 )); + printf("C_DB_Q1 %x \n",read_u32(&pcidev,C_DB_Q1 )); */ /* Make sure EEH is supported */ ret = ioctl(container, VFIO_CHECK_EXTENSION, VFIO_EEH); @@ -330,7 +333,7 @@ int main(int argc, char **argv) //close(fd); eeh_dev_break_debugfs(buf, len); } - + pe_op.op = VFIO_EEH_PE_GET_STATE; ret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); @@ -372,17 +375,8 @@ int main(int argc, char **argv) c = getc(stdin); } /* read MMIO */ - printf("CAP %x \n",read_u32(&pcidev, CAP)); - printf("VS %x \n",read_u32(&pcidev, VS)); - printf("CC %x \n",read_u32(&pcidev, CC)); - printf("CSTS %x \n",read_u32(&pcidev, CSTS)); - printf("AQA %x \n",read_u32(&pcidev, AQA)); - printf("ASQ %x \n",read_u32(&pcidev, ASQ)); - printf("ACQ %x \n",read_u32(&pcidev, ACQ)); - printf("S_DB %x \n",read_u32(&pcidev, S_DB)); - printf("C_DB %x \n",read_u32(&pcidev, C_DB)); - printf("S_DB_Q1 %x \n",read_u32(&pcidev,S_DB_Q1)); - printf("C_DB_Q1 %x \n",read_u32(&pcidev,C_DB_Q1 )); + for(i = 0; i < 0x10; i+=0x4 ) + printf("MMIO register offset 0x%X value 0x%X \n",i, read_u32(&pcidev, i)); pe_op.op = VFIO_EEH_PE_GET_STATE; ret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); From 9fccfb08977773021c42ef656db1341d33216629 Mon Sep 17 00:00:00 2001 From: Narayana Murty N Date: Fri, 16 Aug 2024 19:44:30 -0500 Subject: [PATCH 09/12] Adding EEH testing support for vfio This patch adds vfio-spapr-eeh-test.c test case to test EEH process form VFIO userspace. It dumps the pci config space and few 32 bit MMIO registers before and after injectiong error. It uses VFIO_EEH_PE_RESET_HOT to recover from EEH state and enables the device by calling VFIO_EEH_PE_CONFIGURE. --- vfio-spapr-eeh-test.c | 122 +++++++++++------------------------------- 1 file changed, 30 insertions(+), 92 deletions(-) diff --git a/vfio-spapr-eeh-test.c b/vfio-spapr-eeh-test.c index 5f50060..3c098d5 100644 --- a/vfio-spapr-eeh-test.c +++ b/vfio-spapr-eeh-test.c @@ -36,18 +36,6 @@ #include #include -#define CAP 0x0 -#define VS 0x8 -#define CC 0x14 -#define CSTS 0x1c -#define AQA 0x24 -#define ASQ 0x28 -#define ACQ 0x30 -#define S_DB 0x1000 -#define C_DB 0x1004 -#define S_DB_Q1 0x1008 -#define C_DB_Q1 0x100c - void usage(char *name) { @@ -55,31 +43,22 @@ void usage(char *name) } - - struct device_t { - struct vfio_region_info regs[VFIO_PCI_NUM_REGIONS]; - struct vfio_irq_info irqs[VFIO_PCI_NUM_IRQS]; +struct device_t { + struct vfio_region_info regs[VFIO_PCI_NUM_REGIONS]; + struct vfio_irq_info irqs[VFIO_PCI_NUM_IRQS]; void* mmio_addr; // mmio address (BAR0); - } pcidev; +} pcidev; - struct vfio_region_info* bar_info; +struct vfio_region_info* bar_info; - static inline void write_u32(struct device_t* dev, int offset, uint32_t value) { - __asm__ volatile("" : : : "memory"); - *((volatile uint32_t*)(dev->mmio_addr + offset)) = value; - } +static inline void write_u32(struct device_t* dev, int offset, uint32_t value) { + __asm__ volatile("" : : : "memory"); + *((volatile uint32_t*)(dev->mmio_addr + offset)) = value; +} - static inline uint32_t read_u32(struct device_t* dev, int offset) { - __asm__ volatile("" : : : "memory"); - return *((volatile uint32_t*)(dev->mmio_addr + offset)); - } -static void eeh_dev_break_debugfs(char *bdf, int bdf_len) -{ - int fd; - printf("buf out : %s %d\n", bdf, bdf_len); - fd = open("/sys/kernel/debug/powerpc/eeh_dev_break", O_WRONLY); - write(fd, bdf, bdf_len); - close(fd); +static inline uint32_t read_u32(struct device_t* dev, int offset) { + __asm__ volatile("" : : : "memory"); + return *((volatile uint32_t*)(dev->mmio_addr + offset)); } int main(int argc, char **argv) @@ -262,22 +241,6 @@ int main(int argc, char **argv) pcidev.mmio_addr = mmap(NULL, bar_info->size, PROT_READ | PROT_WRITE, MAP_SHARED, device, bar_info->offset); - /* read MMIO */ - for(i = 0; i < 0x10; i+=0x4 ) - printf("MMIO register offset 0x%X value 0x%X \n",i, read_u32(&pcidev, i)); - /*write_u32(&pcidev, 0x100c, 0x1); - printf("CAP %x \n",read_u32(&pcidev, CAP)); - printf("VS %x \n",read_u32(&pcidev, VS)); - printf("CC %x \n",read_u32(&pcidev, CC)); - printf("CSTS %x \n",read_u32(&pcidev, CSTS)); - printf("AQA %x \n",read_u32(&pcidev, AQA)); - printf("ASQ %x \n",read_u32(&pcidev, ASQ)); - printf("ACQ %x \n",read_u32(&pcidev, ACQ)); - printf("S_DB %x \n",read_u32(&pcidev, S_DB)); - printf("C_DB %x \n",read_u32(&pcidev, C_DB)); - printf("S_DB_Q1 %x \n",read_u32(&pcidev,S_DB_Q1)); - printf("C_DB_Q1 %x \n",read_u32(&pcidev,C_DB_Q1 )); */ - /* Make sure EEH is supported */ ret = ioctl(container, VFIO_CHECK_EXTENSION, VFIO_EEH); if(ret < 0) { @@ -291,12 +254,7 @@ int main(int argc, char **argv) printf("EEH enable on device failed\n"); perror(""); } - /* Check the PE's state and make sure it's in functional state */ - pe_op.op = VFIO_EEH_PE_GET_STATE; - if(ioctl(container, VFIO_EEH_PE_OP, &pe_op) ==-1) { - perror(""); - } //pci read config printf("\ndump configuration space registers\n"); pret = pread(device, buf, 512, pcidev.regs[VFIO_PCI_CONFIG_REGION_INDEX].offset); @@ -308,9 +266,12 @@ int main(int argc, char **argv) { for(i=0; i<10; i++) printf("%x \n",buf[i]); - //printf("read config offset = %ld physical val 0x%X\n", offset, phys_val); - c = getc(stdin); } + + /* dump some 32bit MMIO registers */ + for(i = 0; i < 0x10; i+=0x4 ) + printf("MMIO register offset 0x%X value 0x%X \n",i, read_u32(&pcidev, i)); + /* Inject EEH error, which is expected to be caused by 32-bits @@ -318,20 +279,14 @@ int main(int argc, char **argv) */ pe_op.op = VFIO_EEH_PE_INJECT_ERR; pe_op.err.type = EEH_ERR_TYPE_32; - pe_op.err.func = EEH_ERR_FUNC_LD_CFG_ADDR; + pe_op.err.func = EEH_ERR_FUNC_LD_MEM_ADDR; pe_op.err.addr = 0ul; pe_op.err.mask = 0ul; pret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); if(pret < 0) { perror(""); - char buf[]="0028:01:00.0"; - int fd; - int len = sizeof(buf); - printf("buf out : %s %d\n", buf, len); - //fd = open("/sys/kernel/debug/powerpc/eeh_dev_break", O_WRONLY); - //write(fd, &buf, (len-1)); - //close(fd); - eeh_dev_break_debugfs(buf, len); + printf("VFIO_EEH_PE_INJECT_ERR failed , buf out : %s \n", path); + return pret; } @@ -341,13 +296,11 @@ int main(int argc, char **argv) perror(""); } printf("VFIO_EEH_PE_STATE after injectiong error %x ", ret); - c = getc(stdin); /* When 0xFF's returned from reading PCI config space or IO BARs * of the PCI device. Check the PE's state to see if that has been * frozen. */ - //write_u32(&pcidev, 0x100c, 0x1); //pci read config printf("\ndump configuration space registers\n"); pret = pread(device, buf, 512, pcidev.regs[VFIO_PCI_CONFIG_REGION_INDEX].offset); @@ -358,22 +311,9 @@ int main(int argc, char **argv) else { for(i=0; i<10; i++) - printf("%x \n",buf[i]); - //printf("read config offset = %ld physical val 0x%X\n", offset, phys_val); - c = getc(stdin); - } - - //pci read config - pret = pread(device, &phys_val, 2,offset); - if(pret < 0) - { - perror("read onfig: "); - } - else - { - printf("read config offset = %ld physical val 0x%X\n", offset, phys_val); - c = getc(stdin); + printf("%d. 0x%X \n",i,buf[i]); } + /* read MMIO */ for(i = 0; i < 0x10; i+=0x4 ) printf("MMIO register offset 0x%X value 0x%X \n",i, read_u32(&pcidev, i)); @@ -393,32 +333,30 @@ int main(int argc, char **argv) * standard part of PCI config space, AER registers are dumped * as logs for further analysis. */ - printf("VFIO_EEH_PE_UNFREEZE_IO\n"); - fgetc(stdin); pe_op.op = VFIO_EEH_PE_UNFREEZE_IO; ioctl(container, VFIO_EEH_PE_OP, &pe_op); + printf("VFIO_EEH_PE_UNFREEZE_IO\n"); /* * Issue PE reset: hot or fundamental reset. Usually, hot reset * is enough. However, the firmware of some PCI adapters would * require fundamental reset. */ - printf("Press any key to exit\n"); - fgetc(stdin); pe_op.op = VFIO_EEH_PE_RESET_HOT; ioctl(container, VFIO_EEH_PE_OP, &pe_op); pe_op.op = VFIO_EEH_PE_RESET_DEACTIVATE; ioctl(container, VFIO_EEH_PE_OP, &pe_op); - printf("Press any key to exit\n"); - fgetc(stdin); /* Configure the PCI bridges for the affected PE */ pe_op.op = VFIO_EEH_PE_CONFIGURE; - ioctl(container, VFIO_EEH_PE_OP, &pe_op); - + ret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); + if(!ret) { + printf("Success\n"); + printf("Press any key to exit\n"); + } else { + printf("VFIO_EEH_PE_CONFIGURE failed \n"); + } - printf("Success\n"); - printf("Press any key to exit\n"); fgetc(stdin); return 0; From a37c083c7c65e59ad4f1c27c7cdc13b6fec6032a Mon Sep 17 00:00:00 2001 From: Narayana Murty N Date: Wed, 21 Aug 2024 04:52:21 -0500 Subject: [PATCH 10/12] Readme update --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index df83a9b..3a88187 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,14 @@ $gcc -o vfio-iommu-map-unmap vfio-iommu-map-unmap.c $gcc -o vfio-huge-guest-test vfio-huge-guest-test.c +$gcc -o vfio-spapr-eeh-test vfio-spapr-eeh-test.c + $ vfio-pci-device-open \ \ $ vfio-iommu-map-unmap ssss:bb:dd.f $ vfio-huge-guest-test \ +For EEH test: + +$ vfio-spapr-eeh-test \ \ From 724eb0f2963d83049cfc9e347f33bd91ee0340fe Mon Sep 17 00:00:00 2001 From: Narayana Murty N Date: Tue, 19 Nov 2024 04:09:46 -0600 Subject: [PATCH 11/12] Adding dealy after thr slot_reset --- vfio-spapr-eeh-test.c | 58 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/vfio-spapr-eeh-test.c b/vfio-spapr-eeh-test.c index 3c098d5..3b72926 100644 --- a/vfio-spapr-eeh-test.c +++ b/vfio-spapr-eeh-test.c @@ -255,6 +255,14 @@ int main(int argc, char **argv) perror(""); } + /* Check the PE state */ + pe_op.op = VFIO_EEH_PE_GET_STATE; + ret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); + if(ret < 0) { + perror(""); + } + printf("VFIO_EEH_PE_STATE initial state of PE %x ", ret); + //pci read config printf("\ndump configuration space registers\n"); pret = pread(device, buf, 512, pcidev.regs[VFIO_PCI_CONFIG_REGION_INDEX].offset); @@ -264,8 +272,8 @@ int main(int argc, char **argv) } else { - for(i=0; i<10; i++) - printf("%x \n",buf[i]); + for(i=0; i<10; i++) + printf("%d. 0x%X \n",i,buf[i]); } /* dump some 32bit MMIO registers */ @@ -282,6 +290,7 @@ int main(int argc, char **argv) pe_op.err.func = EEH_ERR_FUNC_LD_MEM_ADDR; pe_op.err.addr = 0ul; pe_op.err.mask = 0ul; + pret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); if(pret < 0) { perror(""); @@ -289,6 +298,8 @@ int main(int argc, char **argv) return pret; } + printf("inject error \n"); + fgetc(stdin); pe_op.op = VFIO_EEH_PE_GET_STATE; ret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); @@ -344,6 +355,21 @@ int main(int argc, char **argv) */ pe_op.op = VFIO_EEH_PE_RESET_HOT; ioctl(container, VFIO_EEH_PE_OP, &pe_op); + printf("slot VFIO_EEH_PE_RESET_HOT initiated\n"); + pe_op.op = VFIO_EEH_PE_GET_STATE; + i = 5 ; + while (i--) + { + ret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); + if( ret < 0) { + perror(""); + } + printf("\t."); + sleep(5); + } + + printf("\nVFIO_EEH_PE_GET_STATE %d\n",ret); + pe_op.op = VFIO_EEH_PE_RESET_DEACTIVATE; ioctl(container, VFIO_EEH_PE_OP, &pe_op); @@ -352,12 +378,38 @@ int main(int argc, char **argv) ret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); if(!ret) { printf("Success\n"); - printf("Press any key to exit\n"); } else { printf("VFIO_EEH_PE_CONFIGURE failed \n"); } + /* verify the PE state in operational */ + pe_op.op = VFIO_EEH_PE_GET_STATE; + ret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); + if( ret < 0) { + perror(""); + } + printf("VFIO_EEH_PE_GET_STATE %d \n",ret); + + //pci read config + printf("\ndump configuration space registers\n"); + pret = pread(device, buf, 512, pcidev.regs[VFIO_PCI_CONFIG_REGION_INDEX].offset); + if(pret < 0) + { + perror("read onfig: "); + } + else + { + for(i=0; i<10; i++) + printf("%d. 0x%X \n",i,buf[i]); + } + + /* read MMIO */ + for(i = 0; i < 0x10; i+=0x4 ) + printf("MMIO register offset 0x%X value 0x%X \n",i, read_u32(&pcidev, i)); + + printf("Press any key to exit\n"); fgetc(stdin); + ioctl(device, VFIO_DEVICE_RESET); return 0; } From f641eec7bb4abe7b1a1631fe8e580ff3254ee017 Mon Sep 17 00:00:00 2001 From: Narayana Murty N Date: Sun, 24 Nov 2024 23:14:41 -0600 Subject: [PATCH 12/12] condinitional delay after VFIO_EEH_PE_RESET_HOT --- vfio-spapr-eeh-test.c | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/vfio-spapr-eeh-test.c b/vfio-spapr-eeh-test.c index 3b72926..bda33f2 100644 --- a/vfio-spapr-eeh-test.c +++ b/vfio-spapr-eeh-test.c @@ -15,6 +15,10 @@ #define VFIO_SPAPR_TCE_v2_IOMMU 7 #define VFIO_NOIOMMU_IOMMU 8 +#define EEH_STATE_MIN_WAIT_TIME (1000*1000) +#define EEH_STATE_MAX_WAIT_TIME (300*1000*1000) + +#define MIN(X, Y) (((X) < (Y)) ? (X) : (Y)) #include #include @@ -37,6 +41,7 @@ #include + void usage(char *name) { printf("usage: %s \n", name); @@ -71,6 +76,9 @@ int main(int argc, char **argv) ssize_t pret; uint32_t mask= 0b0100; unsigned int buf[512]; + int max_wait = EEH_STATE_MAX_WAIT_TIME; + int mwait = EEH_STATE_MIN_WAIT_TIME; + struct vfio_group_status group_status = { .argsz = sizeof(group_status) @@ -291,6 +299,7 @@ int main(int argc, char **argv) pe_op.err.addr = 0ul; pe_op.err.mask = 0ul; + pret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); if(pret < 0) { perror(""); @@ -298,9 +307,10 @@ int main(int argc, char **argv) return pret; } - printf("inject error \n"); + printf("VFIO_EEH_PE_INJECT_ERR is complete, press any key to continue\n"); fgetc(stdin); + pe_op.op = VFIO_EEH_PE_GET_STATE; ret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); if(ret < 0) { @@ -357,17 +367,25 @@ int main(int argc, char **argv) ioctl(container, VFIO_EEH_PE_OP, &pe_op); printf("slot VFIO_EEH_PE_RESET_HOT initiated\n"); pe_op.op = VFIO_EEH_PE_GET_STATE; - i = 5 ; - while (i--) + + + while (1) { ret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); - if( ret < 0) { - perror(""); - } - printf("\t."); - sleep(5); + if (ret != EEH_PE_STATE_UNAVAIL) + { + if(ret == EEH_PE_STATE_RESET) + { + /* if the slot in reset active state wait for minimum delay */ + usleep(mwait); + } + break; + } + if(max_wait < 0) + break; + usleep(MIN(mwait, max_wait)); + max_wait -= mwait; } - printf("\nVFIO_EEH_PE_GET_STATE %d\n",ret); pe_op.op = VFIO_EEH_PE_RESET_DEACTIVATE; @@ -377,7 +395,7 @@ int main(int argc, char **argv) pe_op.op = VFIO_EEH_PE_CONFIGURE; ret = ioctl(container, VFIO_EEH_PE_OP, &pe_op); if(!ret) { - printf("Success\n"); + printf("PE configure Success\n"); } else { printf("VFIO_EEH_PE_CONFIGURE failed \n"); } @@ -410,6 +428,6 @@ int main(int argc, char **argv) printf("Press any key to exit\n"); fgetc(stdin); - ioctl(device, VFIO_DEVICE_RESET); +// ioctl(device, VFIO_DEVICE_RESET); return 0; }