From 0ed1ea95607c9e2076778f4106087f63c72d13be Mon Sep 17 00:00:00 2001 From: helenaps Date: Tue, 1 Mar 2016 21:05:37 -0600 Subject: [PATCH 01/10] first_libcxl_upodate updates for "real" libcxl behavior for getting config record info --- libcxl/libcxl.c | 50 +++++++++++++++++++++++++++++++++++++++- libcxl/libcxl.h | 35 ++++------------------------ libcxl/libcxl_internal.h | 3 +++ 3 files changed, 56 insertions(+), 32 deletions(-) diff --git a/libcxl/libcxl.c b/libcxl/libcxl.c index 45fa5ff..7e73da6 100644 --- a/libcxl/libcxl.c +++ b/libcxl/libcxl.c @@ -505,6 +505,7 @@ static void *_psl_loop(void *ptr) uint8_t size; uint64_t addr; uint16_t value; + uint32_t lvalue; int rc; if (!afu) @@ -589,7 +590,8 @@ static void *_psl_loop(void *ptr) afu->int_req.state = LIBCXL_REQ_IDLE; break; case PSLSE_QUERY: - size = sizeof(uint16_t) + sizeof(uint16_t); + size = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint16_t) + + sizeof(uint16_t) + sizeof(uint32_t); if (get_bytes_silent(afu->fd, size, buffer, 1000, 0) < 0) { warn_msg("Socket failure getting PSLSE query"); @@ -600,6 +602,12 @@ static void *_psl_loop(void *ptr) afu->irqs_min = (long)ntohs(value); memcpy((char *)&value, (char *)&(buffer[3]), 2); afu->irqs_max = (long)ntohs(value); + memcpy((char *)&value, (char *)&(buffer[4]), 2); + afu->cr_device = (long)ntohs(value); + memcpy((char *)&value, (char *)&(buffer[6]), 2); + afu->cr_vendor = (long)ntohs(value); + memcpy((char *)&lvalue, (char *)&(buffer[8]), 4); + afu->cr_class = ntohl(lvalue); break; case PSLSE_MEMORY_READ: DPRINTF("AFU MEMORY READ\n"); @@ -1607,3 +1615,43 @@ int cxl_mmio_read32(struct cxl_afu_h *afu, uint64_t offset, uint32_t * data) errno = ENODEV; return -1; } + +int cxl_get_cr_device(struct cxl_afu_h *afu, long cr_num, long *valp) +{ + if (afu == NULL) + return -1; + //uint16_t crnum = cr_num; + // For now, don't worry about cr_num + *valp = afu->cr_device; + return 0; +} + +int cxl_get_cr_vendor(struct cxl_afu_h *afu, long cr_num, long *valp) +{ + if (afu == NULL) + return -1; + //uint16_t crnum = cr_num; + // For now, don't worry about cr_num + *valp = afu->cr_vendor; + return 0; +} + +int cxl_get_cr_class(struct cxl_afu_h *afu, long cr_num, long *valp) +{ + if (afu == NULL) + return -1; + //uint16_t crnum = cr_num; + // For now, don't worry about cr_num + *valp = afu->cr_class; + return 0; +} + +int cxl_get_mmio_size(struct cxl_afu_h *afu, long *valp) +{ + if (afu == NULL) + return -1; + // for now just return constant, later will read value from file + *valp = 0x04000000; + return 0; +} + diff --git a/libcxl/libcxl.h b/libcxl/libcxl.h index 949a812..3014873 100644 --- a/libcxl/libcxl.h +++ b/libcxl/libcxl.h @@ -144,7 +144,7 @@ int cxl_get_api_version_compatible(struct cxl_afu_h *afu, long *valp); int cxl_get_irqs_max(struct cxl_afu_h *afu, long *valp); //int cxl_set_irqs_max(struct cxl_afu_h *afu, long value); int cxl_get_irqs_min(struct cxl_afu_h *afu, long *valp); -//int cxl_get_mmio_size(struct cxl_afu_h *afu, long *valp); +int cxl_get_mmio_size(struct cxl_afu_h *afu, long *valp); //int cxl_get_mode(struct cxl_afu_h *afu, long *valp); //int cxl_set_mode(struct cxl_afu_h *afu, long value); //int cxl_get_modes_supported(struct cxl_afu_h *afu, long *valp); @@ -210,36 +210,9 @@ int cxl_mmio_read32(struct cxl_afu_h *afu, uint64_t offset, uint32_t * data); * Call this once per process prior to any MMIO accesses. */ //int cxl_mmio_install_sigbus_handler(); +int cxl_get_cr_device(struct cxl_afu_h *afu, long cr_num, long *valp); +int cxl_get_cr_vendor(struct cxl_afu_h *afu, long cr_num, long *valp); +int cxl_get_cr_class(struct cxl_afu_h *afu, long cr_num, long *valp); -/* FIXME FIXME Replace me with real functionality */ -static inline int -cxl_get_cr_class(struct cxl_afu_h *afu __attribute__((unused)), - long cr_num __attribute__((unused)), - long *valp) -{ - if (valp) - *valp = 0x00120000; /* accelerator */ - return 0; -} - -static inline int -cxl_get_cr_device(struct cxl_afu_h *afu __attribute__((unused)), - long cr_num __attribute__((unused)), - long *valp) -{ - if (valp) - *valp = 0x00000602; /* CGzip */ - return 0; -} - -static inline int -cxl_get_cr_vendor(struct cxl_afu_h *afu __attribute__((unused)), - long cr_num __attribute__((unused)), - long *valp) -{ - if (valp) - *valp = 0x00001014; /* IBM */ - return 0; -} #endif diff --git a/libcxl/libcxl_internal.h b/libcxl/libcxl_internal.h index c4dd4c8..c8bd9bd 100644 --- a/libcxl/libcxl_internal.h +++ b/libcxl/libcxl_internal.h @@ -74,6 +74,9 @@ struct cxl_afu_h { long mmio_len; long mmio_off; long prefault_mode; + long cr_device; + long cr_vendor; + long cr_class; struct int_req int_req; struct open_req open; struct attach_req attach; From aa857872e2b090c23693bbd7aeb48854da58f35b Mon Sep 17 00:00:00 2001 From: helenaps Date: Tue, 1 Mar 2016 21:14:14 -0600 Subject: [PATCH 02/10] second update for libcxl part two of initial update for getting CR info --- pslse/mmio.c | 25 +++++++++++++++++++++++++ pslse/mmio.h | 7 +++++++ pslse/pslse.c | 15 ++++++++++++++- 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/pslse/mmio.c b/pslse/mmio.c index 4856159..17661b2 100644 --- a/pslse/mmio.c +++ b/pslse/mmio.c @@ -208,6 +208,31 @@ int read_descriptor(struct mmio *mmio, pthread_mutex_t * lock) return -1; } + // NEW BLOCK add code to check for CRs and read them in if available + struct mmio_event *eventdevven, *eventclass; + uint32_t crstart; + uint16_t crnum = mmio->desc.num_of_afu_CRs; + if ( crnum > 0) { + crstart = mmio->desc.AFU_CR_offset; + // allocate + struct config_record *cr_array = malloc(crnum * sizeof(struct config_record *)); + //struct config_record *crptr = &cr_array; + mmio->desc.crptr = cr_array; + // Queue mmio reads + eventdevven = _add_desc(mmio, 1, 1,crstart >> 2, 0L); + eventclass = _add_desc(mmio, 1, 1, (crstart+8) >> 2, 0L); + + // Store data from reads + _wait_for_done(&(eventdevven->state), lock); + cr_array->cr_device = (uint16_t) (eventdevven->data >> 48) & 0xffffl; + cr_array->cr_vendor = (uint16_t) (eventdevven->data >> 32) & 0xffffl; + debug_msg("%x:%x CR dev & vendor", cr_array->cr_device, cr_array->cr_vendor); + free(eventdevven); + _wait_for_done(&(eventclass->state), lock); + cr_array->cr_class = (uint32_t) (eventclass->data >> 32) & 0xffffffffl; + free(eventclass); + } + // end of NEW BLOCK` return 0; } diff --git a/pslse/mmio.h b/pslse/mmio.h index 74716e1..1c41992 100644 --- a/pslse/mmio.h +++ b/pslse/mmio.h @@ -50,6 +50,12 @@ struct mmio_event { struct mmio_event *_next; }; +struct config_record { + uint16_t cr_device; + uint16_t cr_vendor; + uint32_t cr_class; +}; + struct afu_descriptor { uint16_t num_ints_per_process; uint16_t num_of_processes; @@ -64,6 +70,7 @@ struct afu_descriptor { uint64_t PerProcessPSA_offset; uint64_t AFU_EB_len; uint64_t AFU_EB_offset; + struct config_record * crptr; }; struct mmio { diff --git a/pslse/pslse.c b/pslse/pslse.c index 80bb2a9..ec83fed 100644 --- a/pslse/pslse.c +++ b/pslse/pslse.c @@ -115,7 +115,8 @@ static void _query(struct client *client, uint8_t id) psl = _find_psl(id, &major, &minor); size = 1 + sizeof(psl->mmio->desc.num_ints_per_process) + - sizeof(client->max_irqs); + sizeof(client->max_irqs) + sizeof(psl->mmio->desc.crptr->cr_device) + + sizeof(psl->mmio->desc.crptr->cr_vendor) + sizeof(psl->mmio->desc.crptr->cr_class); buffer = (uint8_t *) malloc(size); buffer[0] = PSLSE_QUERY; offset = 1; @@ -127,6 +128,18 @@ static void _query(struct client *client, uint8_t id) client->max_irqs = 2037 / psl->mmio->desc.num_of_processes; memcpy(&(buffer[offset]), (char *)&(client->max_irqs), sizeof(client->max_irqs)); + offset += sizeof(client->max_irqs); + memcpy(&(buffer[offset]), + (char *)&(psl->mmio->desc.crptr->cr_device), + sizeof(psl->mmio->desc.crptr->cr_device)); + offset += sizeof(psl->mmio->desc.crptr->cr_device); + memcpy(&(buffer[offset]), + (char *)&(psl->mmio->desc.crptr->cr_vendor), + sizeof(psl->mmio->desc.crptr->cr_vendor)); + offset += sizeof(psl->mmio->desc.crptr->cr_vendor); + memcpy(&(buffer[offset]), + (char *)&(psl->mmio->desc.crptr->cr_class), + sizeof(psl->mmio->desc.crptr->cr_class)); if (put_bytes(client->fd, size, buffer, psl->dbg_fp, psl->dbg_id, client->context) < 0) { client_drop(client, PSL_IDLE_CYCLES, CLIENT_NONE); From eb2eff703f35ea24c8518120606a797ef5c666b3 Mon Sep 17 00:00:00 2001 From: Helena Purgatorio Date: Mon, 28 Mar 2016 08:52:31 -0500 Subject: [PATCH 03/10] Add functionality requested in Issue #43: cxl_errinfo_size & cxl_errinfo_read --- common/utils.h | 1 + libcxl/libcxl.c | 107 ++++++++++++++++++++++++++++++++++++--- libcxl/libcxl.h | 3 ++ libcxl/libcxl_internal.h | 1 + pslse/mmio.c | 37 +++++++++++++- pslse/mmio.h | 3 +- pslse/psl.c | 9 ++-- pslse/pslse.c | 8 ++- 8 files changed, 155 insertions(+), 14 deletions(-) diff --git a/common/utils.h b/common/utils.h index 7db74d1..b4b055b 100644 --- a/common/utils.h +++ b/common/utils.h @@ -58,6 +58,7 @@ #define PSLSE_MMIO_FAIL 0x12 #define PSLSE_INTERRUPT 0x13 #define PSLSE_AFU_ERROR 0x14 +#define PSLSE_MMIO_EBREAD 0x15 // PSLSE states enum pslse_state { diff --git a/libcxl/libcxl.c b/libcxl/libcxl.c index 7e73da6..be7f92a 100644 --- a/libcxl/libcxl.c +++ b/libcxl/libcxl.c @@ -42,6 +42,14 @@ #define DPRINTF(...) #endif +#ifndef MAX +#define MAX(a,b)(((a)>(b))?(a):(b)) +#endif /* #ifndef MAX */ + +#ifndef MIN +#define MIN(a,b)(((a)<(b))?(a):(b)) +#endif /* #ifndef MIN */ + /* * System constants */ @@ -51,6 +59,7 @@ #define FOURK_MASK 0xFFFFFFFFFFFFF000L #define DSISR 0x4000000040000000L +#define ERR_BUFF_MAX_COPY_SIZE 4096 static int _delay_1ms() { @@ -306,7 +315,7 @@ static void _handle_ack(struct cxl_afu_h *afu) if (!afu) fatal_msg("NULL afu passed to libcxl.c:_handle_ack"); DPRINTF("MMIO ACK\n"); - if (afu->mmio.type == PSLSE_MMIO_READ64) { + if ((afu->mmio.type == PSLSE_MMIO_READ64)| (afu->mmio.type == PSLSE_MMIO_EBREAD)) { if (get_bytes_silent(afu->fd, sizeof(uint64_t), data, 1000, 0) < 0) { warn_msg("Socket failure getting MMIO Ack"); @@ -529,6 +538,7 @@ static void *_psl_loop(void *ptr) case PSLSE_MMIO_WRITE32: _mmio_write32(afu); break; + case PSLSE_MMIO_EBREAD: case PSLSE_MMIO_READ64: case PSLSE_MMIO_READ32: /*fall through */ _mmio_read(afu); @@ -590,8 +600,8 @@ static void *_psl_loop(void *ptr) afu->int_req.state = LIBCXL_REQ_IDLE; break; case PSLSE_QUERY: - size = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint16_t) - + sizeof(uint16_t) + sizeof(uint32_t); + size = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(size_t) + + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t); if (get_bytes_silent(afu->fd, size, buffer, 1000, 0) < 0) { warn_msg("Socket failure getting PSLSE query"); @@ -602,11 +612,13 @@ static void *_psl_loop(void *ptr) afu->irqs_min = (long)ntohs(value); memcpy((char *)&value, (char *)&(buffer[3]), 2); afu->irqs_max = (long)ntohs(value); - memcpy((char *)&value, (char *)&(buffer[4]), 2); + memcpy((char *)&value, (char *)&(buffer[5]), 8); + afu->eb_len = (long)ntohs(value); + memcpy((char *)&value, (char *)&(buffer[12]), 2); afu->cr_device = (long)ntohs(value); - memcpy((char *)&value, (char *)&(buffer[6]), 2); + memcpy((char *)&value, (char *)&(buffer[14]), 2); afu->cr_vendor = (long)ntohs(value); - memcpy((char *)&lvalue, (char *)&(buffer[8]), 4); + memcpy((char *)&lvalue, (char *)&(buffer[16]), 4); afu->cr_class = ntohl(lvalue); break; case PSLSE_MEMORY_READ: @@ -1562,6 +1574,81 @@ int cxl_mmio_read64(struct cxl_afu_h *afu, uint64_t offset, uint64_t * data) return -1; } +int cxl_errinfo_read(struct cxl_afu_h *afu, void *dst, off_t off, size_t len) +{ + off_t aligned_start, last_byte; + off_t index1, index2; + uint8_t *buffer; + size_t total_read_length; + + if ((afu == NULL) || !afu->mapped) + goto bread64_fail; + if (len == 0 || off < 0 || (size_t)off >= afu->eb_len) + return 0; + + /* calculate aligned read window */ + len = MIN((size_t)(afu->eb_len - off), len); + aligned_start = off & 0xfff8; + last_byte = aligned_start + len + (off & 0x7); + total_read_length = (((off & 0x7) + len + 0x7) >>3) << 3 ; + buffer = (uint8_t* )calloc(total_read_length +8, sizeof(uint8_t)); + if (!buffer) + return -ENOMEM; + uint64_t *wbuf = (uint64_t *)buffer; + uint8_t *bbuf = (uint8_t *)buffer; + + /* max we can copy in one read is PAGE_SIZE */ + if (total_read_length > ERR_BUFF_MAX_COPY_SIZE) { + total_read_length = ERR_BUFF_MAX_COPY_SIZE; + len = ERR_BUFF_MAX_COPY_SIZE - (off & 0x7); + } + + /* perform aligned read from the mmio region */ + index1 = 0; + while (aligned_start <= last_byte) { + // Send MMIO request to PSLSE + afu->mmio.type = PSLSE_MMIO_EBREAD; + afu->mmio.addr = (uint32_t) aligned_start ; + afu->mmio.state = LIBCXL_REQ_REQUEST; + while (afu->mmio.state != LIBCXL_REQ_IDLE) /*infinite loop */ + _delay_1ms(); + if (!afu->opened) + goto bread64_fail; + // if offset, have to potentially do BE->LE swap + if ((off & 0x7) >0) + afu->mmio.data = htonll(afu->mmio.data); + wbuf[index1] = afu->mmio.data; + aligned_start = aligned_start + 8; + ++index1; + } + memcpy(&wbuf[0], &bbuf[off & 0x7], len); + // if offset we have to do LE->BE swap back + if ((off & 0x7) > 0) { + index2 = 0; + total_read_length = len; + while (total_read_length !=0) { + if (total_read_length < 8) + // set total_read_length to 0 + total_read_length = 0; + else { + wbuf[index2]= htonll(wbuf[index2]); + ++index2; + total_read_length = total_read_length -8; + } + } + } + memcpy(dst, &wbuf[0], len); + free(buffer); + // return # of bytes read + return len; + + bread64_fail: + if (buffer) + free(buffer); + errno = ENODEV; + return -1; +} + int cxl_mmio_write32(struct cxl_afu_h *afu, uint64_t offset, uint32_t data) { if (offset & 0x3) { @@ -1655,3 +1742,11 @@ int cxl_get_mmio_size(struct cxl_afu_h *afu, long *valp) return 0; } +int cxl_errinfo_size(struct cxl_afu_h *afu, size_t *valp) +{ + if (afu == NULL) + return -1; + *valp = afu->eb_len; + return 0; +} + diff --git a/libcxl/libcxl.h b/libcxl/libcxl.h index 3014873..e65ad92 100644 --- a/libcxl/libcxl.h +++ b/libcxl/libcxl.h @@ -213,6 +213,9 @@ int cxl_mmio_read32(struct cxl_afu_h *afu, uint64_t offset, uint32_t * data); int cxl_get_cr_device(struct cxl_afu_h *afu, long cr_num, long *valp); int cxl_get_cr_vendor(struct cxl_afu_h *afu, long cr_num, long *valp); int cxl_get_cr_class(struct cxl_afu_h *afu, long cr_num, long *valp); +int cxl_errinfo_size(struct cxl_afu_h *afu, size_t *valp); +int cxl_errinfo_read(struct cxl_afu_h *afu, void *dst, off_t off, size_t len); + #endif diff --git a/libcxl/libcxl_internal.h b/libcxl/libcxl_internal.h index c8bd9bd..039b964 100644 --- a/libcxl/libcxl_internal.h +++ b/libcxl/libcxl_internal.h @@ -74,6 +74,7 @@ struct cxl_afu_h { long mmio_len; long mmio_off; long prefault_mode; + size_t eb_len; long cr_device; long cr_vendor; long cr_class; diff --git a/pslse/mmio.c b/pslse/mmio.c index 17661b2..ee9d1b6 100644 --- a/pslse/mmio.c +++ b/pslse/mmio.c @@ -75,7 +75,7 @@ static struct mmio_event *_add_event(struct mmio *mmio, struct client *client, return event; event->rnw = rnw; event->dw = dw; - if (client == NULL) { + if (client == NULL) { // don't try to recalculate the address event->addr = addr; } else { @@ -441,9 +441,38 @@ static struct mmio_event *_handle_mmio_read(struct mmio *mmio, return NULL; } +// Add mmio read event of error buffer at offset to list +static struct mmio_event *_handle_mmio_read_eb(struct mmio *mmio, + struct client *client, int dw) +{ + struct mmio_event *event; + uint32_t offset; + int fd = client->fd; + + if (get_bytes_silent(fd, 4, (uint8_t *) & offset, mmio->timeout, + &(client->abort)) < 0) { + goto read_fail; + } + offset = ntohl(offset); + offset = offset + (uint32_t)mmio->desc.AFU_EB_offset; + debug_msg("offset for eb read is %x\n", offset); +// event = _add_event(mmio, client, 1, dw, offset>>2, 1, 0); + event = _add_desc(mmio, 1, dw, offset>>2, 0); +// _wait_for_done(&(event->state), psl->lock); + return event; + + read_fail: + // Socket connection is dead + debug_msg("%s:_handle_mmio_read failed context=%d", + mmio->afu_name, client->context); + client_drop(client, PSL_IDLE_CYCLES, CLIENT_NONE); + return NULL; +} + + // Handle MMIO request from client struct mmio_event *handle_mmio(struct mmio *mmio, struct client *client, - int rnw, int dw) + int rnw, int dw, int eb_rd) { uint8_t ack; @@ -456,6 +485,10 @@ struct mmio_event *handle_mmio(struct mmio *mmio, struct client *client, } return NULL; } + + if (eb_rd) + return _handle_mmio_read_eb(mmio, client, dw); + if (rnw) return _handle_mmio_read(mmio, client, dw); else diff --git a/pslse/mmio.h b/pslse/mmio.h index 1c41992..470f2bc 100644 --- a/pslse/mmio.h +++ b/pslse/mmio.h @@ -42,6 +42,7 @@ struct mmio_event { uint32_t rnw; uint32_t dw; + uint32_t eb_rd; uint32_t addr; uint32_t desc; uint64_t data; @@ -99,7 +100,7 @@ void handle_mmio_ack(struct mmio *mmio, uint32_t parity_enabled); void handle_mmio_map(struct mmio *mmio, struct client *client); struct mmio_event *handle_mmio(struct mmio *mmio, struct client *client, - int rnw, int dw); + int rnw, int dw, int eb_rd); struct mmio_event *handle_mmio_done(struct mmio *mmio, struct client *client); diff --git a/pslse/psl.c b/pslse/psl.c index 32eaa62..944c7c6 100644 --- a/pslse/psl.c +++ b/pslse/psl.c @@ -445,6 +445,7 @@ static void _handle_client(struct psl *psl, struct client *client) struct cmd_event *cmd; uint8_t buffer[MAX_LINE_CHARS]; int dw = 0; + int eb_rd = 0; // Handle MMIO done if (client->mmio_access != NULL) { @@ -490,12 +491,14 @@ static void _handle_client(struct psl *psl, struct client *client) case PSLSE_MMIO_WRITE64: dw = 1; case PSLSE_MMIO_WRITE32: /*fall through */ - mmio = handle_mmio(psl->mmio, client, 0, dw); + mmio = handle_mmio(psl->mmio, client, 0, dw, 0); break; - case PSLSE_MMIO_READ64: + case PSLSE_MMIO_EBREAD: + eb_rd = 1; + case PSLSE_MMIO_READ64: /*fall through */ dw = 1; case PSLSE_MMIO_READ32: /*fall through */ - mmio = handle_mmio(psl->mmio, client, 1, dw); + mmio = handle_mmio(psl->mmio, client, 1, dw, eb_rd); break; default: error_msg("Unexpected 0x%02x from client on socket", buffer[0], client->fd); diff --git a/pslse/pslse.c b/pslse/pslse.c index ec83fed..e03e694 100644 --- a/pslse/pslse.c +++ b/pslse/pslse.c @@ -114,8 +114,8 @@ static void _query(struct client *client, uint8_t id) int size, offset; psl = _find_psl(id, &major, &minor); - size = 1 + sizeof(psl->mmio->desc.num_ints_per_process) + - sizeof(client->max_irqs) + sizeof(psl->mmio->desc.crptr->cr_device) + + size = 1 + sizeof(psl->mmio->desc.num_ints_per_process) + sizeof(client->max_irqs) + + sizeof(psl->mmio->desc.AFU_EB_len) + sizeof(psl->mmio->desc.crptr->cr_device) + sizeof(psl->mmio->desc.crptr->cr_vendor) + sizeof(psl->mmio->desc.crptr->cr_class); buffer = (uint8_t *) malloc(size); buffer[0] = PSLSE_QUERY; @@ -129,6 +129,10 @@ static void _query(struct client *client, uint8_t id) memcpy(&(buffer[offset]), (char *)&(client->max_irqs), sizeof(client->max_irqs)); offset += sizeof(client->max_irqs); + memcpy(&(buffer[offset]), + (char *)&(psl->mmio->desc.AFU_EB_len), + sizeof(psl->mmio->desc.AFU_EB_len)); + offset += sizeof(psl->mmio->desc.AFU_EB_len); memcpy(&(buffer[offset]), (char *)&(psl->mmio->desc.crptr->cr_device), sizeof(psl->mmio->desc.crptr->cr_device)); From e3f6818b39130d58b59298cd6b4712b3691050e8 Mon Sep 17 00:00:00 2001 From: Helena Purgatorio Date: Fri, 22 Apr 2016 15:00:17 -0500 Subject: [PATCH 04/10] Changes to enable Group 1 list of libcxl functions for pslse. --- common/debug.h | 4 + common/utils.h | 1 + libcxl/libcxl.c | 144 ++++++++++++++++++++++++++++++++-- libcxl/libcxl.h | 24 +++--- pslse/mmio.c | 2 + pslse/parms.c | 17 ++++ pslse/parms.h | 4 + pslse/psl.c | 5 ++ pslse/psl.h | 5 ++ pslse/pslse.c | 16 +++- pslse/pslse.parms | 14 +++- pslse/shim_host.dat | 2 +- test/afu/Descriptor.cpp | 33 ++++++-- test/afu/afu_descriptor.cfg | 46 ++++++++++- test/regress/bad_commands.xml | 7 +- test/regress/directed.xml | 3 + test/regress/interrupt.xml | 5 ++ test/regress/main.xml | 5 ++ test/regress/parity_addr.xml | 5 ++ test/regress/parity_code.xml | 5 ++ test/regress/parity_mmio.xml | 5 ++ test/regress/parity_read.xml | 5 ++ test/regress/parity_tag.xml | 5 ++ 23 files changed, 327 insertions(+), 35 deletions(-) diff --git a/common/debug.h b/common/debug.h index d84b8de..0224654 100644 --- a/common/debug.h +++ b/common/debug.h @@ -60,6 +60,10 @@ typedef uint8_t DBG_HEADER; #define DBG_PARM_PAGED_PERCENT 0x4 #define DBG_PARM_REORDER_PERCENT 0x5 #define DBG_PARM_BUFFER_PERCENT 0x6 +#define DBG_CAIA_VERSION 0x7 +#define DBG_PSL_REV_LVL 0x8 +#define DBG_IMAGE_LOADED 0x9 +#define DBG_BASE_IMAGE 0xA size_t debug_get_64(FILE * fp, uint64_t * value); size_t debug_get_32(FILE * fp, uint32_t * value); diff --git a/common/utils.h b/common/utils.h index b4b055b..ffb5093 100644 --- a/common/utils.h +++ b/common/utils.h @@ -59,6 +59,7 @@ #define PSLSE_INTERRUPT 0x13 #define PSLSE_AFU_ERROR 0x14 #define PSLSE_MMIO_EBREAD 0x15 +#define PSLSE_VSEC_INFO 0x16 // PSLSE states enum pslse_state { diff --git a/libcxl/libcxl.c b/libcxl/libcxl.c index be7f92a..2eb3b9c 100644 --- a/libcxl/libcxl.c +++ b/libcxl/libcxl.c @@ -599,8 +599,9 @@ static void *_psl_loop(void *ptr) afu->irqs_max = ntohs(value); afu->int_req.state = LIBCXL_REQ_IDLE; break; - case PSLSE_QUERY: - size = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(size_t) + + case PSLSE_QUERY: { + size = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint16_t) + + sizeof(uint64_t) + sizeof(uint64_t) + sizeof(size_t) + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t); if (get_bytes_silent(afu->fd, size, buffer, 1000, 0) < 0) { @@ -612,15 +613,24 @@ static void *_psl_loop(void *ptr) afu->irqs_min = (long)ntohs(value); memcpy((char *)&value, (char *)&(buffer[3]), 2); afu->irqs_max = (long)ntohs(value); - memcpy((char *)&value, (char *)&(buffer[5]), 8); + memcpy((char *)&value, (char *)&(buffer[5]), 2); + afu->modes_supported = (long)ntohs(value); + memcpy((char *)&value, (char *)&(buffer[7]), 8); + afu->mmio_len = (long)ntohs(value); + memcpy((char *)&value, (char *)&(buffer[15]), 8); + afu->mmio_off = (long)ntohs(value); + memcpy((char *)&value, (char *)&(buffer[23]), 8); afu->eb_len = (long)ntohs(value); - memcpy((char *)&value, (char *)&(buffer[12]), 2); + memcpy((char *)&value, (char *)&(buffer[30]), 2); afu->cr_device = (long)ntohs(value); - memcpy((char *)&value, (char *)&(buffer[14]), 2); + memcpy((char *)&value, (char *)&(buffer[32]), 2); afu->cr_vendor = (long)ntohs(value); - memcpy((char *)&lvalue, (char *)&(buffer[16]), 4); + memcpy((char *)&lvalue, (char *)&(buffer[34]), 4); afu->cr_class = ntohl(lvalue); + //no better place to put this right now + afu->prefault_mode = CXL_PREFAULT_MODE_NONE; break; + } case PSLSE_MEMORY_READ: DPRINTF("AFU MEMORY READ\n"); if (get_bytes_silent(afu->fd, 1, buffer, 1000, 0) < 0) { @@ -1271,12 +1281,15 @@ struct cxl_afu_h *cxl_afu_open_h(struct cxl_afu_h *afu, enum cxl_views view) switch (view) { case CXL_VIEW_DEDICATED: afu_type = 'd'; + afu->mode = CXL_MODE_DEDICATED; break; case CXL_VIEW_MASTER: afu_type = 'm'; + afu->mode = CXL_MODE_DIRECTED; break; case CXL_VIEW_SLAVE: afu_type = 's'; + afu->mode = CXL_MODE_DIRECTED; break; default: errno = ENODEV; @@ -1709,6 +1722,7 @@ int cxl_get_cr_device(struct cxl_afu_h *afu, long cr_num, long *valp) return -1; //uint16_t crnum = cr_num; // For now, don't worry about cr_num + //*valp = htons(afu->cr_device); *valp = afu->cr_device; return 0; } @@ -1719,6 +1733,7 @@ int cxl_get_cr_vendor(struct cxl_afu_h *afu, long cr_num, long *valp) return -1; //uint16_t crnum = cr_num; // For now, don't worry about cr_num + //*valp = htons(afu->cr_vendor); *valp = afu->cr_vendor; return 0; } @@ -1729,6 +1744,7 @@ int cxl_get_cr_class(struct cxl_afu_h *afu, long cr_num, long *valp) return -1; //uint16_t crnum = cr_num; // For now, don't worry about cr_num + //*valp = htonl(afu->cr_class); *valp = afu->cr_class; return 0; } @@ -1750,3 +1766,119 @@ int cxl_errinfo_size(struct cxl_afu_h *afu, size_t *valp) return 0; } +int cxl_get_pp_mmio_len(struct cxl_afu_h *afu, long *valp) +{ + if (afu == NULL) + return -1; + *valp = afu->mmio_len; + return 0; +} + +int cxl_get_pp_mmio_off(struct cxl_afu_h *afu, long *valp) +{ + if (afu == NULL) + return -1; + *valp = afu->mmio_off; + return 0; +} + +int cxl_get_modes_supported(struct cxl_afu_h *afu, long *valp) +{ +//List of the modes this AFU supports. One per line. +//Valid entries are: "dedicated_process" and "afu_directed" + if (afu == NULL) + return -1; + *valp = afu->modes_supported; + return 0; +} + +int cxl_get_mode(struct cxl_afu_h *afu, long *valp) +{ + if (afu == NULL) + return -1; + *valp = afu->mode; + return 0; +} + +int cxl_set_mode(struct cxl_afu_h *afu, long value) +{ +//Writing will change the mode provided that no user contexts are attached. + if (afu == NULL) + return -1; + //check to be sure no contexts are attached before setting this, could be hard to tell? + afu->mode = value; + // do we also need to change afu_type to match mode now?? + return 0; +} + +int cxl_get_prefault_mode(struct cxl_afu_h *afu, enum cxl_prefault_mode *valp) +{ +//Get the mode for prefaulting in segments into the segment table +//when performing the START_WORK ioctl. Possible values: +// none: No prefaulting (default) +// work_element_descriptor: Treat the work element +// descriptor as an effective address and +// prefault what it points to. +// all: all segments process calling START_WORK maps. + if (afu == NULL) + return -1; + *valp = afu->prefault_mode; + return 0; +} + +int cxl_set_prefault_mode(struct cxl_afu_h *afu, enum cxl_prefault_mode value) +{ +//Set the mode for prefaulting in segments into the segment table +//when performing the START_WORK ioctl. Possible values: +// none: No prefaulting (default) +// work_element_descriptor: Treat the work element +// descriptor as an effective address and +// prefault what it points to. +// all: all segments process calling START_WORK maps. + if (afu == NULL) + return -1; + if (value == CXL_PREFAULT_MODE_NONE || CXL_PREFAULT_MODE_WED || CXL_PREFAULT_MODE_ALL) + afu->prefault_mode = value; +//Probably should return error msg if value wasn't a "good" value + return 0; +} + +int cxl_get_base_image(struct cxl_adapter_h *adapter, long *valp) +{ + if (adapter == NULL) + return -1; + // for now just return constant, later will read value from file + *valp = 0x0; + return 0; +} + + +int cxl_get_caia_version(struct cxl_adapter_h *adapter, long *majorp,long *minorp) +{ + if (adapter == NULL) + return -1; + // for now just return constant, later will read value from file + *majorp = 0x1; + *minorp = 0x0; + return 0; +} + +int cxl_get_image_loaded(struct cxl_adapter_h *adapter, enum cxl_image *valp) +{ + if (adapter == NULL) + return -1; + // for now just return constant, later will read value from file + *valp = CXL_IMAGE_USER; + return 0; +} + +int cxl_get_psl_revision(struct cxl_adapter_h *adapter, long *valp) +{ + if (adapter == NULL) + return -1; + // for now just return constant, later will read value from file + *valp = 0x0; + return 0; +} + + diff --git a/libcxl/libcxl.h b/libcxl/libcxl.h index e65ad92..a60a4bb 100644 --- a/libcxl/libcxl.h +++ b/libcxl/libcxl.h @@ -145,19 +145,19 @@ int cxl_get_irqs_max(struct cxl_afu_h *afu, long *valp); //int cxl_set_irqs_max(struct cxl_afu_h *afu, long value); int cxl_get_irqs_min(struct cxl_afu_h *afu, long *valp); int cxl_get_mmio_size(struct cxl_afu_h *afu, long *valp); -//int cxl_get_mode(struct cxl_afu_h *afu, long *valp); -//int cxl_set_mode(struct cxl_afu_h *afu, long value); -//int cxl_get_modes_supported(struct cxl_afu_h *afu, long *valp); -//int cxl_get_prefault_mode(struct cxl_afu_h *afu, enum cxl_prefault_mode *valp); -//int cxl_set_prefault_mode(struct cxl_afu_h *afu, enum cxl_prefault_mode value); +int cxl_get_mode(struct cxl_afu_h *afu, long *valp); +int cxl_set_mode(struct cxl_afu_h *afu, long value); +int cxl_get_modes_supported(struct cxl_afu_h *afu, long *valp); +int cxl_get_prefault_mode(struct cxl_afu_h *afu, enum cxl_prefault_mode *valp); +int cxl_set_prefault_mode(struct cxl_afu_h *afu, enum cxl_prefault_mode value); //int cxl_get_dev(struct cxl_afu_h *afu, long *majorp, long *minorp); -//int cxl_get_pp_mmio_len(struct cxl_afu_h *afu, long *valp); -//int cxl_get_pp_mmio_off(struct cxl_afu_h *afu, long *valp); -//int cxl_get_base_image(struct cxl_adapter_h *adapter, long *valp); -//int cxl_get_caia_version(struct cxl_adapter_h *adapter, long *majorp, -// long *minorp); -//int cxl_get_image_loaded(struct cxl_adapter_h *adapter, enum cxl_image *valp); -//int cxl_get_psl_revision(struct cxl_adapter_h *adapter, long *valp); +int cxl_get_pp_mmio_len(struct cxl_afu_h *afu, long *valp); +int cxl_get_pp_mmio_off(struct cxl_afu_h *afu, long *valp); +int cxl_get_base_image(struct cxl_adapter_h *adapter, long *valp); +int cxl_get_caia_version(struct cxl_adapter_h *adapter, long *majorp, + long *minorp); +int cxl_get_image_loaded(struct cxl_adapter_h *adapter, enum cxl_image *valp); +int cxl_get_psl_revision(struct cxl_adapter_h *adapter, long *valp); /* * Events diff --git a/pslse/mmio.c b/pslse/mmio.c index ee9d1b6..672907c 100644 --- a/pslse/mmio.c +++ b/pslse/mmio.c @@ -224,10 +224,12 @@ int read_descriptor(struct mmio *mmio, pthread_mutex_t * lock) // Store data from reads _wait_for_done(&(eventdevven->state), lock); + debug_msg("XXXX: DATA: = %016x\n", eventdevven->data); cr_array->cr_device = (uint16_t) (eventdevven->data >> 48) & 0xffffl; cr_array->cr_vendor = (uint16_t) (eventdevven->data >> 32) & 0xffffl; debug_msg("%x:%x CR dev & vendor", cr_array->cr_device, cr_array->cr_vendor); free(eventdevven); + debug_msg("%x:%x CR dev & vendor swapped", ntohs(cr_array->cr_device),ntohs(cr_array->cr_vendor)); _wait_for_done(&(eventclass->state), lock); cr_array->cr_class = (uint32_t) (eventclass->data >> 32) & 0xffffffffl; free(eventclass); diff --git a/pslse/parms.c b/pslse/parms.c index 6cc1416..43a21b6 100644 --- a/pslse/parms.c +++ b/pslse/parms.c @@ -184,6 +184,18 @@ struct parms *parse_parms(char *filename, FILE * dbg_fp) parms->buffer_percent = data; debug_parm(dbg_fp, DBG_PARM_BUFFER_PERCENT, parms->buffer_percent); + } else if (!(strcmp(parm, "CAIA_VERSION"))) { + parms->caia_version = atoi(value); + debug_parm(dbg_fp, DBG_CAIA_VERSION, parms->caia_version); + } else if (!(strcmp(parm, "PSL_REV_LEVEL"))) { + parms->psl_rev_level = atoi(value); + debug_parm(dbg_fp, DBG_PSL_REV_LVL, parms->psl_rev_level); + } else if (!(strcmp(parm, "IMAGE_LOADED"))) { + parms->image_loaded = atoi(value); + debug_parm(dbg_fp, DBG_IMAGE_LOADED, parms->image_loaded); + } else if (!(strcmp(parm, "BASE_IMAGE_REV_LEVEL"))) { + parms->base_image = atoi(value); + debug_parm(dbg_fp, DBG_BASE_IMAGE, parms->base_image); } else { warn_msg("Ignoring invalid parm in %s: %s\n", filename, parm); @@ -208,6 +220,11 @@ struct parms *parse_parms(char *filename, FILE * dbg_fp) printf("\tPaged = %d%%\n", parms->paged_percent); printf("\tReorder = %d%%\n", parms->reorder_percent); printf("\tBuffer = %d%%\n", parms->buffer_percent); +//When we start reading these values in from pslse.parms, uncomment +// printf("\tCAIA_Ver = %4d\n", parms->caia_version); +// printf("\tPSL_REV = %d\n", parms->psl_rev_level); +// printf("\tImage_Loaded = %d\n", parms->image_loaded); +// printf("\tBase_Image = %d\n", parms->base_image); // Adjust timeout to milliseconds parms->timeout *= 1000; diff --git a/pslse/parms.h b/pslse/parms.h index 1ee5a39..4b0aca5 100644 --- a/pslse/parms.h +++ b/pslse/parms.h @@ -27,6 +27,10 @@ struct parms { unsigned int paged_percent; unsigned int reorder_percent; unsigned int buffer_percent; + unsigned int caia_version; + unsigned int psl_rev_level; + unsigned int image_loaded; + unsigned int base_image; }; // Randomly decide to allow response to AFU diff --git a/pslse/psl.c b/pslse/psl.c index 944c7c6..dc846e1 100644 --- a/pslse/psl.c +++ b/pslse/psl.c @@ -774,6 +774,11 @@ uint16_t psl_init(struct psl **head, struct parms *parms, char *id, char *host, perror("cmd_init"); goto init_fail; } + // Load in VSEC data (read in from pslse.parms file) + psl->vsec_caia_version = parms->caia_version; + psl->vsec_psl_rev_level= parms->psl_rev_level; + psl->vsec_image_loaded= parms->image_loaded; + psl->vsec_base_image= parms->base_image; // Set credits for AFU if (psl_aux1_change(psl->afu_event, psl->cmd->credits) != PSL_SUCCESS) { warn_msg("Unable to set credits"); diff --git a/pslse/psl.h b/pslse/psl.h index 7e3114e..29d7c02 100644 --- a/pslse/psl.h +++ b/pslse/psl.h @@ -28,6 +28,7 @@ #include "parms.h" #include "../common/utils.h" + struct psl { struct AFU_EVENT *afu_event; pthread_t thread; @@ -54,6 +55,10 @@ struct psl { int attached_clients; int timeout; int has_been_reset; + uint16_t vsec_caia_version; + uint16_t vsec_psl_rev_level; + uint16_t vsec_image_loaded; + uint16_t vsec_base_image; }; uint16_t psl_init(struct psl **head, struct parms *parms, char *id, char *host, diff --git a/pslse/pslse.c b/pslse/pslse.c index e03e694..1c073fc 100644 --- a/pslse/pslse.c +++ b/pslse/pslse.c @@ -114,7 +114,9 @@ static void _query(struct client *client, uint8_t id) int size, offset; psl = _find_psl(id, &major, &minor); - size = 1 + sizeof(psl->mmio->desc.num_ints_per_process) + sizeof(client->max_irqs) + + size = 1 + sizeof(psl->mmio->desc.num_ints_per_process) + sizeof(client->max_irqs) + + sizeof(psl->mmio->desc.req_prog_model) + + sizeof(psl->mmio->desc.PerProcessPSA) + sizeof(psl->mmio->desc.PerProcessPSA_offset) + sizeof(psl->mmio->desc.AFU_EB_len) + sizeof(psl->mmio->desc.crptr->cr_device) + sizeof(psl->mmio->desc.crptr->cr_vendor) + sizeof(psl->mmio->desc.crptr->cr_class); buffer = (uint8_t *) malloc(size); @@ -129,6 +131,18 @@ static void _query(struct client *client, uint8_t id) memcpy(&(buffer[offset]), (char *)&(client->max_irqs), sizeof(client->max_irqs)); offset += sizeof(client->max_irqs); + memcpy(&(buffer[offset]), + (char *)&(psl->mmio->desc.req_prog_model), + sizeof(psl->mmio->desc.req_prog_model)); + offset += sizeof(psl->mmio->desc.req_prog_model); + memcpy(&(buffer[offset]), + (char *)&(psl->mmio->desc.PerProcessPSA), + sizeof(psl->mmio->desc.PerProcessPSA)); + offset += sizeof(psl->mmio->desc.PerProcessPSA); + memcpy(&(buffer[offset]), + (char *)&(psl->mmio->desc.PerProcessPSA_offset), + sizeof(psl->mmio->desc.PerProcessPSA_offset)); + offset += sizeof(psl->mmio->desc.PerProcessPSA_offset); memcpy(&(buffer[offset]), (char *)&(psl->mmio->desc.AFU_EB_len), sizeof(psl->mmio->desc.AFU_EB_len)); diff --git a/pslse/pslse.parms b/pslse/pslse.parms index 4c1954c..85070e3 100644 --- a/pslse/pslse.parms +++ b/pslse/pslse.parms @@ -22,8 +22,8 @@ # Randomization seed. Set this to force reproducible sequence of event # NOTE: Must be a single value, not a min,max range -SEED:13 - +#SEED:13 +SEED:1461247482 # Percentage chance of PSL driving any pending responses in a clock cycle. # Can not be 0 otherwise PSL will never generate responses to commands. # Setting to 100 will cause all responses to be very rapid and generally @@ -33,10 +33,18 @@ SEED:13 RESPONSE_PERCENT:10,20 # Percentage chance of PSL responding with PAGED for any command response. -PAGED_PERCENT:2,4 +#PAGED_PERCENT:2,4 +PAGED_PERCENT:0 # Percentage chance of PSL reordering the execution of commands. REORDER_PERCENT:80,90 # Percentage chance of PSL generating extra buffer read/write activity. BUFFER_PERCENT:80,90 + +# VSEC data lines +#CAIA_VERSION:0100 +#PSL_REV_LEVEL:0 +#IMAGE_LOADED:1 +#BASE_IMAGE_REV_LEVEL:0 + diff --git a/pslse/shim_host.dat b/pslse/shim_host.dat index 2adcfa0..411754f 100644 --- a/pslse/shim_host.dat +++ b/pslse/shim_host.dat @@ -1,7 +1,7 @@ # A copy of this file should be placed in the run directory. # Each line maps an AFU device to it's simulator host:port. # Update as needed to match your simulation environment. -# Comments must be on there own line. +# Comments must be on their own line. # # Line format is as follows: # AFU_DEVICE,HOSTNAME:PORT diff --git a/test/afu/Descriptor.cpp b/test/afu/Descriptor.cpp index 91e2e89..19170c8 100644 --- a/test/afu/Descriptor.cpp +++ b/test/afu/Descriptor.cpp @@ -1,6 +1,8 @@ #include "Descriptor.h" +#include #include +#include #include #include #include @@ -26,7 +28,7 @@ Descriptor::parse_descriptor_file (string filename) error_msg ("Descriptor::parse_descriptor_file: failed to open file %s", filename.c_str ()); - string line, field, colon, s_value; + string line, field, colon, s_value, s_data; while (getline (file, line)) { // skip comments and empty lines @@ -36,19 +38,36 @@ Descriptor::parse_descriptor_file (string filename) ss >> field >> colon >> s_value; + uint64_t value, data; + + // Test for data values + if (field == "data") { + if (s_value.substr (0, 2) == "0x") + s_value.erase(0, 2); + getline (file, s_data); + if (s_data.substr (0, 2) == "0x") + s_data.erase(0, 2); + value = strtoull(s_value.c_str(), NULL, 16); + data = strtoull(s_data.c_str(), NULL, 16); + info_msg ("Descriptor: setting offset 0x%x with value 0x%016llx", + value, data); + uint64_t offset = to_vector_index(value); + while (offset >= regs.size()) + regs.push_back(0); + regs[offset] = data; + continue; + } + // re-output s_value as unsigned int - uint64_t value; if (s_value.substr (0, 2) == "0x") { stringstream temp (s_value.substr (2)); - temp >> std::hex >> value; - info_msg ("Descriptor: setting %s with value %x", field.c_str (), + info_msg ("Descriptor: setting %s with value 0x%x", field.c_str (), value); } else { stringstream temp (s_value); - temp >> value; info_msg ("Descriptor: setting %s with value %d", field.c_str (), value); @@ -105,9 +124,7 @@ Descriptor::parse_descriptor_file (string filename) uint32_t Descriptor::to_vector_index (uint32_t byte_address) const { - return - byte_address >> - 3; + return byte_address >> 3; } uint64_t diff --git a/test/afu/afu_descriptor.cfg b/test/afu/afu_descriptor.cfg index 7e2c117..f290ae1 100644 --- a/test/afu/afu_descriptor.cfg +++ b/test/afu/afu_descriptor.cfg @@ -5,9 +5,49 @@ # make sure to have space(s) before AND after the colon # values can be decimal or hex (hex must start with 0x) -# reg0x00 +# Single process AFU +# Dedicated mode uses 0x01, directed uses 0x04 +#num_of_processes : 4 num_of_processes : 1 +# Dedicated mode uses 0x8010, directed uses 0x8004 +#reg_prog_model : 0x8004 reg_prog_model : 0x8010 - -# reg0x30 +# PSA space required +# Dedicated mode uses 0x01, directed uses 0x03 +#PerProcessPSA_control : 0x03 PerProcessPSA_control : 0x01 +PerProcessPSA_length : 0x1 +#PerProcessPSA_offset : 0x1 +PerProcessPSA_offset : 0x1000 + +# Define 1 configuration record of 256 bytes at 0x100 +num_of_afu_CRs : 1 +AFU_CR_len : 0x100 +AFU_CR_offset : 0x100 + +# Define 1 4KB error buffer at 0x1000 +AFU_EB_len : 0x1000 +AFU_EB_offset : 0x1000 + +# Set vendor & device ID in configuration record (address 0x100) +#0x61ca141013005640 +#0x0000120000000000 +data : 0x100 +0x141061ca13005640 +data : 0x108 +0x0000001200000000 +#Set up data area for error buffer test +data : 0x1000 +0x0001020304050607 +data : 0x1008 +0x08090a0b0c0d0e0f +data : 0x1010 +0x1011121314151617 +data : 0x1018 +0x18191a1b1c1d1e1f +data : 0x1020 +0x2021222324252627 +data : 0x1028 +0x28292a2b2c2d2e2f +data : 0x1030 +0x3031333334353637 diff --git a/test/regress/bad_commands.xml b/test/regress/bad_commands.xml index 037024a..8ae62f1 100644 --- a/test/regress/bad_commands.xml +++ b/test/regress/bad_commands.xml @@ -6,6 +6,11 @@ 1 0x8010 0x01 + 0x01 + 0x1000 + 1 + 0x100 + 0x100 25,50 @@ -14,7 +19,7 @@ 0 ERROR - + diff --git a/test/regress/directed.xml b/test/regress/directed.xml index c3c8f76..42e5d9f 100644 --- a/test/regress/directed.xml +++ b/test/regress/directed.xml @@ -8,6 +8,9 @@ 0x03 0x1 0x1 + 1 + 0x100 + 0x100 10,20 diff --git a/test/regress/interrupt.xml b/test/regress/interrupt.xml index f441b09..092b5c9 100644 --- a/test/regress/interrupt.xml +++ b/test/regress/interrupt.xml @@ -8,6 +8,11 @@ 1 0x8010 0x01 + 0x01 + 0x1000 + 1 + 0x100 + 0x100 1 diff --git a/test/regress/main.xml b/test/regress/main.xml index aec1875..29595bd 100644 --- a/test/regress/main.xml +++ b/test/regress/main.xml @@ -6,6 +6,11 @@ 1 0x8010 0x01 + 0x01 + 0x1000 + 1 + 0x100 + 0x100 10,20 diff --git a/test/regress/parity_addr.xml b/test/regress/parity_addr.xml index 658e7cb..446cf8d 100644 --- a/test/regress/parity_addr.xml +++ b/test/regress/parity_addr.xml @@ -6,6 +6,11 @@ 1 0x8010 0x01 + 0x01 + 0x1000 + 1 + 0x100 + 0x100 100 diff --git a/test/regress/parity_code.xml b/test/regress/parity_code.xml index 2e30793..c2fb816 100644 --- a/test/regress/parity_code.xml +++ b/test/regress/parity_code.xml @@ -6,6 +6,11 @@ 1 0x8010 0x01 + 0x01 + 0x1000 + 1 + 0x100 + 0x100 100 diff --git a/test/regress/parity_mmio.xml b/test/regress/parity_mmio.xml index 5336302..db45bdb 100644 --- a/test/regress/parity_mmio.xml +++ b/test/regress/parity_mmio.xml @@ -6,6 +6,11 @@ 1 0x8010 0x01 + 0x01 + 0x1000 + 1 + 0x100 + 0x100 100 diff --git a/test/regress/parity_read.xml b/test/regress/parity_read.xml index 27d4b58..2fb1a82 100644 --- a/test/regress/parity_read.xml +++ b/test/regress/parity_read.xml @@ -6,6 +6,11 @@ 1 0x8010 0x01 + 0x01 + 0x1000 + 1 + 0x100 + 0x100 100 diff --git a/test/regress/parity_tag.xml b/test/regress/parity_tag.xml index b6cf0a1..0eef127 100644 --- a/test/regress/parity_tag.xml +++ b/test/regress/parity_tag.xml @@ -6,6 +6,11 @@ 1 0x8010 0x01 + 0x01 + 0x1000 + 1 + 0x100 + 0x100 100 From abde71ea3a35572c540d6294331e71dd49953a09 Mon Sep 17 00:00:00 2001 From: Helena Purgatorio Date: Fri, 22 Apr 2016 15:52:14 -0500 Subject: [PATCH 05/10] Fixed problem where cr_vendor & cr_device values were swapped. --- pslse/mmio.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pslse/mmio.c b/pslse/mmio.c index 672907c..649160f 100644 --- a/pslse/mmio.c +++ b/pslse/mmio.c @@ -225,8 +225,10 @@ int read_descriptor(struct mmio *mmio, pthread_mutex_t * lock) // Store data from reads _wait_for_done(&(eventdevven->state), lock); debug_msg("XXXX: DATA: = %016x\n", eventdevven->data); - cr_array->cr_device = (uint16_t) (eventdevven->data >> 48) & 0xffffl; - cr_array->cr_vendor = (uint16_t) (eventdevven->data >> 32) & 0xffffl; + cr_array->cr_vendor = (uint16_t) (eventdevven->data >> 48) & 0xffffl; + //cr_array->cr_device = (uint16_t) (eventdevven->data >> 48) & 0xffffl; + //cr_array->cr_vendor = (uint16_t) (eventdevven->data >> 32) & 0xffffl; + cr_array->cr_device = (uint16_t) (eventdevven->data >> 32) & 0xffffl; debug_msg("%x:%x CR dev & vendor", cr_array->cr_device, cr_array->cr_vendor); free(eventdevven); debug_msg("%x:%x CR dev & vendor swapped", ntohs(cr_array->cr_device),ntohs(cr_array->cr_vendor)); From 4b9a3252fb51584e87fb3672ddea3d66fc8389b6 Mon Sep 17 00:00:00 2001 From: Helena Purgatorio Date: Tue, 3 May 2016 13:17:42 -0500 Subject: [PATCH 06/10] Update debug.log reader code to interpret LLCMDs correctly. --- common/debug.c | 39 +++++++++++++++++++++++++++++++++++ common/debug.h | 4 ++++ debug/main.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++ pslse/job.c | 6 ++++-- 4 files changed, 103 insertions(+), 2 deletions(-) diff --git a/common/debug.c b/common/debug.c index 1210849..ab3c0af 100644 --- a/common/debug.c +++ b/common/debug.c @@ -131,6 +131,8 @@ static void _debug_send_32_32(FILE * fp, DBG_HEADER header, uint32_t value0, } } + + static void _debug_send_id_8_16(FILE * fp, DBG_HEADER header, uint8_t id, uint8_t value0, uint16_t value1) { @@ -156,6 +158,33 @@ static void _debug_send_id_8_16(FILE * fp, DBG_HEADER header, uint8_t id, } } +static void _debug_send_id_32_64(FILE * fp, DBG_HEADER header, uint8_t id, + uint32_t value0, uint64_t value1) +{ + char *buffer; + size_t size; + int offset; + + offset = 0; + header = adjust_header(header); + size = + sizeof(DBG_HEADER) + sizeof(id) + sizeof(value0) + sizeof(value1); + if ((buffer = (char *)malloc(size)) != NULL) { + memcpy(buffer, (char *)&header, sizeof(DBG_HEADER)); + offset += sizeof(header); + buffer[offset] = id; + offset += sizeof(id); + value0 = htonl(value0); + memcpy(buffer + offset, (char *)&value0, sizeof(value0)); + offset += sizeof(value0); + value1 = htonll(value1); + memcpy(buffer + offset, (char *)&value1, sizeof(value1)); + fwrite(buffer, size, 1, fp); + free(buffer); + } +} + + static void _debug_send_id_8_16_16(FILE * fp, DBG_HEADER header, uint8_t id, uint8_t value0, uint16_t value1, uint16_t value2) @@ -313,6 +342,16 @@ void debug_job_send(FILE * fp, uint8_t id, uint32_t code) _debug_send_id_32(fp, DBG_HEADER_JOB_SEND, id, code); } +void debug_pe_add(FILE * fp, uint8_t id, uint32_t code, uint64_t addr) +{ + _debug_send_id_32_64(fp, DBG_HEADER_PE_ADD, id, code, addr); +} + +void debug_pe_send(FILE * fp, uint8_t id, uint32_t code, uint64_t addr) +{ + _debug_send_id_32_64(fp, DBG_HEADER_PE_SEND, id, code, addr); +} + void debug_job_aux2(FILE * fp, uint8_t id, uint8_t aux2) { _debug_send_id_8(fp, DBG_HEADER_JOB_AUX2, id, aux2); diff --git a/common/debug.h b/common/debug.h index 0224654..408cf81 100644 --- a/common/debug.h +++ b/common/debug.h @@ -45,6 +45,8 @@ typedef uint8_t DBG_HEADER; #define DBG_HEADER_CMD_BUFFER_WRITE 0x14 #define DBG_HEADER_CMD_BUFFER_READ 0x15 #define DBG_HEADER_CMD_RESPONSE 0x16 +#define DBG_HEADER_PE_ADD 0x18 +#define DBG_HEADER_PE_SEND 0x19 #define DBG_AUX2_DONE 0x80 #define DBG_AUX2_RUNNING 0x40 @@ -88,6 +90,8 @@ void debug_context_remove(FILE * fp, uint8_t id, uint16_t context); void debug_job_add(FILE * fp, uint8_t id, uint32_t code); void debug_job_send(FILE * fp, uint8_t id, uint32_t code); void debug_job_aux2(FILE * fp, uint8_t id, uint8_t aux2); +void debug_pe_add(FILE * fp, uint8_t id, uint32_t code, uint64_t addr); +void debug_pe_send(FILE * fp, uint8_t id, uint32_t code, uint64_t addr); void debug_parm(FILE * fp, uint32_t parm, uint32_t value); void debug_mmio_map(FILE * fp, uint8_t id, uint16_t context); void debug_mmio_add(FILE * fp, uint8_t id, uint16_t context, uint8_t rnw, diff --git a/debug/main.c b/debug/main.c index df158e3..5ac4c1f 100644 --- a/debug/main.c +++ b/debug/main.c @@ -55,6 +55,9 @@ static int _parse_parm(FILE * fp) case DBG_PARM_TIMEOUT: printf("PARM:TIMEOUT=%d\n", value); break; + case DBG_PARM_CREDITS: + printf("PARM:CREDITS=%d\n", value); + break; case DBG_PARM_RESP_PERCENT: printf("PARM:REPSONSE_PERCENT=%d\n", value); break; @@ -164,6 +167,54 @@ static int _parse_job(FILE * fp, DBG_HEADER header) return 0; } +static int _parse_pe(FILE * fp, DBG_HEADER header) +{ + uint32_t code; + uint8_t id; + uint64_t addr; + char *name; + + if (debug_get_8(fp, &id) < 1) + return -1; + if (debug_get_32(fp, &code) < 1) + return -1; + name = _afu_name(id); + + printf("%s:JOB: ", name); + switch (header) { + case DBG_HEADER_PE_ADD: + printf("Added "); + break; + case DBG_HEADER_PE_SEND: + printf("Sent "); + break; + default: + free(name); + return -1; + } + printf("LLCMD "); + if (debug_get_64(fp, &addr) < 1) { + printf("No LLCMD addr?"); + return -1; + } + switch (addr) { + case PSL_LLCMD_ADD: + printf("ADD"); + break; + case PSL_LLCMD_REMOVE: + printf("REMOVE"); + break; + case PSL_LLCMD_TERMINATE: + printf("TERMINATE"); + break; + default: + printf(" Unknown LLCMD:0x%016"PRIx64, addr); + } + printf("\n"); + free(name); + return 0; +} + static int _parse_map(FILE * fp, DBG_HEADER header) { uint16_t context; @@ -583,6 +634,11 @@ int main(int argc, char **argv) if (_parse_job(fp, header) < 0) return -1; break; + case DBG_HEADER_PE_ADD: + case DBG_HEADER_PE_SEND: + if (_parse_pe(fp, header) < 0) + return -1; + break; case DBG_HEADER_JOB_AUX2: if (_parse_aux(fp, header) < 0) return -1; diff --git a/pslse/job.c b/pslse/job.c index 9dd234f..6a90755 100644 --- a/pslse/job.c +++ b/pslse/job.c @@ -90,7 +90,8 @@ struct job_event *add_pe(struct job *job, uint32_t code, uint64_t addr) job->afu_name, job->dbg_id, event, tail ); // DEBUG - debug_job_add(job->dbg_fp, job->dbg_id, event->code); + //debug_job_add(job->dbg_fp, job->dbg_id, event->code); + debug_pe_add(job->dbg_fp, job->dbg_id, event->code, addr); return event; } @@ -141,7 +142,8 @@ void send_pe(struct job *job) event->code, event->addr); // DEBUG - debug_job_send(job->dbg_fp, job->dbg_id, event->code); + //debug_job_send(job->dbg_fp, job->dbg_id, event->code); + debug_pe_send(job->dbg_fp, job->dbg_id, event->code, event->addr); } return; default: From e865af7b9ce7319027ba6fd4045a9fa7cb93c056 Mon Sep 17 00:00:00 2001 From: Helena Purgatorio Date: Fri, 6 May 2016 11:15:59 -0500 Subject: [PATCH 07/10] Add initial support for cxl_set_irqs_max (doesn't notify AFU of lower limit yet) --- libcxl/libcxl.c | 53 +++++++++++++++++++++++++++++++++++-------------- libcxl/libcxl.h | 2 +- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/libcxl/libcxl.c b/libcxl/libcxl.c index 2eb3b9c..07ca371 100644 --- a/libcxl/libcxl.c +++ b/libcxl/libcxl.c @@ -601,7 +601,7 @@ static void *_psl_loop(void *ptr) break; case PSLSE_QUERY: { size = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint16_t) + - sizeof(uint64_t) + sizeof(uint64_t) + sizeof(size_t) + + sizeof(uint64_t) + sizeof(uint64_t) + sizeof(uint64_t) + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t); if (get_bytes_silent(afu->fd, size, buffer, 1000, 0) < 0) { @@ -609,18 +609,19 @@ static void *_psl_loop(void *ptr) _all_idle(afu); break; } - memcpy((char *)&value, (char *)&(buffer[1]), 2); - afu->irqs_min = (long)ntohs(value); - memcpy((char *)&value, (char *)&(buffer[3]), 2); - afu->irqs_max = (long)ntohs(value); - memcpy((char *)&value, (char *)&(buffer[5]), 2); - afu->modes_supported = (long)ntohs(value); - memcpy((char *)&value, (char *)&(buffer[7]), 8); - afu->mmio_len = (long)ntohs(value); - memcpy((char *)&value, (char *)&(buffer[15]), 8); - afu->mmio_off = (long)ntohs(value); - memcpy((char *)&value, (char *)&(buffer[23]), 8); - afu->eb_len = (long)ntohs(value); + memcpy((char *)&value, (char *)&(buffer[0]), 2); + afu->irqs_min = (long)(value); + memcpy((char *)&value, (char *)&(buffer[2]), 2); + afu->irqs_max = (long)(value); + memcpy((char *)&value, (char *)&(buffer[4]), 2); + afu->modes_supported = (long)(value); + memcpy((char *)&value, (char *)&(buffer[6]), 8); + afu->mmio_len = (long)(value); + memcpy((char *)&value, (char *)&(buffer[14]), 8); + afu->mmio_off = (long)(value); + memcpy((char *)&value, (char *)&(buffer[22]), 8); + //afu->eb_len = (long)ntohll(value); + afu->eb_len = (long)(value); memcpy((char *)&value, (char *)&(buffer[30]), 2); afu->cr_device = (long)ntohs(value); memcpy((char *)&value, (char *)&(buffer[32]), 2); @@ -1426,20 +1427,42 @@ int cxl_get_api_version_compatible(struct cxl_afu_h *afu, long *valp) int cxl_get_irqs_max(struct cxl_afu_h *afu, long *valp) { - if ((afu == NULL) || (afu->opened)) + if (!afu) { + warn_msg("cxl_get_irqs_max: No AFU given"); + errno = ENODEV; return -1; + } *valp = afu->irqs_max; return 0; } int cxl_get_irqs_min(struct cxl_afu_h *afu, long *valp) { - if ((afu == NULL) || (afu->opened)) + if (!afu) { + warn_msg("cxl_get_irqs_min: No AFU given"); + errno = ENODEV; return -1; + } *valp = afu->irqs_min; return 0; } +int cxl_set_irqs_max(struct cxl_afu_h *afu, long value) +{ + if (!afu) { + warn_msg("cxl_set_irqs_max: No AFU given"); + errno = ENODEV; + return -1; + } + if (value > afu->irqs_max) + warn_msg("cxl_set_irqs_max: value is greater than limit, ignoring \n"); + else + afu->irqs_max = value; + //TODO Send the new irqs_max value back to psl's client struct + return 0; +} + + int cxl_event_pending(struct cxl_afu_h *afu) { if (afu->events[0] != NULL) diff --git a/libcxl/libcxl.h b/libcxl/libcxl.h index a60a4bb..3809348 100644 --- a/libcxl/libcxl.h +++ b/libcxl/libcxl.h @@ -142,7 +142,7 @@ enum cxl_image { int cxl_get_api_version(struct cxl_afu_h *afu, long *valp); int cxl_get_api_version_compatible(struct cxl_afu_h *afu, long *valp); int cxl_get_irqs_max(struct cxl_afu_h *afu, long *valp); -//int cxl_set_irqs_max(struct cxl_afu_h *afu, long value); +int cxl_set_irqs_max(struct cxl_afu_h *afu, long value); int cxl_get_irqs_min(struct cxl_afu_h *afu, long *valp); int cxl_get_mmio_size(struct cxl_afu_h *afu, long *valp); int cxl_get_mode(struct cxl_afu_h *afu, long *valp); From 795b32dab61a363ba5e968e66e2e41a75ad0d118 Mon Sep 17 00:00:00 2001 From: Helena Purgatorio Date: Tue, 17 May 2016 11:07:07 -0500 Subject: [PATCH 08/10] Updates to AFU.cpp to send j_cack back for LLCMDs. Updates to libcxl_get_api & api_compatible (fix incorrect afu open check). Add fake sigbus handler --- libcxl/libcxl.c | 107 ++++++++++++++++++++++++++++++++++++++++++++++- libcxl/libcxl.h | 23 +++++++++- test/afu/AFU.cpp | 34 +++++++++++++++ 3 files changed, 160 insertions(+), 4 deletions(-) diff --git a/libcxl/libcxl.c b/libcxl/libcxl.c index 07ca371..511b1c7 100644 --- a/libcxl/libcxl.c +++ b/libcxl/libcxl.c @@ -1411,7 +1411,7 @@ int cxl_afu_fd(struct cxl_afu_h *afu) int cxl_get_api_version(struct cxl_afu_h *afu, long *valp) { - if ((afu == NULL) || (afu->opened)) + if ((afu == NULL) || (!afu->opened)) return -1; *valp = API_VERSION; return 0; @@ -1419,7 +1419,7 @@ int cxl_get_api_version(struct cxl_afu_h *afu, long *valp) int cxl_get_api_version_compatible(struct cxl_afu_h *afu, long *valp) { - if ((afu == NULL) || (afu->opened)) + if ((afu == NULL) || (!afu->opened)) return -1; *valp = API_VERSION_COMPATIBLE; return 0; @@ -1904,4 +1904,107 @@ int cxl_get_psl_revision(struct cxl_adapter_h *adapter, long *valp) return 0; } +inline +int cxl_afu_attach_work(struct cxl_afu_h *afu, + struct cxl_ioctl_start_work *work) +{ + if (afu == NULL || work == NULL) { + errno = EINVAL; + return -1; + } + afu->int_req.max = work->num_interrupts; +; + return cxl_afu_attach(afu, work->work_element_descriptor); +} + +inline +struct cxl_ioctl_start_work *cxl_work_alloc() +{ + return calloc(1, sizeof(struct cxl_ioctl_start_work)); +} + +inline +int cxl_work_free(struct cxl_ioctl_start_work *work) +{ + if (work == NULL) { + errno = EINVAL; + return -1; + } + free(work); + return 0; +} + +inline +int cxl_work_get_amr(struct cxl_ioctl_start_work *work, __u64 *valp) +{ + if (work == NULL) { + errno = EINVAL; + return -1; + } + *valp = work->amr; + return 0; +} + +inline +int cxl_work_get_num_irqs(struct cxl_ioctl_start_work *work, __s16 *valp) +{ + if (work == NULL) { + errno = EINVAL; + return -1; + } + *valp = work->num_interrupts; + return 0; +} + +inline +int cxl_work_get_wed(struct cxl_ioctl_start_work *work, __u64 *valp) +{ + if (work == NULL) { + errno = EINVAL; + return -1; + } + *valp = work->work_element_descriptor; + return 0; +} + +inline +int cxl_work_set_amr(struct cxl_ioctl_start_work *work, __u64 amr) +{ + if (work == NULL) { + errno = EINVAL; + return -1; + } + work->amr = amr; + if (amr) + work->flags |= CXL_START_WORK_AMR; + else + work->flags &= ~(CXL_START_WORK_AMR); + return 0; +} + +inline +int cxl_work_set_num_irqs(struct cxl_ioctl_start_work *work, __s16 irqs) +{ + if (work == NULL) { + errno = EINVAL; + return -1; + } + work->num_interrupts = irqs; + if (irqs >= 0) + work->flags |= CXL_START_WORK_NUM_IRQS; + else + work->flags &= ~(CXL_START_WORK_NUM_IRQS); + return 0; +} + +inline +int cxl_work_set_wed(struct cxl_ioctl_start_work *work, __u64 wed) +{ + if (work == NULL) { + errno = EINVAL; + return -1; + } + work->work_element_descriptor = wed; + return 0; +} diff --git a/libcxl/libcxl.h b/libcxl/libcxl.h index 3809348..739ee7e 100644 --- a/libcxl/libcxl.h +++ b/libcxl/libcxl.h @@ -32,6 +32,7 @@ */ struct cxl_adapter_h; struct cxl_afu_h; +struct cxl_ioctl_start_work; /* * Adapter Enumeration @@ -92,9 +93,22 @@ int cxl_afu_opened(struct cxl_afu_h *afu); /* * Attach AFU context to this process */ +struct cxl_ioctl_start_work *cxl_work_alloc(void); +int cxl_work_free(struct cxl_ioctl_start_work *work); +int cxl_work_get_amr(struct cxl_ioctl_start_work *work, __u64 *valp); +int cxl_work_get_num_irqs(struct cxl_ioctl_start_work *work, __s16 *valp); +int cxl_work_get_wed(struct cxl_ioctl_start_work *work, __u64 *valp); +int cxl_work_set_amr(struct cxl_ioctl_start_work *work, __u64 amr); +int cxl_work_set_num_irqs(struct cxl_ioctl_start_work *work, __s16 num_irqs); +int cxl_work_set_wed(struct cxl_ioctl_start_work *work, __u64 wed); + +int cxl_afu_attach(struct cxl_afu_h *afu, uint64_t wed); +int cxl_afu_attach_work(struct cxl_afu_h *afu, + struct cxl_ioctl_start_work *work); + +/* Deprecated interface */ int cxl_afu_attach_full(struct cxl_afu_h *afu, uint64_t wed, uint16_t num_interrupts, uint64_t amr); -int cxl_afu_attach(struct cxl_afu_h *afu, uint64_t wed); /* * Get AFU process element @@ -209,7 +223,12 @@ int cxl_mmio_read32(struct cxl_afu_h *afu, uint64_t offset, uint32_t * data); * * Call this once per process prior to any MMIO accesses. */ -//int cxl_mmio_install_sigbus_handler(); +//use JK's temp fix for this +static inline int cxl_mmio_install_sigbus_handler(void) +{ +/* nothing to be done yet */ +return 0; +} int cxl_get_cr_device(struct cxl_afu_h *afu, long cr_num, long *valp); int cxl_get_cr_vendor(struct cxl_afu_h *afu, long cr_num, long *valp); int cxl_get_cr_class(struct cxl_afu_h *afu, long cr_num, long *valp); diff --git a/test/afu/AFU.cpp b/test/afu/AFU.cpp index dd2808c..e6040b5 100644 --- a/test/afu/AFU.cpp +++ b/test/afu/AFU.cpp @@ -320,6 +320,19 @@ AFU::resolve_control_event () if ((afu_event.job_address & 0xFFFF) == 0) { machine_controller = context_to_mc[0]; highest_priority_mc = context_to_mc.end (); + printf("PSL_LLCMD_ADD create new Machine Controller\n"); + } + printf("afu_event.job_cack_llcmd = %d\n", afu_event.job_cack_llcmd); + // set job acknowledge + afu_event.job_cack_llcmd = 1; + debug_msg("PSL_LLCMD_ADD"); + // signal job_cack_back to PSL + if (psl_afu_aux2_change + (&afu_event, afu_event.job_running, afu_event.job_done, 1, + afu_event.job_error, afu_event.job_yield, + afu_event.timebase_request, afu_event.parity_enable, + afu_event.buffer_read_latency) != PSL_SUCCESS) { + error_msg("AFU: failed to assert job_cack_llcmd for PSL_LLCMD_ADD\n"); } break; case PSL_LLCMD_TERMINATE: @@ -330,6 +343,17 @@ AFU::resolve_control_event () } context_to_mc[afu_event. job_address & 0xFFFF]->disable_all_machines (); + debug_msg("PSL_LLCMD_TERMINATE"); + afu_event.job_cack_llcmd = 1; + // signal job_cack_back to PSL + if (psl_afu_aux2_change + (&afu_event, afu_event.job_running, afu_event.job_done, 1, + afu_event.job_error, afu_event.job_yield, + afu_event.timebase_request, afu_event.parity_enable, + afu_event.buffer_read_latency) != PSL_SUCCESS) { + error_msg("AFU: failed to assert job_cack_llcmd for PSL_LLCMD_TERMINATE\n"); + } + break; case PSL_LLCMD_REMOVE: //TODO also make sure ADD->TERMINATE->REMOVE @@ -348,6 +372,16 @@ AFU::resolve_control_event () delete context_to_mc[afu_event.job_address & 0xFFFF]; context_to_mc.erase (afu_event.job_address & 0xFFFF); + debug_msg("PSL_LLCMD_REMOVE"); + afu_event.job_cack_llcmd = 1; + // signal job_cack_back to PSL + if (psl_afu_aux2_change + (&afu_event, afu_event.job_running, afu_event.job_done, 1, + afu_event.job_error, afu_event.job_yield, + afu_event.timebase_request, afu_event.parity_enable, + afu_event.buffer_read_latency) != PSL_SUCCESS) { + error_msg("AFU: failed to assert job_cack_llcmd for PSL_LLCMD_REMOVE\n"); + } break; default: error_msg ("AFU: this LLCMD code is currently not supported"); From 0ceaf48f9275d5658729f693a266cb55c294e5be Mon Sep 17 00:00:00 2001 From: Helena Purgatorio Date: Fri, 27 May 2016 10:32:33 -0500 Subject: [PATCH 09/10] Update testAFU with new parm (jerror) to allow testing of AFU generated errors.Update psl.c w.fix to _handle_afu for directed & dedicated warning msg --- pslse/psl.c | 5 ++++- test/afu/AFU.cpp | 32 ++++++++++++++++++++++++-------- test/afu/AFU.h | 3 ++- test/afu/Descriptor.h | 2 ++ test/afu/main.cpp | 15 +++++++++++++-- test/tests/interrupt1.c | 11 +++++------ 6 files changed, 50 insertions(+), 18 deletions(-) diff --git a/pslse/psl.c b/pslse/psl.c index dc846e1..5e4e32e 100644 --- a/pslse/psl.c +++ b/pslse/psl.c @@ -262,6 +262,8 @@ int _handle_aux2(struct psl *psl, uint32_t * parity, uint32_t * latency, debug_msg("%s:_handle_aux2: JOB done", job->afu_name); dbg_aux2 |= DBG_AUX2_DONE; *error = job_error; + //debug_msg("%s,%d:_handle_aux2, jerror is %x ", + // job->afu_name, job->dbg_id, job_error ); if (job->job != NULL) { event = job->job; // Is job_done for reset or start? @@ -406,6 +408,7 @@ static void _handle_afu(struct psl *psl) buffer[0] = PSLSE_AFU_ERROR; error = htonll(error); memcpy((char *)&(buffer[1]), (char *)&error, sizeof(error)); + warn_msg("%s: Received JERROR: 0x%016"PRIx64" in afu-dedicated mode", psl->name, error); if (put_bytes (client->fd, size, buffer, psl->dbg_fp, psl->dbg_id, 0) < 0) { @@ -420,7 +423,7 @@ static void _handle_afu(struct psl *psl) for (i = 0; i < psl->max_clients; i++) { if (psl->client[i] == NULL) continue; - client_drop(client, PSL_IDLE_CYCLES, CLIENT_NONE); + client_drop(psl->client[i], PSL_IDLE_CYCLES, CLIENT_NONE); } } } diff --git a/test/afu/AFU.cpp b/test/afu/AFU.cpp index e6040b5..e268d95 100644 --- a/test/afu/AFU.cpp +++ b/test/afu/AFU.cpp @@ -15,7 +15,7 @@ using std::vector; #define CONTEXT_SIZE 0x400 #define CONTEXT_MASK (CONTEXT_SIZE - 1) -AFU::AFU (int port, string filename, bool parity): +AFU::AFU (int port, string filename, bool parity, bool jerror): descriptor (filename), context_to_mc () { @@ -31,6 +31,10 @@ AFU::AFU (int port, string filename, bool parity): error_msg ("AFU: failed to set parity_enable and latency"); } + if (jerror) + set_jerror_not_run = true; + else + set_jerror_not_run = false; set_seed (); state = IDLE; @@ -191,7 +195,6 @@ AFU::start () debug_msg ("AFU: machine completed"); reset_machine_controllers (); - if (psl_afu_aux2_change (&afu_event, 0, 1, afu_event.job_cack_llcmd, afu_event.job_error, afu_event.job_yield, @@ -293,17 +296,30 @@ AFU::resolve_control_event () error_msg ("AFU: Parity error in job_address"); } - // assert job_running + // assert job_running unless jerror set, then assert job_done & job_error + if (set_jerror_not_run) { + // HPADD assert JERROR too if (psl_afu_aux2_change + (&afu_event, 0, 1, afu_event.job_cack_llcmd, + 0xdeaddeaddeaddead, afu_event.job_yield, + afu_event.timebase_request, afu_event.parity_enable, + afu_event.buffer_read_latency) != PSL_SUCCESS) { + error_msg ("AFU: failed to assert job_error"); + } + state = IDLE; + debug_msg ("AFU: AFU REPORTING ERROR "); + } + else { + if (psl_afu_aux2_change (&afu_event, 1, afu_event.job_done, afu_event.job_cack_llcmd, afu_event.job_error, afu_event.job_yield, afu_event.timebase_request, afu_event.parity_enable, afu_event.buffer_read_latency) != PSL_SUCCESS) { - error_msg ("AFU: failed to assert job_running"); - } - - state = RUNNING; - debug_msg ("AFU: AFU RUNNING"); + error_msg ("AFU: failed to assert job_running"); + } + state = RUNNING; + debug_msg ("AFU: AFU RUNNING"); + } } // command for directed mode else if (afu_event.job_code == PSL_JOB_LLCMD) { diff --git a/test/afu/AFU.h b/test/afu/AFU.h index 3050c4f..dca9b4b 100644 --- a/test/afu/AFU.h +++ b/test/afu/AFU.h @@ -49,11 +49,12 @@ class AFU void reset_machine_controllers (); bool get_mmio_read_parity (); + bool set_jerror_not_run; public: /* constructor sets up descriptor from config file, establishes server socket connection and waits for client to connect */ - AFU (int port, std::string filename, bool parity); + AFU (int port, std::string filename, bool parity, bool jerror); /* starts the main loop of the afu test platform */ void start (); diff --git a/test/afu/Descriptor.h b/test/afu/Descriptor.h index 7728cfe..0480836 100644 --- a/test/afu/Descriptor.h +++ b/test/afu/Descriptor.h @@ -49,6 +49,8 @@ class Descriptor uint64_t get_reg (uint32_t word_address, uint32_t mmio_double) const; + bool set_jerror () const; + bool is_dedicated () const; bool is_directed () const; diff --git a/test/afu/main.cpp b/test/afu/main.cpp index b5ba87f..3735fc1 100644 --- a/test/afu/main.cpp +++ b/test/afu/main.cpp @@ -11,7 +11,7 @@ main (int argc, char *argv[]) { if (argc < 3) { fprintf (stderr, - "Not eneough arguments. Usage: ./afu port_number descriptor_file [parity]\n"); + "Not enough arguments. Usage: ./afu port_number descriptor_file [parity] [jerror]\n"); exit (1); } @@ -20,6 +20,7 @@ main (int argc, char *argv[]) string descriptor_file (argv[2]); bool parity = false; + bool jerror = false; stringstream ss; @@ -31,7 +32,17 @@ main (int argc, char *argv[]) parity = true; } - AFU afu (port, descriptor_file, parity); + if (argc == 4 && string (argv[3]) == "jerror") { + printf ("MAIN: AFU will send jerror not running\n"); + jerror = true; + } + + if (argc == 5 && string (argv[4]) == "jerror") { + printf ("MAIN: AFU will send jerror not running\n"); + jerror = true; + } + + AFU afu (port, descriptor_file, parity, jerror); afu.start (); debug_msg ("main: AFU quitting"); diff --git a/test/tests/interrupt1.c b/test/tests/interrupt1.c index 80d0217..1535b0e 100644 --- a/test/tests/interrupt1.c +++ b/test/tests/interrupt1.c @@ -96,12 +96,11 @@ int main(int argc, char *argv[]) perror("cxl_afu_open_h"); goto done; } -// FIXME: cxl_get_irqs_max() is broken! -// if (cxl_get_irqs_max(afu_h, &max_irqs) < 0) { -// fprintf(stderr, "\nNo AFU found!\n\n"); -// goto done; -// } - max_irqs = 2000; + if (cxl_get_irqs_max(afu_h, &max_irqs) < 0) { + fprintf(stderr, "\nNo AFU found!\n\n"); + goto done; + } +// max_irqs = 2000; irq = rand() % max_irqs; // Set WED to random value From 692b3f8a797b8abf4488baea750099bd705b444f Mon Sep 17 00:00:00 2001 From: Helena Purgatorio Date: Mon, 13 Jun 2016 13:02:12 -0500 Subject: [PATCH 10/10] Use 32-bit mmio read for config record info; fixes for directed_mmio testcase. --- pslse/mmio.c | 24 ++++++++++++++---------- pslse/psl.c | 2 +- test/regress/directed.xml | 2 +- test/tests/directed_mmio.c | 2 +- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/pslse/mmio.c b/pslse/mmio.c index 649160f..1e1a8b8 100644 --- a/pslse/mmio.c +++ b/pslse/mmio.c @@ -105,9 +105,10 @@ static struct mmio_event *_add_event(struct mmio *mmio, struct client *client, event->_next = NULL; // debug the mmio and print the input address and the translated address - /* debug_msg("_add_event: %s: WRITE%d word=0x%05x (0x%05x) data=0x%s", */ - /* mmio->afu_name, event->dw ? 64 : 32, */ - /* event->addr, addr, event->data); */ + // debug_msg("_add_event: %s: WRITE%d word=0x%05x (0x%05x) data=0x%s", + debug_msg("_add_event:: WRITE word=0x%05x (0x%05x) data=0x%x", + // mmio->afu_name, event->dw ? 64 : 32, + event->addr, addr, event->data); // Add to end of list list = &(mmio->list); @@ -219,16 +220,19 @@ int read_descriptor(struct mmio *mmio, pthread_mutex_t * lock) //struct config_record *crptr = &cr_array; mmio->desc.crptr = cr_array; // Queue mmio reads - eventdevven = _add_desc(mmio, 1, 1,crstart >> 2, 0L); - eventclass = _add_desc(mmio, 1, 1, (crstart+8) >> 2, 0L); + // Only do 32-bit mmio for config record data + //eventdevven = _add_desc(mmio, 1, 1,crstart >> 2, 0L); + eventdevven = _add_desc(mmio, 1, 0,crstart >> 2, 0L); + //eventclass = _add_desc(mmio, 1, 1, (crstart+8) >> 2, 0L); + eventclass = _add_desc(mmio, 1, 0, (crstart+8) >> 2, 0L); // Store data from reads _wait_for_done(&(eventdevven->state), lock); - debug_msg("XXXX: DATA: = %016x\n", eventdevven->data); - cr_array->cr_vendor = (uint16_t) (eventdevven->data >> 48) & 0xffffl; - //cr_array->cr_device = (uint16_t) (eventdevven->data >> 48) & 0xffffl; - //cr_array->cr_vendor = (uint16_t) (eventdevven->data >> 32) & 0xffffl; - cr_array->cr_device = (uint16_t) (eventdevven->data >> 32) & 0xffffl; + //debug_msg("XXXX: DATA: = %08x\n", eventdevven->data); + //cr_array->cr_vendor = (uint16_t) (eventdevven->data >> 48) & 0xffffl; + cr_array->cr_vendor = (uint16_t) (eventdevven->data >> 16); + //cr_array->cr_device = (uint16_t) (eventdevven->data >> 32) 0xffffl; + cr_array->cr_device = (uint16_t) (eventdevven->data ); debug_msg("%x:%x CR dev & vendor", cr_array->cr_device, cr_array->cr_vendor); free(eventdevven); debug_msg("%x:%x CR dev & vendor swapped", ntohs(cr_array->cr_device),ntohs(cr_array->cr_vendor)); diff --git a/pslse/psl.c b/pslse/psl.c index 5e4e32e..93acf50 100644 --- a/pslse/psl.c +++ b/pslse/psl.c @@ -131,7 +131,7 @@ static void _attach(struct psl *psl, struct client *client) } psl->attached_clients++; - info_msg( "Attached client context %d: current attached clients = %d\n", client->context, psl->attached_clients ); + info_msg( "Attached client context %d: current attached clients = %d: client type = %c\n", client->context, psl->attached_clients, client->type ); // for master and slave send llcmd add // master "wed" is 0x0005000000000000 can actually use client->context here as well since context = 0 diff --git a/test/regress/directed.xml b/test/regress/directed.xml index 42e5d9f..81ca4e5 100644 --- a/test/regress/directed.xml +++ b/test/regress/directed.xml @@ -7,7 +7,7 @@ 0x8004 0x03 0x1 - 0x1 + 0x1000 1 0x100 0x100 diff --git a/test/tests/directed_mmio.c b/test/tests/directed_mmio.c index 2ba0151..5d11aca 100644 --- a/test/tests/directed_mmio.c +++ b/test/tests/directed_mmio.c @@ -166,7 +166,7 @@ int main(int argc, char *argv[]) goto done; } - sleep(60); // pause to let me run another version of this... + sleep(10); // pause to let me run another version of this... // Check WED is found if (wed_check != wed) {