Index: projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/krping.c =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/krping.c (nonexistent) +++ projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/krping.c (revision 330503) @@ -0,0 +1,3427 @@ +/* + * Copyright (c) 2005 Ammasso, Inc. All rights reserved. + * Copyright (c) 2006-2009 Open Grid Computing, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include "krping.h" +#include "getopt.h" + +extern int krping_debug; +#define DEBUG_LOG(cb, x...) if (krping_debug) log(LOG_INFO, x) +#define PRINTF(cb, x...) log(LOG_INFO, x) +#define BIND_INFO 1 + +MODULE_AUTHOR("Steve Wise"); +MODULE_DESCRIPTION("RDMA ping client/server"); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_VERSION(krping, 1); +MODULE_DEPEND(krping, linuxkpi, 1, 1, 1); + +static __inline uint64_t +get_cycles(void) +{ + uint32_t low, high; + __asm __volatile("rdtsc" : "=a" (low), "=d" (high)); + return (low | ((u_int64_t)high << 32)); +} + +typedef uint64_t cycles_t; + +enum mem_type { + DMA = 1, + FASTREG = 2, + MW = 3, + MR = 4 +}; + +static const struct krping_option krping_opts[] = { + {"count", OPT_INT, 'C'}, + {"size", OPT_INT, 'S'}, + {"addr", OPT_STRING, 'a'}, + {"addr6", OPT_STRING, 'A'}, + {"port", OPT_INT, 'p'}, + {"verbose", OPT_NOPARAM, 'v'}, + {"validate", OPT_NOPARAM, 'V'}, + {"server", OPT_NOPARAM, 's'}, + {"client", OPT_NOPARAM, 'c'}, + {"mem_mode", OPT_STRING, 'm'}, + {"server_inv", OPT_NOPARAM, 'I'}, + {"wlat", OPT_NOPARAM, 'l'}, + {"rlat", OPT_NOPARAM, 'L'}, + {"bw", OPT_NOPARAM, 'B'}, + {"duplex", OPT_NOPARAM, 'd'}, + {"txdepth", OPT_INT, 'T'}, + {"poll", OPT_NOPARAM, 'P'}, + {"local_dma_lkey", OPT_NOPARAM, 'Z'}, + {"read_inv", OPT_NOPARAM, 'R'}, + {"fr", OPT_INT, 'f'}, + {NULL, 0, 0} +}; + +#define htonll(x) cpu_to_be64((x)) +#define ntohll(x) cpu_to_be64((x)) + +static struct mutex krping_mutex; + +/* + * List of running krping threads. + */ +static LIST_HEAD(krping_cbs); + +/* + * krping "ping/pong" loop: + * client sends source rkey/addr/len + * server receives source rkey/add/len + * server rdma reads "ping" data from source + * server sends "go ahead" on rdma read completion + * client sends sink rkey/addr/len + * server receives sink rkey/addr/len + * server rdma writes "pong" data to sink + * server sends "go ahead" on rdma write completion + * + */ + +/* + * These states are used to signal events between the completion handler + * and the main client or server thread. + * + * Once CONNECTED, they cycle through RDMA_READ_ADV, RDMA_WRITE_ADV, + * and RDMA_WRITE_COMPLETE for each ping. + */ +enum test_state { + IDLE = 1, + CONNECT_REQUEST, + ADDR_RESOLVED, + ROUTE_RESOLVED, + CONNECTED, + RDMA_READ_ADV, + RDMA_READ_COMPLETE, + RDMA_WRITE_ADV, + RDMA_WRITE_COMPLETE, + ERROR +}; + +struct krping_rdma_info { + uint64_t buf; + uint32_t rkey; + uint32_t size; +}; + +/* + * Default max buffer size for IO... + */ +#define RPING_BUFSIZE 128*1024 +#define RPING_SQ_DEPTH 64 + +/* + * Control block struct. + */ +struct krping_cb { + void *cookie; + int server; /* 0 iff client */ + struct ib_cq *cq; + struct ib_pd *pd; + struct ib_qp *qp; + + enum mem_type mem; + struct ib_mr *dma_mr; + + struct ib_fast_reg_page_list *page_list; + int page_list_len; + struct ib_send_wr fastreg_wr; + struct ib_send_wr invalidate_wr; + struct ib_mr *fastreg_mr; + int server_invalidate; + int read_inv; + u8 key; + + struct ib_mw *mw; + struct ib_mw_bind bind_attr; + + struct ib_recv_wr rq_wr; /* recv work request record */ + struct ib_sge recv_sgl; /* recv single SGE */ + struct krping_rdma_info recv_buf;/* malloc'd buffer */ + u64 recv_dma_addr; + DECLARE_PCI_UNMAP_ADDR(recv_mapping) + struct ib_mr *recv_mr; + + struct ib_send_wr sq_wr; /* send work requrest record */ + struct ib_sge send_sgl; + struct krping_rdma_info send_buf;/* single send buf */ + u64 send_dma_addr; + DECLARE_PCI_UNMAP_ADDR(send_mapping) + struct ib_mr *send_mr; + + struct ib_send_wr rdma_sq_wr; /* rdma work request record */ + struct ib_sge rdma_sgl; /* rdma single SGE */ + char *rdma_buf; /* used as rdma sink */ + u64 rdma_dma_addr; + DECLARE_PCI_UNMAP_ADDR(rdma_mapping) + struct ib_mr *rdma_mr; + + uint32_t remote_rkey; /* remote guys RKEY */ + uint64_t remote_addr; /* remote guys TO */ + uint32_t remote_len; /* remote guys LEN */ + + char *start_buf; /* rdma read src */ + u64 start_dma_addr; + DECLARE_PCI_UNMAP_ADDR(start_mapping) + struct ib_mr *start_mr; + + enum test_state state; /* used for cond/signalling */ + wait_queue_head_t sem; + struct krping_stats stats; + + uint16_t port; /* dst port in NBO */ + union { + struct in_addr v4; + struct in6_addr v6; + } addr; /* dst addr in NBO */ + int addr_type; /* AF_INET or AF_INET6 */ + char *addr_str; /* dst addr string */ + int verbose; /* verbose logging */ + int count; /* ping count */ + int size; /* ping data size */ + int validate; /* validate ping data */ + int wlat; /* run wlat test */ + int rlat; /* run rlat test */ + int bw; /* run bw test */ + int duplex; /* run bw full duplex test */ + int poll; /* poll or block for rlat test */ + int txdepth; /* SQ depth */ + int local_dma_lkey; /* use 0 for lkey */ + int frtest; /* fastreg test */ + int testnum; + + /* CM stuff */ + struct rdma_cm_id *cm_id; /* connection on client side,*/ + /* listener on server side. */ + struct rdma_cm_id *child_cm_id; /* connection on server side */ + struct list_head list; +}; + +static int krping_cma_event_handler(struct rdma_cm_id *cma_id, + struct rdma_cm_event *event) +{ + int ret; + struct krping_cb *cb = cma_id->context; + + DEBUG_LOG(cb, "cma_event type %d cma_id %p (%s)\n", event->event, + cma_id, (cma_id == cb->cm_id) ? "parent" : "child"); + + switch (event->event) { + case RDMA_CM_EVENT_ADDR_RESOLVED: + cb->state = ADDR_RESOLVED; + ret = rdma_resolve_route(cma_id, 2000); + if (ret) { + PRINTF(cb, "rdma_resolve_route error %d\n", ret); + wake_up_interruptible(&cb->sem); + } + break; + + case RDMA_CM_EVENT_ROUTE_RESOLVED: + cb->state = ROUTE_RESOLVED; + cb->child_cm_id = cma_id; + wake_up_interruptible(&cb->sem); + break; + + case RDMA_CM_EVENT_CONNECT_REQUEST: + if (cb->state == IDLE) { + cb->state = CONNECT_REQUEST; + cb->child_cm_id = cma_id; + } else { + PRINTF(cb, "Received connection request in wrong state" + " (%d)\n", cb->state); + } + DEBUG_LOG(cb, "child cma %p\n", cb->child_cm_id); + wake_up_interruptible(&cb->sem); + break; + + case RDMA_CM_EVENT_ESTABLISHED: + DEBUG_LOG(cb, "ESTABLISHED\n"); + if (!cb->server) { + cb->state = CONNECTED; + } + wake_up_interruptible(&cb->sem); + break; + + case RDMA_CM_EVENT_ADDR_ERROR: + case RDMA_CM_EVENT_ROUTE_ERROR: + case RDMA_CM_EVENT_CONNECT_ERROR: + case RDMA_CM_EVENT_UNREACHABLE: + case RDMA_CM_EVENT_REJECTED: + PRINTF(cb, "cma event %d, error %d\n", event->event, + event->status); + cb->state = ERROR; + wake_up_interruptible(&cb->sem); + break; + + case RDMA_CM_EVENT_DISCONNECTED: + PRINTF(cb, "DISCONNECT EVENT...\n"); + cb->state = ERROR; + wake_up_interruptible(&cb->sem); + break; + + case RDMA_CM_EVENT_DEVICE_REMOVAL: + PRINTF(cb, "cma detected device removal!!!!\n"); + break; + + default: + PRINTF(cb, "oof bad type!\n"); + wake_up_interruptible(&cb->sem); + break; + } + return 0; +} + +static int server_recv(struct krping_cb *cb, struct ib_wc *wc) +{ + if (wc->byte_len != sizeof(cb->recv_buf)) { + PRINTF(cb, "Received bogus data, size %d\n", + wc->byte_len); + return -1; + } + + cb->remote_rkey = ntohl(cb->recv_buf.rkey); + cb->remote_addr = ntohll(cb->recv_buf.buf); + cb->remote_len = ntohl(cb->recv_buf.size); + DEBUG_LOG(cb, "Received rkey %x addr %llx len %d from peer\n", + cb->remote_rkey, (unsigned long long)cb->remote_addr, + cb->remote_len); + + if (cb->state <= CONNECTED || cb->state == RDMA_WRITE_COMPLETE) + cb->state = RDMA_READ_ADV; + else + cb->state = RDMA_WRITE_ADV; + + return 0; +} + +static int client_recv(struct krping_cb *cb, struct ib_wc *wc) +{ + if (wc->byte_len != sizeof(cb->recv_buf)) { + PRINTF(cb, "Received bogus data, size %d\n", + wc->byte_len); + return -1; + } + + if (cb->state == RDMA_READ_ADV) + cb->state = RDMA_WRITE_ADV; + else + cb->state = RDMA_WRITE_COMPLETE; + + return 0; +} + +static void krping_cq_event_handler(struct ib_cq *cq, void *ctx) +{ + struct krping_cb *cb = ctx; + struct ib_wc wc; + struct ib_recv_wr *bad_wr; + int ret; + + BUG_ON(cb->cq != cq); + if (cb->state == ERROR) { + PRINTF(cb, "cq completion in ERROR state\n"); + return; + } + if (!cb->wlat && !cb->rlat && !cb->bw && !cb->frtest) + ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); + while ((ret = ib_poll_cq(cb->cq, 1, &wc)) == 1) { + if (wc.status) { + if (wc.status == IB_WC_WR_FLUSH_ERR) { + DEBUG_LOG(cb, "cq flushed\n"); + continue; + } else { + PRINTF(cb, "cq completion failed with " + "wr_id %jx status %d opcode %d vender_err %x\n", + (uintmax_t)wc.wr_id, wc.status, wc.opcode, wc.vendor_err); + goto error; + } + } + + switch (wc.opcode) { + case IB_WC_SEND: + DEBUG_LOG(cb, "send completion\n"); + cb->stats.send_bytes += cb->send_sgl.length; + cb->stats.send_msgs++; + break; + + case IB_WC_RDMA_WRITE: + DEBUG_LOG(cb, "rdma write completion\n"); + cb->stats.write_bytes += cb->rdma_sq_wr.sg_list->length; + cb->stats.write_msgs++; + cb->state = RDMA_WRITE_COMPLETE; + wake_up_interruptible(&cb->sem); + break; + + case IB_WC_RDMA_READ: + DEBUG_LOG(cb, "rdma read completion\n"); + cb->stats.read_bytes += cb->rdma_sq_wr.sg_list->length; + cb->stats.read_msgs++; + cb->state = RDMA_READ_COMPLETE; + wake_up_interruptible(&cb->sem); + break; + + case IB_WC_RECV: + DEBUG_LOG(cb, "recv completion\n"); + cb->stats.recv_bytes += sizeof(cb->recv_buf); + cb->stats.recv_msgs++; + if (cb->wlat || cb->rlat || cb->bw || cb->frtest) + ret = server_recv(cb, &wc); + else + ret = cb->server ? server_recv(cb, &wc) : + client_recv(cb, &wc); + if (ret) { + PRINTF(cb, "recv wc error: %d\n", ret); + goto error; + } + + ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post recv error: %d\n", + ret); + goto error; + } + wake_up_interruptible(&cb->sem); + break; + + default: + PRINTF(cb, + "%s:%d Unexpected opcode %d, Shutting down\n", + __func__, __LINE__, wc.opcode); + goto error; + } + } + if (ret) { + PRINTF(cb, "poll error %d\n", ret); + goto error; + } + return; +error: + cb->state = ERROR; + wake_up_interruptible(&cb->sem); +} + +static int krping_accept(struct krping_cb *cb) +{ + struct rdma_conn_param conn_param; + int ret; + + DEBUG_LOG(cb, "accepting client connection request\n"); + + memset(&conn_param, 0, sizeof conn_param); + conn_param.responder_resources = 1; + conn_param.initiator_depth = 1; + + ret = rdma_accept(cb->child_cm_id, &conn_param); + if (ret) { + PRINTF(cb, "rdma_accept error: %d\n", ret); + return ret; + } + + if (!cb->wlat && !cb->rlat && !cb->bw && !cb->frtest) { + wait_event_interruptible(cb->sem, cb->state >= CONNECTED); + if (cb->state == ERROR) { + PRINTF(cb, "wait for CONNECTED state %d\n", + cb->state); + return -1; + } + } + return 0; +} + +static void krping_setup_wr(struct krping_cb *cb) +{ + cb->recv_sgl.addr = cb->recv_dma_addr; + cb->recv_sgl.length = sizeof cb->recv_buf; + if (cb->local_dma_lkey) + cb->recv_sgl.lkey = cb->qp->device->local_dma_lkey; + else if (cb->mem == DMA) + cb->recv_sgl.lkey = cb->dma_mr->lkey; + else + cb->recv_sgl.lkey = cb->recv_mr->lkey; + cb->rq_wr.sg_list = &cb->recv_sgl; + cb->rq_wr.num_sge = 1; + + cb->send_sgl.addr = cb->send_dma_addr; + cb->send_sgl.length = sizeof cb->send_buf; + if (cb->local_dma_lkey) + cb->send_sgl.lkey = cb->qp->device->local_dma_lkey; + else if (cb->mem == DMA) + cb->send_sgl.lkey = cb->dma_mr->lkey; + else + cb->send_sgl.lkey = cb->send_mr->lkey; + + cb->sq_wr.opcode = IB_WR_SEND; + cb->sq_wr.send_flags = IB_SEND_SIGNALED; + cb->sq_wr.sg_list = &cb->send_sgl; + cb->sq_wr.num_sge = 1; + + if (cb->server || cb->wlat || cb->rlat || cb->bw || cb->frtest) { + cb->rdma_sgl.addr = cb->rdma_dma_addr; + if (cb->mem == MR) + cb->rdma_sgl.lkey = cb->rdma_mr->lkey; + cb->rdma_sq_wr.send_flags = IB_SEND_SIGNALED; + cb->rdma_sq_wr.sg_list = &cb->rdma_sgl; + cb->rdma_sq_wr.num_sge = 1; + } + + switch(cb->mem) { + case FASTREG: + + /* + * A chain of 2 WRs, INVALDATE_MR + FAST_REG_MR. + * both unsignaled. The client uses them to reregister + * the rdma buffers with a new key each iteration. + */ + cb->fastreg_wr.opcode = IB_WR_FAST_REG_MR; + cb->fastreg_wr.wr.fast_reg.page_shift = PAGE_SHIFT; + cb->fastreg_wr.wr.fast_reg.length = cb->size; + cb->fastreg_wr.wr.fast_reg.page_list = cb->page_list; + cb->fastreg_wr.wr.fast_reg.page_list_len = cb->page_list_len; + + cb->invalidate_wr.next = &cb->fastreg_wr; + cb->invalidate_wr.opcode = IB_WR_LOCAL_INV; + break; + case MW: + cb->bind_attr.wr_id = 0xabbaabba; + cb->bind_attr.send_flags = 0; /* unsignaled */ +#ifdef BIND_INFO + cb->bind_attr.bind_info.length = cb->size; +#else + cb->bind_attr.length = cb->size; +#endif + break; + default: + break; + } +} + +static int krping_setup_buffers(struct krping_cb *cb) +{ + int ret; + struct ib_phys_buf buf; + u64 iovbase; + + DEBUG_LOG(cb, "krping_setup_buffers called on cb %p\n", cb); + + cb->recv_dma_addr = dma_map_single(cb->pd->device->dma_device, + &cb->recv_buf, + sizeof(cb->recv_buf), DMA_BIDIRECTIONAL); + pci_unmap_addr_set(cb, recv_mapping, cb->recv_dma_addr); + cb->send_dma_addr = dma_map_single(cb->pd->device->dma_device, + &cb->send_buf, sizeof(cb->send_buf), + DMA_BIDIRECTIONAL); + pci_unmap_addr_set(cb, send_mapping, cb->send_dma_addr); + + if (cb->mem == DMA) { + cb->dma_mr = ib_get_dma_mr(cb->pd, IB_ACCESS_LOCAL_WRITE| + IB_ACCESS_REMOTE_READ| + IB_ACCESS_REMOTE_WRITE); + if (IS_ERR(cb->dma_mr)) { + DEBUG_LOG(cb, "reg_dmamr failed\n"); + ret = PTR_ERR(cb->dma_mr); + goto bail; + } + } else { + if (!cb->local_dma_lkey) { + buf.addr = cb->recv_dma_addr; + buf.size = sizeof cb->recv_buf; + DEBUG_LOG(cb, "recv buf dma_addr %jx size %d\n", + (uintmax_t)buf.addr, (int)buf.size); + iovbase = cb->recv_dma_addr; + cb->recv_mr = ib_reg_phys_mr(cb->pd, &buf, 1, + IB_ACCESS_LOCAL_WRITE, + &iovbase); + + if (IS_ERR(cb->recv_mr)) { + DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); + ret = PTR_ERR(cb->recv_mr); + goto bail; + } + + buf.addr = cb->send_dma_addr; + buf.size = sizeof cb->send_buf; + DEBUG_LOG(cb, "send buf dma_addr %jx size %d\n", + (uintmax_t)buf.addr, (int)buf.size); + iovbase = cb->send_dma_addr; + cb->send_mr = ib_reg_phys_mr(cb->pd, &buf, 1, + 0, &iovbase); + + if (IS_ERR(cb->send_mr)) { + DEBUG_LOG(cb, "send_buf reg_mr failed\n"); + ret = PTR_ERR(cb->send_mr); + goto bail; + } + } + } + + cb->rdma_buf = kmalloc(cb->size, GFP_KERNEL); + if (!cb->rdma_buf) { + DEBUG_LOG(cb, "rdma_buf malloc failed\n"); + ret = -ENOMEM; + goto bail; + } + + cb->rdma_dma_addr = dma_map_single(cb->pd->device->dma_device, + cb->rdma_buf, cb->size, + DMA_BIDIRECTIONAL); + pci_unmap_addr_set(cb, rdma_mapping, cb->rdma_dma_addr); + if (cb->mem != DMA) { + switch (cb->mem) { + case FASTREG: + cb->page_list_len = (((cb->size - 1) & PAGE_MASK) + + PAGE_SIZE) >> PAGE_SHIFT; + cb->page_list = ib_alloc_fast_reg_page_list( + cb->pd->device, + cb->page_list_len); + if (IS_ERR(cb->page_list)) { + DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); + ret = PTR_ERR(cb->page_list); + goto bail; + } + cb->fastreg_mr = ib_alloc_fast_reg_mr(cb->pd, + cb->page_list->max_page_list_len); + if (IS_ERR(cb->fastreg_mr)) { + DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); + ret = PTR_ERR(cb->fastreg_mr); + goto bail; + } + DEBUG_LOG(cb, "fastreg rkey 0x%x page_list %p" + " page_list_len %u\n", cb->fastreg_mr->rkey, + cb->page_list, cb->page_list_len); + break; + case MW: + cb->mw = ib_alloc_mw(cb->pd,IB_MW_TYPE_1); + if (IS_ERR(cb->mw)) { + DEBUG_LOG(cb, "recv_buf alloc_mw failed\n"); + ret = PTR_ERR(cb->mw); + goto bail; + } + DEBUG_LOG(cb, "mw rkey 0x%x\n", cb->mw->rkey); + /*FALLTHROUGH*/ + case MR: + buf.addr = cb->rdma_dma_addr; + buf.size = cb->size; + iovbase = cb->rdma_dma_addr; + cb->rdma_mr = ib_reg_phys_mr(cb->pd, &buf, 1, + IB_ACCESS_LOCAL_WRITE| + IB_ACCESS_REMOTE_READ| + IB_ACCESS_REMOTE_WRITE, + &iovbase); + if (IS_ERR(cb->rdma_mr)) { + DEBUG_LOG(cb, "rdma_buf reg_mr failed\n"); + ret = PTR_ERR(cb->rdma_mr); + goto bail; + } + DEBUG_LOG(cb, "rdma buf dma_addr %jx size %d mr rkey 0x%x\n", + (uintmax_t)buf.addr, (int)buf.size, cb->rdma_mr->rkey); + break; + default: + ret = -EINVAL; + goto bail; + break; + } + } + + if (!cb->server || cb->wlat || cb->rlat || cb->bw || cb->frtest) { + + cb->start_buf = kmalloc(cb->size, GFP_KERNEL); + if (!cb->start_buf) { + DEBUG_LOG(cb, "start_buf malloc failed\n"); + ret = -ENOMEM; + goto bail; + } + + cb->start_dma_addr = dma_map_single(cb->pd->device->dma_device, + cb->start_buf, cb->size, + DMA_BIDIRECTIONAL); + pci_unmap_addr_set(cb, start_mapping, cb->start_dma_addr); + + if (cb->mem == MR || cb->mem == MW) { + unsigned flags = IB_ACCESS_REMOTE_READ; + + if (cb->wlat || cb->rlat || cb->bw || cb->frtest) { + flags |= IB_ACCESS_LOCAL_WRITE | + IB_ACCESS_REMOTE_WRITE; + } + + buf.addr = cb->start_dma_addr; + buf.size = cb->size; + DEBUG_LOG(cb, "start buf dma_addr %jx size %d\n", + (uintmax_t)buf.addr, (int)buf.size); + iovbase = cb->start_dma_addr; + cb->start_mr = ib_reg_phys_mr(cb->pd, &buf, 1, + flags, + &iovbase); + + if (IS_ERR(cb->start_mr)) { + DEBUG_LOG(cb, "start_buf reg_mr failed\n"); + ret = PTR_ERR(cb->start_mr); + goto bail; + } + } + } + + krping_setup_wr(cb); + DEBUG_LOG(cb, "allocated & registered buffers...\n"); + return 0; +bail: + if (cb->fastreg_mr && !IS_ERR(cb->fastreg_mr)) + ib_dereg_mr(cb->fastreg_mr); + if (cb->mw && !IS_ERR(cb->mw)) + ib_dealloc_mw(cb->mw); + if (cb->rdma_mr && !IS_ERR(cb->rdma_mr)) + ib_dereg_mr(cb->rdma_mr); + if (cb->page_list && !IS_ERR(cb->page_list)) + ib_free_fast_reg_page_list(cb->page_list); + if (cb->dma_mr && !IS_ERR(cb->dma_mr)) + ib_dereg_mr(cb->dma_mr); + if (cb->recv_mr && !IS_ERR(cb->recv_mr)) + ib_dereg_mr(cb->recv_mr); + if (cb->send_mr && !IS_ERR(cb->send_mr)) + ib_dereg_mr(cb->send_mr); + if (cb->rdma_buf) + kfree(cb->rdma_buf); + if (cb->start_buf) + kfree(cb->start_buf); + return ret; +} + +static void krping_free_buffers(struct krping_cb *cb) +{ + DEBUG_LOG(cb, "krping_free_buffers called on cb %p\n", cb); + + if (cb->dma_mr) + ib_dereg_mr(cb->dma_mr); + if (cb->send_mr) + ib_dereg_mr(cb->send_mr); + if (cb->recv_mr) + ib_dereg_mr(cb->recv_mr); + if (cb->rdma_mr) + ib_dereg_mr(cb->rdma_mr); + if (cb->start_mr) + ib_dereg_mr(cb->start_mr); + if (cb->fastreg_mr) + ib_dereg_mr(cb->fastreg_mr); + if (cb->mw) + ib_dealloc_mw(cb->mw); + + dma_unmap_single(cb->pd->device->dma_device, + pci_unmap_addr(cb, recv_mapping), + sizeof(cb->recv_buf), DMA_BIDIRECTIONAL); + dma_unmap_single(cb->pd->device->dma_device, + pci_unmap_addr(cb, send_mapping), + sizeof(cb->send_buf), DMA_BIDIRECTIONAL); + dma_unmap_single(cb->pd->device->dma_device, + pci_unmap_addr(cb, rdma_mapping), + cb->size, DMA_BIDIRECTIONAL); + kfree(cb->rdma_buf); + if (cb->start_buf) { + dma_unmap_single(cb->pd->device->dma_device, + pci_unmap_addr(cb, start_mapping), + cb->size, DMA_BIDIRECTIONAL); + kfree(cb->start_buf); + } +} + +static int krping_create_qp(struct krping_cb *cb) +{ + struct ib_qp_init_attr init_attr; + int ret; + + memset(&init_attr, 0, sizeof(init_attr)); + init_attr.cap.max_send_wr = cb->txdepth; + init_attr.cap.max_recv_wr = 2; + init_attr.cap.max_recv_sge = 1; + init_attr.cap.max_send_sge = 1; + init_attr.qp_type = IB_QPT_RC; + init_attr.send_cq = cb->cq; + init_attr.recv_cq = cb->cq; + init_attr.sq_sig_type = IB_SIGNAL_REQ_WR; + + if (cb->server) { + ret = rdma_create_qp(cb->child_cm_id, cb->pd, &init_attr); + if (!ret) + cb->qp = cb->child_cm_id->qp; + } else { + ret = rdma_create_qp(cb->cm_id, cb->pd, &init_attr); + if (!ret) + cb->qp = cb->cm_id->qp; + } + + return ret; +} + +static void krping_free_qp(struct krping_cb *cb) +{ + ib_destroy_qp(cb->qp); + ib_destroy_cq(cb->cq); + ib_dealloc_pd(cb->pd); +} + +static int krping_setup_qp(struct krping_cb *cb, struct rdma_cm_id *cm_id) +{ + int ret; + cb->pd = ib_alloc_pd(cm_id->device); + if (IS_ERR(cb->pd)) { + PRINTF(cb, "ib_alloc_pd failed\n"); + return PTR_ERR(cb->pd); + } + DEBUG_LOG(cb, "created pd %p\n", cb->pd); + + strlcpy(cb->stats.name, cb->pd->device->name, sizeof(cb->stats.name)); + + cb->cq = ib_create_cq(cm_id->device, krping_cq_event_handler, NULL, + cb, cb->txdepth * 2, 0); + if (IS_ERR(cb->cq)) { + PRINTF(cb, "ib_create_cq failed\n"); + ret = PTR_ERR(cb->cq); + goto err1; + } + DEBUG_LOG(cb, "created cq %p\n", cb->cq); + + if (!cb->wlat && !cb->rlat && !cb->bw && !cb->frtest) { + ret = ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); + if (ret) { + PRINTF(cb, "ib_create_cq failed\n"); + goto err2; + } + } + + ret = krping_create_qp(cb); + if (ret) { + PRINTF(cb, "krping_create_qp failed: %d\n", ret); + goto err2; + } + DEBUG_LOG(cb, "created qp %p\n", cb->qp); + return 0; +err2: + ib_destroy_cq(cb->cq); +err1: + ib_dealloc_pd(cb->pd); + return ret; +} + +/* + * return the (possibly rebound) rkey for the rdma buffer. + * FASTREG mode: invalidate and rebind via fastreg wr. + * MW mode: rebind the MW. + * other modes: just return the mr rkey. + */ +static u32 krping_rdma_rkey(struct krping_cb *cb, u64 buf, int post_inv) +{ + u32 rkey = 0xffffffff; + u64 p; + struct ib_send_wr *bad_wr; + int i; + int ret; + + switch (cb->mem) { + case FASTREG: + cb->invalidate_wr.ex.invalidate_rkey = cb->fastreg_mr->rkey; + + /* + * Update the fastreg key. + */ + ib_update_fast_reg_key(cb->fastreg_mr, ++cb->key); + cb->fastreg_wr.wr.fast_reg.rkey = cb->fastreg_mr->rkey; + + /* + * Update the fastreg WR with new buf info. + */ + if (buf == (u64)cb->start_dma_addr) + cb->fastreg_wr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_READ; + else + cb->fastreg_wr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; + cb->fastreg_wr.wr.fast_reg.iova_start = buf; + p = (u64)(buf & PAGE_MASK); + for (i=0; i < cb->fastreg_wr.wr.fast_reg.page_list_len; + i++, p += PAGE_SIZE) { + cb->page_list->page_list[i] = p; + DEBUG_LOG(cb, "page_list[%d] 0x%jx\n", i, (uintmax_t)p); + } + + DEBUG_LOG(cb, "post_inv = %d, fastreg new rkey 0x%x shift %u len %u" + " iova_start %jx page_list_len %u\n", + post_inv, + cb->fastreg_wr.wr.fast_reg.rkey, + cb->fastreg_wr.wr.fast_reg.page_shift, + (unsigned)cb->fastreg_wr.wr.fast_reg.length, + (uintmax_t)cb->fastreg_wr.wr.fast_reg.iova_start, + cb->fastreg_wr.wr.fast_reg.page_list_len); + + if (post_inv) + ret = ib_post_send(cb->qp, &cb->invalidate_wr, &bad_wr); + else + ret = ib_post_send(cb->qp, &cb->fastreg_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + cb->state = ERROR; + } + rkey = cb->fastreg_mr->rkey; + break; + case MW: + /* + * Update the MW with new buf info. + */ + if (buf == (u64)cb->start_dma_addr) { +#ifdef BIND_INFO + cb->bind_attr.bind_info.mw_access_flags = IB_ACCESS_REMOTE_READ; + cb->bind_attr.bind_info.mr = cb->start_mr; +#else + cb->bind_attr.mw_access_flags = IB_ACCESS_REMOTE_READ; + cb->bind_attr.mr = cb->start_mr; +#endif + } else { +#ifdef BIND_INFO + cb->bind_attr.bind_info.mw_access_flags = IB_ACCESS_REMOTE_WRITE; + cb->bind_attr.bind_info.mr = cb->rdma_mr; +#else + cb->bind_attr.mw_access_flags = IB_ACCESS_REMOTE_WRITE; + cb->bind_attr.mr = cb->rdma_mr; +#endif + } +#ifdef BIND_INFO + cb->bind_attr.bind_info.addr = buf; +#else + cb->bind_attr.addr = buf; +#endif + DEBUG_LOG(cb, "binding mw rkey 0x%x to buf %jx mr rkey 0x%x\n", +#ifdef BIND_INFO + cb->mw->rkey, (uintmax_t)buf, cb->bind_attr.bind_info.mr->rkey); +#else + cb->mw->rkey, buf, cb->bind_attr.mr->rkey); +#endif + ret = ib_bind_mw(cb->qp, cb->mw, &cb->bind_attr); + if (ret) { + PRINTF(cb, "bind mw error %d\n", ret); + cb->state = ERROR; + } else + rkey = cb->mw->rkey; + break; + case MR: + if (buf == (u64)cb->start_dma_addr) + rkey = cb->start_mr->rkey; + else + rkey = cb->rdma_mr->rkey; + break; + case DMA: + rkey = cb->dma_mr->rkey; + break; + default: + PRINTF(cb, "%s:%d case ERROR\n", __func__, __LINE__); + cb->state = ERROR; + break; + } + return rkey; +} + +static void krping_format_send(struct krping_cb *cb, u64 buf) +{ + struct krping_rdma_info *info = &cb->send_buf; + u32 rkey; + + /* + * Client side will do fastreg or mw bind before + * advertising the rdma buffer. Server side + * sends have no data. + */ + if (!cb->server || cb->wlat || cb->rlat || cb->bw || cb->frtest) { + rkey = krping_rdma_rkey(cb, buf, !cb->server_invalidate); + info->buf = htonll(buf); + info->rkey = htonl(rkey); + info->size = htonl(cb->size); + DEBUG_LOG(cb, "RDMA addr %llx rkey %x len %d\n", + (unsigned long long)buf, rkey, cb->size); + } +} + +static void krping_test_server(struct krping_cb *cb) +{ + struct ib_send_wr *bad_wr, inv; + int ret; + + while (1) { + /* Wait for client's Start STAG/TO/Len */ + wait_event_interruptible(cb->sem, cb->state >= RDMA_READ_ADV); + if (cb->state != RDMA_READ_ADV) { + PRINTF(cb, "wait for RDMA_READ_ADV state %d\n", + cb->state); + break; + } + + DEBUG_LOG(cb, "server received sink adv\n"); + + cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; + cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; + cb->rdma_sq_wr.sg_list->length = cb->remote_len; + cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, 1); + + /* Issue RDMA Read. */ + if (cb->read_inv) + cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ_WITH_INV; + else { + + cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ; + if (cb->mem == FASTREG) { + /* + * Immediately follow the read with a + * fenced LOCAL_INV. + */ + cb->rdma_sq_wr.next = &inv; + memset(&inv, 0, sizeof inv); + inv.opcode = IB_WR_LOCAL_INV; + inv.ex.invalidate_rkey = cb->fastreg_mr->rkey; + inv.send_flags = IB_SEND_FENCE; + } + } + + ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + break; + } + cb->rdma_sq_wr.next = NULL; + + DEBUG_LOG(cb, "server posted rdma read req \n"); + + /* Wait for read completion */ + wait_event_interruptible(cb->sem, + cb->state >= RDMA_READ_COMPLETE); + if (cb->state != RDMA_READ_COMPLETE) { + PRINTF(cb, + "wait for RDMA_READ_COMPLETE state %d\n", + cb->state); + break; + } + DEBUG_LOG(cb, "server received read complete\n"); + + /* Display data in recv buf */ + if (cb->verbose) { + if (strlen(cb->rdma_buf) > 128) { + char msgbuf[128]; + + strlcpy(msgbuf, cb->rdma_buf, sizeof(msgbuf)); + PRINTF(cb, "server ping data stripped: %s\n", + msgbuf); + } else + PRINTF(cb, "server ping data: %s\n", + cb->rdma_buf); + } + + /* Tell client to continue */ + if (cb->server && cb->server_invalidate) { + cb->sq_wr.ex.invalidate_rkey = cb->remote_rkey; + cb->sq_wr.opcode = IB_WR_SEND_WITH_INV; + DEBUG_LOG(cb, "send-w-inv rkey 0x%x\n", cb->remote_rkey); + } + ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + break; + } + DEBUG_LOG(cb, "server posted go ahead\n"); + + /* Wait for client's RDMA STAG/TO/Len */ + wait_event_interruptible(cb->sem, cb->state >= RDMA_WRITE_ADV); + if (cb->state != RDMA_WRITE_ADV) { + PRINTF(cb, + "wait for RDMA_WRITE_ADV state %d\n", + cb->state); + break; + } + DEBUG_LOG(cb, "server received sink adv\n"); + + /* RDMA Write echo data */ + cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; + cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; + cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; + cb->rdma_sq_wr.sg_list->length = strlen(cb->rdma_buf) + 1; + if (cb->local_dma_lkey) + cb->rdma_sgl.lkey = cb->qp->device->local_dma_lkey; + else + cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, 0); + + DEBUG_LOG(cb, "rdma write from lkey %x laddr %llx len %d\n", + cb->rdma_sq_wr.sg_list->lkey, + (unsigned long long)cb->rdma_sq_wr.sg_list->addr, + cb->rdma_sq_wr.sg_list->length); + + ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + break; + } + + /* Wait for completion */ + ret = wait_event_interruptible(cb->sem, cb->state >= + RDMA_WRITE_COMPLETE); + if (cb->state != RDMA_WRITE_COMPLETE) { + PRINTF(cb, + "wait for RDMA_WRITE_COMPLETE state %d\n", + cb->state); + break; + } + DEBUG_LOG(cb, "server rdma write complete \n"); + + cb->state = CONNECTED; + + /* Tell client to begin again */ + if (cb->server && cb->server_invalidate) { + cb->sq_wr.ex.invalidate_rkey = cb->remote_rkey; + cb->sq_wr.opcode = IB_WR_SEND_WITH_INV; + DEBUG_LOG(cb, "send-w-inv rkey 0x%x\n", cb->remote_rkey); + } + ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + break; + } + DEBUG_LOG(cb, "server posted go ahead\n"); + } +} + +static void rlat_test(struct krping_cb *cb) +{ + int scnt; + int iters = cb->count; + struct timeval start_tv, stop_tv; + int ret; + struct ib_wc wc; + struct ib_send_wr *bad_wr; + int ne; + + scnt = 0; + cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ; + cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; + cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; + cb->rdma_sq_wr.sg_list->length = cb->size; + + microtime(&start_tv); + if (!cb->poll) { + cb->state = RDMA_READ_ADV; + ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); + } + while (scnt < iters) { + + cb->state = RDMA_READ_ADV; + ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); + if (ret) { + PRINTF(cb, + "Couldn't post send: ret=%d scnt %d\n", + ret, scnt); + return; + } + + do { + if (!cb->poll) { + wait_event_interruptible(cb->sem, + cb->state != RDMA_READ_ADV); + if (cb->state == RDMA_READ_COMPLETE) { + ne = 1; + ib_req_notify_cq(cb->cq, + IB_CQ_NEXT_COMP); + } else { + ne = -1; + } + } else + ne = ib_poll_cq(cb->cq, 1, &wc); + if (cb->state == ERROR) { + PRINTF(cb, + "state == ERROR...bailing scnt %d\n", + scnt); + return; + } + } while (ne == 0); + + if (ne < 0) { + PRINTF(cb, "poll CQ failed %d\n", ne); + return; + } + if (cb->poll && wc.status != IB_WC_SUCCESS) { + PRINTF(cb, "Completion wth error at %s:\n", + cb->server ? "server" : "client"); + PRINTF(cb, "Failed status %d: wr_id %d\n", + wc.status, (int) wc.wr_id); + return; + } + ++scnt; + } + microtime(&stop_tv); + + if (stop_tv.tv_usec < start_tv.tv_usec) { + stop_tv.tv_usec += 1000000; + stop_tv.tv_sec -= 1; + } + + PRINTF(cb, "delta sec %lu delta usec %lu iter %d size %d\n", + (unsigned long)(stop_tv.tv_sec - start_tv.tv_sec), + (unsigned long)(stop_tv.tv_usec - start_tv.tv_usec), + scnt, cb->size); +} + +static void wlat_test(struct krping_cb *cb) +{ + int ccnt, scnt, rcnt; + int iters=cb->count; + volatile char *poll_buf = (char *) cb->start_buf; + char *buf = (char *)cb->rdma_buf; + struct timeval start_tv, stop_tv; + cycles_t *post_cycles_start, *post_cycles_stop; + cycles_t *poll_cycles_start, *poll_cycles_stop; + cycles_t *last_poll_cycles_start; + cycles_t sum_poll = 0, sum_post = 0, sum_last_poll = 0; + int i; + int cycle_iters = 1000; + + ccnt = 0; + scnt = 0; + rcnt = 0; + + post_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); + if (!post_cycles_start) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } + post_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); + if (!post_cycles_stop) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } + poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); + if (!poll_cycles_start) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } + poll_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); + if (!poll_cycles_stop) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } + last_poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), + GFP_KERNEL); + if (!last_poll_cycles_start) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } + cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; + cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; + cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; + cb->rdma_sq_wr.sg_list->length = cb->size; + + if (cycle_iters > iters) + cycle_iters = iters; + microtime(&start_tv); + while (scnt < iters || ccnt < iters || rcnt < iters) { + + /* Wait till buffer changes. */ + if (rcnt < iters && !(scnt < 1 && !cb->server)) { + ++rcnt; + while (*poll_buf != (char)rcnt) { + if (cb->state == ERROR) { + PRINTF(cb, + "state = ERROR, bailing\n"); + return; + } + } + } + + if (scnt < iters) { + struct ib_send_wr *bad_wr; + + *buf = (char)scnt+1; + if (scnt < cycle_iters) + post_cycles_start[scnt] = get_cycles(); + if (ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr)) { + PRINTF(cb, + "Couldn't post send: scnt=%d\n", + scnt); + return; + } + if (scnt < cycle_iters) + post_cycles_stop[scnt] = get_cycles(); + scnt++; + } + + if (ccnt < iters) { + struct ib_wc wc; + int ne; + + if (ccnt < cycle_iters) + poll_cycles_start[ccnt] = get_cycles(); + do { + if (ccnt < cycle_iters) + last_poll_cycles_start[ccnt] = + get_cycles(); + ne = ib_poll_cq(cb->cq, 1, &wc); + } while (ne == 0); + if (ccnt < cycle_iters) + poll_cycles_stop[ccnt] = get_cycles(); + ++ccnt; + + if (ne < 0) { + PRINTF(cb, "poll CQ failed %d\n", ne); + return; + } + if (wc.status != IB_WC_SUCCESS) { + PRINTF(cb, + "Completion wth error at %s:\n", + cb->server ? "server" : "client"); + PRINTF(cb, + "Failed status %d: wr_id %d\n", + wc.status, (int) wc.wr_id); + PRINTF(cb, + "scnt=%d, rcnt=%d, ccnt=%d\n", + scnt, rcnt, ccnt); + return; + } + } + } + microtime(&stop_tv); + + if (stop_tv.tv_usec < start_tv.tv_usec) { + stop_tv.tv_usec += 1000000; + stop_tv.tv_sec -= 1; + } + + for (i=0; i < cycle_iters; i++) { + sum_post += post_cycles_stop[i] - post_cycles_start[i]; + sum_poll += poll_cycles_stop[i] - poll_cycles_start[i]; + sum_last_poll += poll_cycles_stop[i]-last_poll_cycles_start[i]; + } + PRINTF(cb, + "delta sec %lu delta usec %lu iter %d size %d cycle_iters %d" + " sum_post %llu sum_poll %llu sum_last_poll %llu\n", + (unsigned long)(stop_tv.tv_sec - start_tv.tv_sec), + (unsigned long)(stop_tv.tv_usec - start_tv.tv_usec), + scnt, cb->size, cycle_iters, + (unsigned long long)sum_post, (unsigned long long)sum_poll, + (unsigned long long)sum_last_poll); + kfree(post_cycles_start); + kfree(post_cycles_stop); + kfree(poll_cycles_start); + kfree(poll_cycles_stop); + kfree(last_poll_cycles_start); +} + +static void bw_test(struct krping_cb *cb) +{ + int ccnt, scnt, rcnt; + int iters=cb->count; + struct timeval start_tv, stop_tv; + cycles_t *post_cycles_start, *post_cycles_stop; + cycles_t *poll_cycles_start, *poll_cycles_stop; + cycles_t *last_poll_cycles_start; + cycles_t sum_poll = 0, sum_post = 0, sum_last_poll = 0; + int i; + int cycle_iters = 1000; + + ccnt = 0; + scnt = 0; + rcnt = 0; + + post_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); + if (!post_cycles_start) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } + post_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); + if (!post_cycles_stop) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } + poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); + if (!poll_cycles_start) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } + poll_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); + if (!poll_cycles_stop) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } + last_poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), + GFP_KERNEL); + if (!last_poll_cycles_start) { + PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); + return; + } + cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; + cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; + cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; + cb->rdma_sq_wr.sg_list->length = cb->size; + + if (cycle_iters > iters) + cycle_iters = iters; + microtime(&start_tv); + while (scnt < iters || ccnt < iters) { + + while (scnt < iters && scnt - ccnt < cb->txdepth) { + struct ib_send_wr *bad_wr; + + if (scnt < cycle_iters) + post_cycles_start[scnt] = get_cycles(); + if (ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr)) { + PRINTF(cb, + "Couldn't post send: scnt=%d\n", + scnt); + return; + } + if (scnt < cycle_iters) + post_cycles_stop[scnt] = get_cycles(); + ++scnt; + } + + if (ccnt < iters) { + int ne; + struct ib_wc wc; + + if (ccnt < cycle_iters) + poll_cycles_start[ccnt] = get_cycles(); + do { + if (ccnt < cycle_iters) + last_poll_cycles_start[ccnt] = + get_cycles(); + ne = ib_poll_cq(cb->cq, 1, &wc); + } while (ne == 0); + if (ccnt < cycle_iters) + poll_cycles_stop[ccnt] = get_cycles(); + ccnt += 1; + + if (ne < 0) { + PRINTF(cb, "poll CQ failed %d\n", ne); + return; + } + if (wc.status != IB_WC_SUCCESS) { + PRINTF(cb, + "Completion wth error at %s:\n", + cb->server ? "server" : "client"); + PRINTF(cb, + "Failed status %d: wr_id %d\n", + wc.status, (int) wc.wr_id); + return; + } + } + } + microtime(&stop_tv); + + if (stop_tv.tv_usec < start_tv.tv_usec) { + stop_tv.tv_usec += 1000000; + stop_tv.tv_sec -= 1; + } + + for (i=0; i < cycle_iters; i++) { + sum_post += post_cycles_stop[i] - post_cycles_start[i]; + sum_poll += poll_cycles_stop[i] - poll_cycles_start[i]; + sum_last_poll += poll_cycles_stop[i]-last_poll_cycles_start[i]; + } + PRINTF(cb, + "delta sec %lu delta usec %lu iter %d size %d cycle_iters %d" + " sum_post %llu sum_poll %llu sum_last_poll %llu\n", + (unsigned long)(stop_tv.tv_sec - start_tv.tv_sec), + (unsigned long)(stop_tv.tv_usec - start_tv.tv_usec), + scnt, cb->size, cycle_iters, + (unsigned long long)sum_post, (unsigned long long)sum_poll, + (unsigned long long)sum_last_poll); + kfree(post_cycles_start); + kfree(post_cycles_stop); + kfree(poll_cycles_start); + kfree(poll_cycles_stop); + kfree(last_poll_cycles_start); +} + +static void krping_rlat_test_server(struct krping_cb *cb) +{ + struct ib_send_wr *bad_wr; + struct ib_wc wc; + int ret; + + /* Spin waiting for client's Start STAG/TO/Len */ + while (cb->state < RDMA_READ_ADV) { + krping_cq_event_handler(cb->cq, cb); + } + + /* Send STAG/TO/Len to client */ + krping_format_send(cb, cb->start_dma_addr); + ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + return; + } + + /* Spin waiting for send completion */ + while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); + if (ret < 0) { + PRINTF(cb, "poll error %d\n", ret); + return; + } + if (wc.status) { + PRINTF(cb, "send completiong error %d\n", wc.status); + return; + } + wait_event_interruptible(cb->sem, cb->state == ERROR); +} + +static void krping_wlat_test_server(struct krping_cb *cb) +{ + struct ib_send_wr *bad_wr; + struct ib_wc wc; + int ret; + + /* Spin waiting for client's Start STAG/TO/Len */ + while (cb->state < RDMA_READ_ADV) { + krping_cq_event_handler(cb->cq, cb); + } + + /* Send STAG/TO/Len to client */ + krping_format_send(cb, cb->start_dma_addr); + ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + return; + } + + /* Spin waiting for send completion */ + while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); + if (ret < 0) { + PRINTF(cb, "poll error %d\n", ret); + return; + } + if (wc.status) { + PRINTF(cb, "send completiong error %d\n", wc.status); + return; + } + + wlat_test(cb); + wait_event_interruptible(cb->sem, cb->state == ERROR); +} + +static void krping_bw_test_server(struct krping_cb *cb) +{ + struct ib_send_wr *bad_wr; + struct ib_wc wc; + int ret; + + /* Spin waiting for client's Start STAG/TO/Len */ + while (cb->state < RDMA_READ_ADV) { + krping_cq_event_handler(cb->cq, cb); + } + + /* Send STAG/TO/Len to client */ + krping_format_send(cb, cb->start_dma_addr); + ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + return; + } + + /* Spin waiting for send completion */ + while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); + if (ret < 0) { + PRINTF(cb, "poll error %d\n", ret); + return; + } + if (wc.status) { + PRINTF(cb, "send completiong error %d\n", wc.status); + return; + } + + if (cb->duplex) + bw_test(cb); + wait_event_interruptible(cb->sem, cb->state == ERROR); +} + +static int fastreg_supported(struct krping_cb *cb, int server) +{ + struct ib_device *dev = server?cb->child_cm_id->device: + cb->cm_id->device; + struct ib_device_attr attr; + int ret; + + ret = ib_query_device(dev, &attr); + if (ret) { + PRINTF(cb, "ib_query_device failed ret %d\n", ret); + return 0; + } + if (!(attr.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) { + PRINTF(cb, "Fastreg not supported - device_cap_flags 0x%llx\n", + (unsigned long long)attr.device_cap_flags); + return 0; + } + DEBUG_LOG(cb, "Fastreg supported - device_cap_flags 0x%jx\n", + (uintmax_t)attr.device_cap_flags); + return 1; +} + +static int krping_bind_server(struct krping_cb *cb) +{ + union { + struct sockaddr_in v4; + struct sockaddr_in6 v6; + } sin; + int ret; + + memset(&sin, 0, sizeof(sin)); + + switch (cb->addr_type) { + case AF_INET: + sin.v4.sin_len = sizeof sin.v4; + sin.v4.sin_family = AF_INET; + sin.v4.sin_addr = cb->addr.v4; + sin.v4.sin_port = cb->port; + break; + case AF_INET6: + sin.v6.sin6_len = sizeof sin.v6; + sin.v6.sin6_family = AF_INET6; + sin.v6.sin6_addr = cb->addr.v6; + sin.v6.sin6_port = cb->port; + break; + default: + return (-EINVAL); + } + + ret = rdma_bind_addr(cb->cm_id, (struct sockaddr *) &sin); + if (ret) { + PRINTF(cb, "rdma_bind_addr error %d\n", ret); + return ret; + } + DEBUG_LOG(cb, "rdma_bind_addr successful\n"); + + DEBUG_LOG(cb, "rdma_listen\n"); + ret = rdma_listen(cb->cm_id, 3); + if (ret) { + PRINTF(cb, "rdma_listen failed: %d\n", ret); + return ret; + } + + wait_event_interruptible(cb->sem, cb->state >= CONNECT_REQUEST); + if (cb->state != CONNECT_REQUEST) { + PRINTF(cb, "wait for CONNECT_REQUEST state %d\n", + cb->state); + return -1; + } + + if (cb->mem == FASTREG && !fastreg_supported(cb, 1)) + return -EINVAL; + + return 0; +} + +/* + * sq-depth worth of fastreg + 0B read-inv pairs, reposting them as the reads + * complete. + * NOTE: every 9 seconds we sleep for 1 second to keep the kernel happy. + */ +static void krping_fr_test5(struct krping_cb *cb) +{ + struct ib_fast_reg_page_list **pl; + struct ib_send_wr *fr, *read, *bad; + struct ib_wc wc; + struct ib_sge *sgl; + u8 key = 0; + struct ib_mr **mr; + u8 **buf; + dma_addr_t *dma_addr; + int i; + int ret; + int plen = (((cb->size - 1) & PAGE_MASK) + PAGE_SIZE) >> PAGE_SHIFT; + time_t start; + int count = 0; + int scnt; + int depth = cb->txdepth >> 1; + + if (!depth) { + PRINTF(cb, "txdepth must be > 1 for this test!\n"); + return; + } + + pl = kzalloc(sizeof *pl * depth, GFP_KERNEL); + DEBUG_LOG(cb, "%s pl %p size %zu\n", __func__, pl, sizeof *pl * depth); + mr = kzalloc(sizeof *mr * depth, GFP_KERNEL); + DEBUG_LOG(cb, "%s mr %p size %zu\n", __func__, mr, sizeof *mr * depth); + fr = kzalloc(sizeof *fr * depth, GFP_KERNEL); + DEBUG_LOG(cb, "%s fr %p size %zu\n", __func__, fr, sizeof *fr * depth); + sgl = kzalloc(sizeof *sgl * depth, GFP_KERNEL); + DEBUG_LOG(cb, "%s sgl %p size %zu\n", __func__, sgl, sizeof *sgl * depth); + read = kzalloc(sizeof *read * depth, GFP_KERNEL); + DEBUG_LOG(cb, "%s read %p size %zu\n", __func__, read, sizeof *read * depth); + buf = kzalloc(sizeof *buf * depth, GFP_KERNEL); + DEBUG_LOG(cb, "%s buf %p size %zu\n", __func__, buf, sizeof *buf * depth); + dma_addr = kzalloc(sizeof *dma_addr * depth, GFP_KERNEL); + DEBUG_LOG(cb, "%s dma_addr %p size %zu\n", __func__, dma_addr, sizeof *dma_addr * depth); + if (!pl || !mr || !fr || !read || !sgl || !buf || !dma_addr) { + PRINTF(cb, "kzalloc failed\n"); + goto err1; + } + + for (scnt = 0; scnt < depth; scnt++) { + pl[scnt] = ib_alloc_fast_reg_page_list(cb->qp->device, plen); + if (IS_ERR(pl[scnt])) { + PRINTF(cb, "alloc_fr_page_list failed %ld\n", + PTR_ERR(pl[scnt])); + goto err2; + } + DEBUG_LOG(cb, "%s pl[%u] %p\n", __func__, scnt, pl[scnt]); + + mr[scnt] = ib_alloc_fast_reg_mr(cb->pd, plen); + if (IS_ERR(mr[scnt])) { + PRINTF(cb, "alloc_fr failed %ld\n", + PTR_ERR(mr[scnt])); + goto err2; + } + DEBUG_LOG(cb, "%s mr[%u] %p\n", __func__, scnt, mr[scnt]); + ib_update_fast_reg_key(mr[scnt], ++key); + + buf[scnt] = kmalloc(cb->size, GFP_KERNEL); + if (!buf[scnt]) { + PRINTF(cb, "kmalloc failed\n"); + ret = -ENOMEM; + goto err2; + } + DEBUG_LOG(cb, "%s buf[%u] %p\n", __func__, scnt, buf[scnt]); + dma_addr[scnt] = dma_map_single(cb->pd->device->dma_device, + buf[scnt], cb->size, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(cb->pd->device->dma_device, + dma_addr[scnt])) { + PRINTF(cb, "dma_map failed\n"); + ret = -ENOMEM; + goto err2; + } + DEBUG_LOG(cb, "%s dma_addr[%u] %p\n", __func__, scnt, (void *)dma_addr[scnt]); + for (i=0; ipage_list[i] = ((unsigned long)dma_addr[scnt] & PAGE_MASK) + (i * PAGE_SIZE); + DEBUG_LOG(cb, "%s pl[%u]->page_list[%u] 0x%jx\n", + __func__, scnt, i, (uintmax_t)pl[scnt]->page_list[i]); + } + + sgl[scnt].lkey = mr[scnt]->rkey; + sgl[scnt].length = cb->size; + sgl[scnt].addr = (u64)buf[scnt]; + DEBUG_LOG(cb, "%s sgl[%u].lkey 0x%x length %u addr 0x%jx\n", + __func__, scnt, sgl[scnt].lkey, sgl[scnt].length, + (uintmax_t)sgl[scnt].addr); + + fr[scnt].opcode = IB_WR_FAST_REG_MR; + fr[scnt].wr_id = scnt; + fr[scnt].send_flags = 0; + fr[scnt].wr.fast_reg.page_shift = PAGE_SHIFT; + fr[scnt].wr.fast_reg.length = cb->size; + fr[scnt].wr.fast_reg.page_list = pl[scnt]; + fr[scnt].wr.fast_reg.page_list_len = plen; + fr[scnt].wr.fast_reg.iova_start = (u64)buf[scnt]; + fr[scnt].wr.fast_reg.access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; + fr[scnt].wr.fast_reg.rkey = mr[scnt]->rkey; + fr[scnt].next = &read[scnt]; + read[scnt].opcode = IB_WR_RDMA_READ_WITH_INV; + read[scnt].wr_id = scnt; + read[scnt].send_flags = IB_SEND_SIGNALED; + read[scnt].wr.rdma.rkey = cb->remote_rkey; + read[scnt].wr.rdma.remote_addr = cb->remote_addr; + read[scnt].num_sge = 1; + read[scnt].sg_list = &sgl[scnt]; + ret = ib_post_send(cb->qp, &fr[scnt], &bad); + if (ret) { + PRINTF(cb, "ib_post_send failed %d\n", ret); + goto err2; + } + } + + start = time_uptime; + DEBUG_LOG(cb, "%s starting IO.\n", __func__); + while (!cb->count || cb->server || count < cb->count) { + if ((time_uptime - start) >= 9) { + DEBUG_LOG(cb, "%s pausing 1 tick! count %u\n", __func__, + count); + wait_event_interruptible_timeout(cb->sem, + cb->state == ERROR, + 1); + if (cb->state == ERROR) + break; + start = time_uptime; + } + do { + ret = ib_poll_cq(cb->cq, 1, &wc); + if (ret < 0) { + PRINTF(cb, "ib_poll_cq failed %d\n", + ret); + goto err2; + } + if (ret == 1) { + if (wc.status) { + PRINTF(cb, + "completion error %u wr_id %ju " + "opcode %d\n", wc.status, + (uintmax_t)wc.wr_id, wc.opcode); + goto err2; + } + count++; + if (count == cb->count) + break; + ib_update_fast_reg_key(mr[wc.wr_id], ++key); + fr[wc.wr_id].wr.fast_reg.rkey = + mr[wc.wr_id]->rkey; + sgl[wc.wr_id].lkey = mr[wc.wr_id]->rkey; + ret = ib_post_send(cb->qp, &fr[wc.wr_id], &bad); + if (ret) { + PRINTF(cb, + "ib_post_send failed %d\n", ret); + goto err2; + } + } else if (krping_sigpending()) { + PRINTF(cb, "signal!\n"); + goto err2; + } + } while (ret == 1); + } + DEBUG_LOG(cb, "%s done!\n", __func__); +err2: + DEBUG_LOG(cb, "sleeping 1 second\n"); + wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); + DEBUG_LOG(cb, "draining the cq...\n"); + do { + ret = ib_poll_cq(cb->cq, 1, &wc); + if (ret < 0) { + PRINTF(cb, "ib_poll_cq failed %d\n", ret); + break; + } + if (ret == 1) { + if (wc.status) { + PRINTF(cb, "completion error %u " + "opcode %u\n", wc.status, wc.opcode); + } + } + } while (ret == 1); + + DEBUG_LOG(cb, "destroying fr mrs!\n"); + for (scnt = 0; scnt < depth; scnt++) { + if (mr[scnt]) { + ib_dereg_mr(mr[scnt]); + DEBUG_LOG(cb, "%s dereg mr %p\n", __func__, mr[scnt]); + } + } + DEBUG_LOG(cb, "unmapping/freeing bufs!\n"); + for (scnt = 0; scnt < depth; scnt++) { + if (buf[scnt]) { + dma_unmap_single(cb->pd->device->dma_device, + dma_addr[scnt], cb->size, + DMA_BIDIRECTIONAL); + kfree(buf[scnt]); + DEBUG_LOG(cb, "%s unmap/free buf %p dma_addr %p\n", __func__, buf[scnt], (void *)dma_addr[scnt]); + } + } + DEBUG_LOG(cb, "destroying fr page lists!\n"); + for (scnt = 0; scnt < depth; scnt++) { + if (pl[scnt]) { + DEBUG_LOG(cb, "%s free pl %p\n", __func__, pl[scnt]); + ib_free_fast_reg_page_list(pl[scnt]); + } + } +err1: + if (pl) + kfree(pl); + if (mr) + kfree(mr); + if (fr) + kfree(fr); + if (read) + kfree(read); + if (sgl) + kfree(sgl); + if (buf) + kfree(buf); + if (dma_addr) + kfree(dma_addr); +} +static void krping_fr_test_server(struct krping_cb *cb) +{ + DEBUG_LOG(cb, "%s waiting for disconnect...\n", __func__); + wait_event_interruptible(cb->sem, cb->state == ERROR); +} + +static void krping_fr_test5_server(struct krping_cb *cb) +{ + struct ib_send_wr *bad_wr; + struct ib_wc wc; + int ret; + + /* Spin waiting for client's Start STAG/TO/Len */ + while (cb->state < RDMA_READ_ADV) { + krping_cq_event_handler(cb->cq, cb); + } + DEBUG_LOG(cb, "%s client STAG %x TO 0x%jx\n", __func__, + cb->remote_rkey, (uintmax_t)cb->remote_addr); + + /* Send STAG/TO/Len to client */ + krping_format_send(cb, cb->start_dma_addr); + ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + return; + } + + /* Spin waiting for send completion */ + while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); + if (ret < 0) { + PRINTF(cb, "poll error %d\n", ret); + return; + } + if (wc.status) { + PRINTF(cb, "send completiong error %d\n", wc.status); + return; + } + + if (cb->duplex) + krping_fr_test5(cb); + DEBUG_LOG(cb, "%s waiting for disconnect...\n", __func__); + wait_event_interruptible(cb->sem, cb->state == ERROR); +} + +static void krping_fr_test5_client(struct krping_cb *cb) +{ + struct ib_send_wr *bad; + struct ib_wc wc; + int ret; + + cb->state = RDMA_READ_ADV; + + /* Send STAG/TO/Len to server */ + krping_format_send(cb, cb->start_dma_addr); + if (cb->state == ERROR) { + PRINTF(cb, "krping_format_send failed\n"); + return; + } + ret = ib_post_send(cb->qp, &cb->sq_wr, &bad); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + return; + } + + /* Spin waiting for send completion */ + while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); + if (ret < 0) { + PRINTF(cb, "poll error %d\n", ret); + return; + } + if (wc.status) { + PRINTF(cb, "send completion error %d\n", wc.status); + return; + } + + /* Spin waiting for server's Start STAG/TO/Len */ + while (cb->state < RDMA_WRITE_ADV) { + krping_cq_event_handler(cb->cq, cb); + } + DEBUG_LOG(cb, "%s server STAG %x TO 0x%jx\n", __func__, cb->remote_rkey, + (uintmax_t)cb->remote_addr); + + return krping_fr_test5(cb); +} + +/* + * sq-depth worth of write + fastreg + inv, reposting them as the invs + * complete. + * NOTE: every 9 seconds we sleep for 1 second to keep the kernel happy. + * If a count is given, then the last IO will have a bogus lkey in the + * write work request. This reproduces a fw bug where the connection + * will get stuck if a fastreg is processed while the ulptx is failing + * the bad write. + */ +static void krping_fr_test6(struct krping_cb *cb) +{ + struct ib_fast_reg_page_list **pl; + struct ib_send_wr *fr, *write, *inv, *bad; + struct ib_wc wc; + struct ib_sge *sgl; + u8 key = 0; + struct ib_mr **mr; + u8 **buf; + dma_addr_t *dma_addr; + int i; + int ret; + int plen = (((cb->size - 1) & PAGE_MASK) + PAGE_SIZE) >> PAGE_SHIFT; + unsigned long start; + int count = 0; + int scnt; + int depth = cb->txdepth / 3; + + if (!depth) { + PRINTF(cb, "txdepth must be > 3 for this test!\n"); + return; + } + + pl = kzalloc(sizeof *pl * depth, GFP_KERNEL); + DEBUG_LOG(cb, "%s pl %p size %zu\n", __func__, pl, sizeof *pl * depth); + + mr = kzalloc(sizeof *mr * depth, GFP_KERNEL); + DEBUG_LOG(cb, "%s mr %p size %zu\n", __func__, mr, sizeof *mr * depth); + + fr = kzalloc(sizeof *fr * depth, GFP_KERNEL); + DEBUG_LOG(cb, "%s fr %p size %zu\n", __func__, fr, sizeof *fr * depth); + + sgl = kzalloc(sizeof *sgl * depth, GFP_KERNEL); + DEBUG_LOG(cb, "%s sgl %p size %zu\n", __func__, sgl, sizeof *sgl * depth); + + write = kzalloc(sizeof *write * depth, GFP_KERNEL); + DEBUG_LOG(cb, "%s read %p size %zu\n", __func__, write, sizeof *write * depth); + + inv = kzalloc(sizeof *inv * depth, GFP_KERNEL); + DEBUG_LOG(cb, "%s inv %p size %zu\n", __func__, inv, sizeof *inv * depth); + + buf = kzalloc(sizeof *buf * depth, GFP_KERNEL); + DEBUG_LOG(cb, "%s buf %p size %zu\n", __func__, buf, sizeof *buf * depth); + + dma_addr = kzalloc(sizeof *dma_addr * depth, GFP_KERNEL); + DEBUG_LOG(cb, "%s dma_addr %p size %zu\n", __func__, dma_addr, sizeof *dma_addr * depth); + + if (!pl || !mr || !fr || !write || !sgl || !buf || !dma_addr) { + PRINTF(cb, "kzalloc failed\n"); + goto err1; + } + + for (scnt = 0; scnt < depth; scnt++) { + pl[scnt] = ib_alloc_fast_reg_page_list(cb->qp->device, plen); + if (IS_ERR(pl[scnt])) { + PRINTF(cb, "alloc_fr_page_list failed %ld\n", + PTR_ERR(pl[scnt])); + goto err2; + } + DEBUG_LOG(cb, "%s pl[%u] %p\n", __func__, scnt, pl[scnt]); + + mr[scnt] = ib_alloc_fast_reg_mr(cb->pd, plen); + if (IS_ERR(mr[scnt])) { + PRINTF(cb, "alloc_fr failed %ld\n", + PTR_ERR(mr[scnt])); + goto err2; + } + DEBUG_LOG(cb, "%s mr[%u] %p\n", __func__, scnt, mr[scnt]); + ib_update_fast_reg_key(mr[scnt], ++key); + + buf[scnt] = kmalloc(cb->size, GFP_KERNEL); + if (!buf[scnt]) { + PRINTF(cb, "kmalloc failed\n"); + ret = -ENOMEM; + goto err2; + } + DEBUG_LOG(cb, "%s buf[%u] %p\n", __func__, scnt, buf[scnt]); + dma_addr[scnt] = dma_map_single(cb->pd->device->dma_device, + buf[scnt], cb->size, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(cb->pd->device->dma_device, + dma_addr[scnt])) { + PRINTF(cb, "dma_map failed\n"); + ret = -ENOMEM; + goto err2; + } + DEBUG_LOG(cb, "%s dma_addr[%u] %p\n", __func__, scnt, (void *)dma_addr[scnt]); + for (i=0; ipage_list[i] = ((unsigned long)dma_addr[scnt] & PAGE_MASK) + (i * PAGE_SIZE); + DEBUG_LOG(cb, "%s pl[%u]->page_list[%u] 0x%jx\n", + __func__, scnt, i, (uintmax_t)pl[scnt]->page_list[i]); + } + + write[scnt].opcode = IB_WR_RDMA_WRITE; + write[scnt].wr_id = scnt; + write[scnt].wr.rdma.rkey = cb->remote_rkey; + write[scnt].wr.rdma.remote_addr = cb->remote_addr; + write[scnt].num_sge = 1; + write[scnt].sg_list = &cb->rdma_sgl; + write[scnt].sg_list->length = cb->size; + write[scnt].next = &fr[scnt]; + + fr[scnt].opcode = IB_WR_FAST_REG_MR; + fr[scnt].wr_id = scnt; + fr[scnt].wr.fast_reg.page_shift = PAGE_SHIFT; + fr[scnt].wr.fast_reg.length = cb->size; + fr[scnt].wr.fast_reg.page_list = pl[scnt]; + fr[scnt].wr.fast_reg.page_list_len = plen; + fr[scnt].wr.fast_reg.iova_start = (u64)buf[scnt]; + fr[scnt].wr.fast_reg.access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; + fr[scnt].wr.fast_reg.rkey = mr[scnt]->rkey; + fr[scnt].next = &inv[scnt]; + + inv[scnt].opcode = IB_WR_LOCAL_INV; + inv[scnt].send_flags = IB_SEND_SIGNALED; + inv[scnt].ex.invalidate_rkey = mr[scnt]->rkey; + + ret = ib_post_send(cb->qp, &write[scnt], &bad); + if (ret) { + PRINTF(cb, "ib_post_send failed %d\n", ret); + goto err2; + } + } + + start = time_uptime; + DEBUG_LOG(cb, "%s starting IO.\n", __func__); + while (!cb->count || cb->server || count < cb->count) { + if ((time_uptime - start) >= 9) { + DEBUG_LOG(cb, "%s pausing 1 tick! count %u\n", __func__, + count); + wait_event_interruptible_timeout(cb->sem, + cb->state == ERROR, + 1); + if (cb->state == ERROR) + break; + start = time_uptime; + } + do { + ret = ib_poll_cq(cb->cq, 1, &wc); + if (ret < 0) { + PRINTF(cb, "ib_poll_cq failed %d\n", + ret); + goto err2; + } + if (ret == 1) { + if (wc.status) { + PRINTF(cb, + "completion error %u wr_id %ju " + "opcode %d\n", wc.status, + (uintmax_t)wc.wr_id, wc.opcode); + goto err2; + } + count++; + if (count == (cb->count -1)) + cb->rdma_sgl.lkey = 0x00dead; + if (count == cb->count) + break; + ib_update_fast_reg_key(mr[wc.wr_id], ++key); + fr[wc.wr_id].wr.fast_reg.rkey = + mr[wc.wr_id]->rkey; + inv[wc.wr_id].ex.invalidate_rkey = + mr[wc.wr_id]->rkey; + ret = ib_post_send(cb->qp, &write[wc.wr_id], &bad); + if (ret) { + PRINTF(cb, + "ib_post_send failed %d\n", ret); + goto err2; + } + } else if (krping_sigpending()){ + PRINTF(cb, "signal!\n"); + goto err2; + } + } while (ret == 1); + } + DEBUG_LOG(cb, "%s done!\n", __func__); +err2: + DEBUG_LOG(cb, "sleeping 1 second\n"); + wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); + DEBUG_LOG(cb, "draining the cq...\n"); + do { + ret = ib_poll_cq(cb->cq, 1, &wc); + if (ret < 0) { + PRINTF(cb, "ib_poll_cq failed %d\n", ret); + break; + } + if (ret == 1) { + if (wc.status) { + PRINTF(cb, "completion error %u " + "opcode %u\n", wc.status, wc.opcode); + } + } + } while (ret == 1); + + DEBUG_LOG(cb, "destroying fr mrs!\n"); + for (scnt = 0; scnt < depth; scnt++) { + if (mr[scnt]) { + ib_dereg_mr(mr[scnt]); + DEBUG_LOG(cb, "%s dereg mr %p\n", __func__, mr[scnt]); + } + } + DEBUG_LOG(cb, "unmapping/freeing bufs!\n"); + for (scnt = 0; scnt < depth; scnt++) { + if (buf[scnt]) { + dma_unmap_single(cb->pd->device->dma_device, + dma_addr[scnt], cb->size, + DMA_BIDIRECTIONAL); + kfree(buf[scnt]); + DEBUG_LOG(cb, "%s unmap/free buf %p dma_addr %p\n", __func__, buf[scnt], (void *)dma_addr[scnt]); + } + } + DEBUG_LOG(cb, "destroying fr page lists!\n"); + for (scnt = 0; scnt < depth; scnt++) { + if (pl[scnt]) { + DEBUG_LOG(cb, "%s free pl %p\n", __func__, pl[scnt]); + ib_free_fast_reg_page_list(pl[scnt]); + } + } +err1: + if (pl) + kfree(pl); + if (mr) + kfree(mr); + if (fr) + kfree(fr); + if (write) + kfree(write); + if (inv) + kfree(inv); + if (sgl) + kfree(sgl); + if (buf) + kfree(buf); + if (dma_addr) + kfree(dma_addr); +} + +static void krping_fr_test6_server(struct krping_cb *cb) +{ + struct ib_send_wr *bad_wr; + struct ib_wc wc; + int ret; + + /* Spin waiting for client's Start STAG/TO/Len */ + while (cb->state < RDMA_READ_ADV) { + krping_cq_event_handler(cb->cq, cb); + } + DEBUG_LOG(cb, "%s client STAG %x TO 0x%jx\n", __func__, + cb->remote_rkey, (uintmax_t)cb->remote_addr); + + /* Send STAG/TO/Len to client */ + krping_format_send(cb, cb->start_dma_addr); + ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + return; + } + + /* Spin waiting for send completion */ + while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); + if (ret < 0) { + PRINTF(cb, "poll error %d\n", ret); + return; + } + if (wc.status) { + PRINTF(cb, "send completiong error %d\n", wc.status); + return; + } + + if (cb->duplex) + krping_fr_test6(cb); + DEBUG_LOG(cb, "%s waiting for disconnect...\n", __func__); + wait_event_interruptible(cb->sem, cb->state == ERROR); +} + +static void krping_fr_test6_client(struct krping_cb *cb) +{ + struct ib_send_wr *bad; + struct ib_wc wc; + int ret; + + cb->state = RDMA_READ_ADV; + + /* Send STAG/TO/Len to server */ + krping_format_send(cb, cb->start_dma_addr); + if (cb->state == ERROR) { + PRINTF(cb, "krping_format_send failed\n"); + return; + } + ret = ib_post_send(cb->qp, &cb->sq_wr, &bad); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + return; + } + + /* Spin waiting for send completion */ + while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); + if (ret < 0) { + PRINTF(cb, "poll error %d\n", ret); + return; + } + if (wc.status) { + PRINTF(cb, "send completion error %d\n", wc.status); + return; + } + + /* Spin waiting for server's Start STAG/TO/Len */ + while (cb->state < RDMA_WRITE_ADV) { + krping_cq_event_handler(cb->cq, cb); + } + DEBUG_LOG(cb, "%s server STAG %x TO 0x%jx\n", __func__, cb->remote_rkey, + (uintmax_t)cb->remote_addr); + + return krping_fr_test6(cb); +} + +static void krping_run_server(struct krping_cb *cb) +{ + struct ib_recv_wr *bad_wr; + int ret; + + ret = krping_bind_server(cb); + if (ret) + return; + + ret = krping_setup_qp(cb, cb->child_cm_id); + if (ret) { + PRINTF(cb, "setup_qp failed: %d\n", ret); + goto err0; + } + + ret = krping_setup_buffers(cb); + if (ret) { + PRINTF(cb, "krping_setup_buffers failed: %d\n", ret); + goto err1; + } + + ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "ib_post_recv failed: %d\n", ret); + goto err2; + } + + ret = krping_accept(cb); + if (ret) { + PRINTF(cb, "connect error %d\n", ret); + goto err2; + } + + if (cb->wlat) + krping_wlat_test_server(cb); + else if (cb->rlat) + krping_rlat_test_server(cb); + else if (cb->bw) + krping_bw_test_server(cb); + else if (cb->frtest) { + switch (cb->testnum) { + case 1: + case 2: + case 3: + case 4: + krping_fr_test_server(cb); + break; + case 5: + krping_fr_test5_server(cb); + break; + case 6: + krping_fr_test6_server(cb); + break; + default: + PRINTF(cb, "unknown fr test %d\n", cb->testnum); + goto err2; + break; + } + } else + krping_test_server(cb); + rdma_disconnect(cb->child_cm_id); +err2: + krping_free_buffers(cb); +err1: + krping_free_qp(cb); +err0: + rdma_destroy_id(cb->child_cm_id); +} + +static void krping_test_client(struct krping_cb *cb) +{ + int ping, start, cc, i, ret; + struct ib_send_wr *bad_wr; + unsigned char c; + + start = 65; + for (ping = 0; !cb->count || ping < cb->count; ping++) { + cb->state = RDMA_READ_ADV; + + /* Put some ascii text in the buffer. */ + cc = sprintf(cb->start_buf, "rdma-ping-%d: ", ping); + for (i = cc, c = start; i < cb->size; i++) { + cb->start_buf[i] = c; + c++; + if (c > 122) + c = 65; + } + start++; + if (start > 122) + start = 65; + cb->start_buf[cb->size - 1] = 0; + + krping_format_send(cb, cb->start_dma_addr); + if (cb->state == ERROR) { + PRINTF(cb, "krping_format_send failed\n"); + break; + } + ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + break; + } + + /* Wait for server to ACK */ + wait_event_interruptible(cb->sem, cb->state >= RDMA_WRITE_ADV); + if (cb->state != RDMA_WRITE_ADV) { + PRINTF(cb, + "wait for RDMA_WRITE_ADV state %d\n", + cb->state); + break; + } + + krping_format_send(cb, cb->rdma_dma_addr); + ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + break; + } + + /* Wait for the server to say the RDMA Write is complete. */ + wait_event_interruptible(cb->sem, + cb->state >= RDMA_WRITE_COMPLETE); + if (cb->state != RDMA_WRITE_COMPLETE) { + PRINTF(cb, + "wait for RDMA_WRITE_COMPLETE state %d\n", + cb->state); + break; + } + + if (cb->validate) + if (memcmp(cb->start_buf, cb->rdma_buf, cb->size)) { + PRINTF(cb, "data mismatch!\n"); + break; + } + + if (cb->verbose) { + if (strlen(cb->rdma_buf) > 128) { + char msgbuf[128]; + + strlcpy(msgbuf, cb->rdma_buf, sizeof(msgbuf)); + PRINTF(cb, "ping data stripped: %s\n", + msgbuf); + } else + PRINTF(cb, "ping data: %s\n", cb->rdma_buf); + } +#ifdef SLOW_KRPING + wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); +#endif + } +} + +static void krping_rlat_test_client(struct krping_cb *cb) +{ + struct ib_send_wr *bad_wr; + struct ib_wc wc; + int ret; + + cb->state = RDMA_READ_ADV; + + /* Send STAG/TO/Len to client */ + krping_format_send(cb, cb->start_dma_addr); + if (cb->state == ERROR) { + PRINTF(cb, "krping_format_send failed\n"); + return; + } + ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + return; + } + + /* Spin waiting for send completion */ + while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); + if (ret < 0) { + PRINTF(cb, "poll error %d\n", ret); + return; + } + if (wc.status) { + PRINTF(cb, "send completion error %d\n", wc.status); + return; + } + + /* Spin waiting for server's Start STAG/TO/Len */ + while (cb->state < RDMA_WRITE_ADV) { + krping_cq_event_handler(cb->cq, cb); + } + +#if 0 +{ + int i; + struct timeval start, stop; + time_t sec; + suseconds_t usec; + unsigned long long elapsed; + struct ib_wc wc; + struct ib_send_wr *bad_wr; + int ne; + + cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; + cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; + cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; + cb->rdma_sq_wr.sg_list->length = 0; + cb->rdma_sq_wr.num_sge = 0; + + microtime(&start); + for (i=0; i < 100000; i++) { + if (ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr)) { + PRINTF(cb, "Couldn't post send\n"); + return; + } + do { + ne = ib_poll_cq(cb->cq, 1, &wc); + } while (ne == 0); + if (ne < 0) { + PRINTF(cb, "poll CQ failed %d\n", ne); + return; + } + if (wc.status != IB_WC_SUCCESS) { + PRINTF(cb, "Completion wth error at %s:\n", + cb->server ? "server" : "client"); + PRINTF(cb, "Failed status %d: wr_id %d\n", + wc.status, (int) wc.wr_id); + return; + } + } + microtime(&stop); + + if (stop.tv_usec < start.tv_usec) { + stop.tv_usec += 1000000; + stop.tv_sec -= 1; + } + sec = stop.tv_sec - start.tv_sec; + usec = stop.tv_usec - start.tv_usec; + elapsed = sec * 1000000 + usec; + PRINTF(cb, "0B-write-lat iters 100000 usec %llu\n", elapsed); +} +#endif + + rlat_test(cb); +} + +static void krping_wlat_test_client(struct krping_cb *cb) +{ + struct ib_send_wr *bad_wr; + struct ib_wc wc; + int ret; + + cb->state = RDMA_READ_ADV; + + /* Send STAG/TO/Len to client */ + krping_format_send(cb, cb->start_dma_addr); + if (cb->state == ERROR) { + PRINTF(cb, "krping_format_send failed\n"); + return; + } + ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + return; + } + + /* Spin waiting for send completion */ + while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); + if (ret < 0) { + PRINTF(cb, "poll error %d\n", ret); + return; + } + if (wc.status) { + PRINTF(cb, "send completion error %d\n", wc.status); + return; + } + + /* Spin waiting for server's Start STAG/TO/Len */ + while (cb->state < RDMA_WRITE_ADV) { + krping_cq_event_handler(cb->cq, cb); + } + + wlat_test(cb); +} + +static void krping_bw_test_client(struct krping_cb *cb) +{ + struct ib_send_wr *bad_wr; + struct ib_wc wc; + int ret; + + cb->state = RDMA_READ_ADV; + + /* Send STAG/TO/Len to client */ + krping_format_send(cb, cb->start_dma_addr); + if (cb->state == ERROR) { + PRINTF(cb, "krping_format_send failed\n"); + return; + } + ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "post send error %d\n", ret); + return; + } + + /* Spin waiting for send completion */ + while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); + if (ret < 0) { + PRINTF(cb, "poll error %d\n", ret); + return; + } + if (wc.status) { + PRINTF(cb, "send completion error %d\n", wc.status); + return; + } + + /* Spin waiting for server's Start STAG/TO/Len */ + while (cb->state < RDMA_WRITE_ADV) { + krping_cq_event_handler(cb->cq, cb); + } + + bw_test(cb); +} + + +/* + * fastreg 2 valid different mrs and verify the completions. + */ +static void krping_fr_test1(struct krping_cb *cb) +{ + struct ib_fast_reg_page_list *pl; + struct ib_send_wr fr, *bad; + struct ib_wc wc; + struct ib_mr *mr1, *mr2; + int i; + int ret; + int size = cb->size; + int plen = (((size - 1) & PAGE_MASK) + PAGE_SIZE) >> PAGE_SHIFT; + int count = 0; + + pl = ib_alloc_fast_reg_page_list(cb->qp->device, plen); + if (IS_ERR(pl)) { + PRINTF(cb, "ib_alloc_fast_reg_page_list failed %ld\n", PTR_ERR(pl)); + return; + } + + mr1 = ib_alloc_fast_reg_mr(cb->pd, plen); + if (IS_ERR(mr1)) { + PRINTF(cb, "ib_alloc_fast_reg_mr failed %ld\n", PTR_ERR(pl)); + goto err1; + } + mr2 = ib_alloc_fast_reg_mr(cb->pd, plen); + if (IS_ERR(mr2)) { + PRINTF(cb, "ib_alloc_fast_reg_mr failed %ld\n", PTR_ERR(pl)); + goto err2; + } + + + for (i=0; ipage_list[i] = i * PAGE_SIZE; + + memset(&fr, 0, sizeof fr); + fr.opcode = IB_WR_FAST_REG_MR; + fr.wr_id = 1; + fr.wr.fast_reg.page_shift = PAGE_SHIFT; + fr.wr.fast_reg.length = size; + fr.wr.fast_reg.page_list = pl; + fr.wr.fast_reg.page_list_len = plen; + fr.wr.fast_reg.iova_start = 0; + fr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; + fr.send_flags = IB_SEND_SIGNALED; + fr.wr.fast_reg.rkey = mr1->rkey; + DEBUG_LOG(cb, "%s fr1: stag 0x%x plen %u size %u depth %u\n", __func__, fr.wr.fast_reg.rkey, plen, cb->size, cb->txdepth); + ret = ib_post_send(cb->qp, &fr, &bad); + if (ret) { + PRINTF(cb, "ib_post_send failed %d\n", ret); + goto err3; + } + fr.wr.fast_reg.rkey = mr2->rkey; + DEBUG_LOG(cb, "%s fr2: stag 0x%x plen %u size %u depth %u\n", __func__, fr.wr.fast_reg.rkey, plen, cb->size, cb->txdepth); + ret = ib_post_send(cb->qp, &fr, &bad); + if (ret) { + PRINTF(cb, "ib_post_send failed %d\n", ret); + goto err3; + } + + DEBUG_LOG(cb, "sleeping 1 second\n"); + wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); + do { + ret = ib_poll_cq(cb->cq, 1, &wc); + if (ret < 0) { + PRINTF(cb, "ib_poll_cq failed %d\n", ret); + goto err3; + } + if (ret == 1) { + DEBUG_LOG(cb, "completion status %u wr %s\n", + wc.status, wc.wr_id == 1 ? "fr" : "inv"); + count++; + } else if (krping_sigpending()) { + PRINTF(cb, "signal!\n"); + goto err3; + } + + wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); + } while (count != 2); +err3: + DEBUG_LOG(cb, "sleeping 1 second\n"); + wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); + DEBUG_LOG(cb, "draining the cq...\n"); + do { + ret = ib_poll_cq(cb->cq, 1, &wc); + if (ret < 0) { + PRINTF(cb, "ib_poll_cq failed %d\n", ret); + break; + } + if (ret == 1) { + PRINTF(cb, "completion %u opcode %u\n", wc.status, wc.opcode); + } + } while (ret == 1); + DEBUG_LOG(cb, "destroying fr mr2!\n"); + + ib_dereg_mr(mr2); +err2: + DEBUG_LOG(cb, "destroying fr mr1!\n"); + ib_dereg_mr(mr1); +err1: + DEBUG_LOG(cb, "destroying fr page list!\n"); + ib_free_fast_reg_page_list(pl); + DEBUG_LOG(cb, "%s done!\n", __func__); +} + +/* + * fastreg the same mr twice, 2nd one should produce error cqe. + */ +static void krping_fr_test2(struct krping_cb *cb) +{ + struct ib_fast_reg_page_list *pl; + struct ib_send_wr fr, *bad; + struct ib_wc wc; + struct ib_mr *mr1; + int i; + int ret; + int size = cb->size; + int plen = (((size - 1) & PAGE_MASK) + PAGE_SIZE) >> PAGE_SHIFT; + int count = 0; + + pl = ib_alloc_fast_reg_page_list(cb->qp->device, plen); + if (IS_ERR(pl)) { + PRINTF(cb, "ib_alloc_fast_reg_page_list failed %ld\n", PTR_ERR(pl)); + return; + } + + mr1 = ib_alloc_fast_reg_mr(cb->pd, plen); + if (IS_ERR(mr1)) { + PRINTF(cb, "ib_alloc_fast_reg_mr failed %ld\n", PTR_ERR(pl)); + goto err1; + } + + for (i=0; ipage_list[i] = i * PAGE_SIZE; + + memset(&fr, 0, sizeof fr); + fr.opcode = IB_WR_FAST_REG_MR; + fr.wr_id = 1; + fr.wr.fast_reg.page_shift = PAGE_SHIFT; + fr.wr.fast_reg.length = size; + fr.wr.fast_reg.page_list = pl; + fr.wr.fast_reg.page_list_len = plen; + fr.wr.fast_reg.iova_start = 0; + fr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; + fr.send_flags = IB_SEND_SIGNALED; + fr.wr.fast_reg.rkey = mr1->rkey; + DEBUG_LOG(cb, "%s fr1: stag 0x%x plen %u size %u depth %u\n", __func__, fr.wr.fast_reg.rkey, plen, cb->size, cb->txdepth); + ret = ib_post_send(cb->qp, &fr, &bad); + if (ret) { + PRINTF(cb, "ib_post_send failed %d\n", ret); + goto err3; + } + DEBUG_LOG(cb, "%s fr2: stag 0x%x plen %u size %u depth %u\n", __func__, fr.wr.fast_reg.rkey, plen, cb->size, cb->txdepth); + ret = ib_post_send(cb->qp, &fr, &bad); + if (ret) { + PRINTF(cb, "ib_post_send failed %d\n", ret); + goto err3; + } + + DEBUG_LOG(cb, "sleeping 1 second\n"); + wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); + do { + ret = ib_poll_cq(cb->cq, 1, &wc); + if (ret < 0) { + PRINTF(cb, "ib_poll_cq failed %d\n", ret); + goto err3; + } + if (ret == 1) { + DEBUG_LOG(cb, "completion status %u wr %s\n", + wc.status, wc.wr_id == 1 ? "fr" : "inv"); + count++; + } else if (krping_sigpending()) { + PRINTF(cb, "signal!\n"); + goto err3; + } + wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); + } while (count != 2); +err3: + DEBUG_LOG(cb, "sleeping 1 second\n"); + wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); + DEBUG_LOG(cb, "draining the cq...\n"); + do { + ret = ib_poll_cq(cb->cq, 1, &wc); + if (ret < 0) { + PRINTF(cb, "ib_poll_cq failed %d\n", ret); + break; + } + if (ret == 1) { + PRINTF(cb, "completion %u opcode %u\n", wc.status, wc.opcode); + } + } while (ret == 1); + DEBUG_LOG(cb, "destroying fr mr1!\n"); + ib_dereg_mr(mr1); +err1: + DEBUG_LOG(cb, "destroying fr page list!\n"); + ib_free_fast_reg_page_list(pl); + DEBUG_LOG(cb, "%s done!\n", __func__); +} + +/* + * fastreg pipelined in a loop as fast as we can until the user interrupts. + * NOTE: every 9 seconds we sleep for 1 second to keep the kernel happy. + */ +static void krping_fr_test3(struct krping_cb *cb) +{ + struct ib_fast_reg_page_list *pl; + struct ib_send_wr fr, inv, *bad; + struct ib_wc wc; + u8 key = 0; + struct ib_mr *mr; + int i; + int ret; + int size = cb->size; + int plen = (((size - 1) & PAGE_MASK) + PAGE_SIZE) >> PAGE_SHIFT; + unsigned long start; + int count = 0; + int scnt = 0; + + + pl = ib_alloc_fast_reg_page_list(cb->qp->device, plen); + if (IS_ERR(pl)) { + PRINTF(cb, "ib_alloc_fast_reg_page_list failed %ld\n", PTR_ERR(pl)); + return; + } + + mr = ib_alloc_fast_reg_mr(cb->pd, plen); + if (IS_ERR(mr)) { + PRINTF(cb, "ib_alloc_fast_reg_mr failed %ld\n", PTR_ERR(pl)); + goto err1; + } + + for (i=0; ipage_list[i] = i * PAGE_SIZE; + + memset(&fr, 0, sizeof fr); + fr.opcode = IB_WR_FAST_REG_MR; + fr.wr.fast_reg.page_shift = PAGE_SHIFT; + fr.wr.fast_reg.length = size; + fr.wr.fast_reg.page_list = pl; + fr.wr.fast_reg.page_list_len = plen; + fr.wr.fast_reg.iova_start = 0; + fr.send_flags = IB_SEND_SIGNALED; + fr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; + fr.next = &inv; + memset(&inv, 0, sizeof inv); + inv.opcode = IB_WR_LOCAL_INV; + inv.send_flags = IB_SEND_SIGNALED; + + DEBUG_LOG(cb, "fr_test: stag index 0x%x plen %u size %u depth %u\n", mr->rkey >> 8, plen, cb->size, cb->txdepth); + start = time_uptime; + while (1) { + if ((time_uptime - start) >= 9) { + DEBUG_LOG(cb, "fr_test: pausing 1 second! count %u latest size %u plen %u\n", count, size, plen); + wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); + if (cb->state == ERROR) + break; + start = time_uptime; + } + while (scnt < (cb->txdepth>>1)) { + ib_update_fast_reg_key(mr, ++key); + fr.wr.fast_reg.rkey = mr->rkey; + inv.ex.invalidate_rkey = mr->rkey; + size = arc4random() % cb->size; + if (size == 0) + size = cb->size; + plen = (((size - 1) & PAGE_MASK) + PAGE_SIZE) >> PAGE_SHIFT; + fr.wr.fast_reg.length = size; + fr.wr.fast_reg.page_list_len = plen; + ret = ib_post_send(cb->qp, &fr, &bad); + if (ret) { + PRINTF(cb, "ib_post_send failed %d\n", ret); + goto err2; + } + scnt+=2; + } + + do { + ret = ib_poll_cq(cb->cq, 1, &wc); + if (ret < 0) { + PRINTF(cb, "ib_poll_cq failed %d\n", ret); + goto err2; + } + if (ret == 1) { + if (wc.status) { + PRINTF(cb, "completion error %u\n", wc.status); + goto err2; + } + count++; + scnt--; + } + else if (krping_sigpending()) { + PRINTF(cb, "signal!\n"); + goto err2; + } + } while (ret == 1); + } +err2: + DEBUG_LOG(cb, "sleeping 1 second\n"); + wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); + DEBUG_LOG(cb, "draining the cq...\n"); + do { + ret = ib_poll_cq(cb->cq, 1, &wc); + if (ret < 0) { + PRINTF(cb, "ib_poll_cq failed %d\n", ret); + break; + } + if (ret == 1) { + if (wc.status) { + PRINTF(cb, "completion error %u opcode %u\n", wc.status, wc.opcode); + } + } + } while (ret == 1); + DEBUG_LOG(cb, "fr_test: done!\n"); + ib_dereg_mr(mr); +err1: + DEBUG_LOG(cb, "destroying fr page list!\n"); + ib_free_fast_reg_page_list(pl); + DEBUG_LOG(cb, "%s done!\n", __func__); +} + +/* + * fastreg 1 and invalidate 1 mr and verify completion. + */ +static void krping_fr_test4(struct krping_cb *cb) +{ + struct ib_fast_reg_page_list *pl; + struct ib_send_wr fr, inv, *bad; + struct ib_wc wc; + struct ib_mr *mr1; + int i; + int ret; + int size = cb->size; + int plen = (((size - 1) & PAGE_MASK) + PAGE_SIZE) >> PAGE_SHIFT; + int count = 0; + + pl = ib_alloc_fast_reg_page_list(cb->qp->device, plen); + if (IS_ERR(pl)) { + PRINTF(cb, "ib_alloc_fast_reg_page_list failed %ld\n", PTR_ERR(pl)); + return; + } + + mr1 = ib_alloc_fast_reg_mr(cb->pd, plen); + if (IS_ERR(mr1)) { + PRINTF(cb, "ib_alloc_fast_reg_mr failed %ld\n", PTR_ERR(pl)); + goto err1; + } + + for (i=0; ipage_list[i] = i * PAGE_SIZE; + + memset(&fr, 0, sizeof fr); + fr.opcode = IB_WR_FAST_REG_MR; + fr.wr_id = 1; + fr.wr.fast_reg.page_shift = PAGE_SHIFT; + fr.wr.fast_reg.length = size; + fr.wr.fast_reg.page_list = pl; + fr.wr.fast_reg.page_list_len = plen; + fr.wr.fast_reg.iova_start = 0; + fr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; + fr.send_flags = IB_SEND_SIGNALED; + fr.wr.fast_reg.rkey = mr1->rkey; + fr.next = &inv; + memset(&inv, 0, sizeof inv); + inv.opcode = IB_WR_LOCAL_INV; + inv.ex.invalidate_rkey = mr1->rkey; + + DEBUG_LOG(cb, "%s fr1: stag 0x%x plen %u size %u depth %u\n", __func__, fr.wr.fast_reg.rkey, plen, cb->size, cb->txdepth); + ret = ib_post_send(cb->qp, &fr, &bad); + if (ret) { + PRINTF(cb, "ib_post_send failed %d\n", ret); + goto err3; + } + DEBUG_LOG(cb, "sleeping 1 second\n"); + wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); + do { + ret = ib_poll_cq(cb->cq, 1, &wc); + if (ret < 0) { + PRINTF(cb, "ib_poll_cq failed %d\n", ret); + goto err3; + } + if (ret == 1) { + DEBUG_LOG(cb, "completion status %u wr %s\n", + wc.status, wc.wr_id == 1 ? "fr" : "inv"); + count++; + } else if (krping_sigpending()) { + PRINTF(cb, "signal!\n"); + goto err3; + } + wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); + } while (count != 1); +err3: + DEBUG_LOG(cb, "sleeping 1 second\n"); + wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); + DEBUG_LOG(cb, "draining the cq...\n"); + do { + ret = ib_poll_cq(cb->cq, 1, &wc); + if (ret < 0) { + PRINTF(cb, "ib_poll_cq failed %d\n", ret); + break; + } + if (ret == 1) { + PRINTF(cb, "completion %u opcode %u\n", wc.status, wc.opcode); + } + } while (ret == 1); + DEBUG_LOG(cb, "destroying fr mr1!\n"); + ib_dereg_mr(mr1); +err1: + DEBUG_LOG(cb, "destroying fr page list!\n"); + ib_free_fast_reg_page_list(pl); + DEBUG_LOG(cb, "%s done!\n", __func__); +} + +static void krping_fr_test(struct krping_cb *cb) +{ + switch (cb->testnum) { + case 1: + krping_fr_test1(cb); + break; + case 2: + krping_fr_test2(cb); + break; + case 3: + krping_fr_test3(cb); + break; + case 4: + krping_fr_test4(cb); + break; + case 5: + krping_fr_test5_client(cb); + break; + case 6: + krping_fr_test6_client(cb); + break; + default: + PRINTF(cb, "Unkown frtest num %u\n", cb->testnum); + break; + } +} + +static int krping_connect_client(struct krping_cb *cb) +{ + struct rdma_conn_param conn_param; + int ret; + + memset(&conn_param, 0, sizeof conn_param); + conn_param.responder_resources = 1; + conn_param.initiator_depth = 1; + conn_param.retry_count = 10; + + ret = rdma_connect(cb->cm_id, &conn_param); + if (ret) { + PRINTF(cb, "rdma_connect error %d\n", ret); + return ret; + } + + wait_event_interruptible(cb->sem, cb->state >= CONNECTED); + if (cb->state == ERROR) { + PRINTF(cb, "wait for CONNECTED state %d\n", cb->state); + return -1; + } + + DEBUG_LOG(cb, "rdma_connect successful\n"); + return 0; +} + +static int krping_bind_client(struct krping_cb *cb) +{ + union { + struct sockaddr_in v4; + struct sockaddr_in6 v6; + } sin; + int ret; + + memset(&sin, 0, sizeof(sin)); + + switch (cb->addr_type) { + case AF_INET: + sin.v4.sin_len = sizeof sin.v4; + sin.v4.sin_family = AF_INET; + sin.v4.sin_addr = cb->addr.v4; + sin.v4.sin_port = cb->port; + break; + case AF_INET6: + sin.v6.sin6_len = sizeof sin.v6; + sin.v6.sin6_family = AF_INET6; + sin.v6.sin6_addr = cb->addr.v6; + sin.v6.sin6_port = cb->port; + break; + default: + return (-EINVAL); + } + + ret = rdma_resolve_addr(cb->cm_id, NULL, (struct sockaddr *) &sin, + 2000); + if (ret) { + PRINTF(cb, "rdma_resolve_addr error %d\n", ret); + return ret; + } + + wait_event_interruptible(cb->sem, cb->state >= ROUTE_RESOLVED); + if (cb->state != ROUTE_RESOLVED) { + PRINTF(cb, + "addr/route resolution did not resolve: state %d\n", + cb->state); + return -EINTR; + } + + if (cb->mem == FASTREG && !fastreg_supported(cb, 0)) + return -EINVAL; + + DEBUG_LOG(cb, "rdma_resolve_addr - rdma_resolve_route successful\n"); + return 0; +} + +static void krping_run_client(struct krping_cb *cb) +{ + struct ib_recv_wr *bad_wr; + int ret; + + ret = krping_bind_client(cb); + if (ret) + return; + + ret = krping_setup_qp(cb, cb->cm_id); + if (ret) { + PRINTF(cb, "setup_qp failed: %d\n", ret); + return; + } + + ret = krping_setup_buffers(cb); + if (ret) { + PRINTF(cb, "krping_setup_buffers failed: %d\n", ret); + goto err1; + } + + ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); + if (ret) { + PRINTF(cb, "ib_post_recv failed: %d\n", ret); + goto err2; + } + + ret = krping_connect_client(cb); + if (ret) { + PRINTF(cb, "connect error %d\n", ret); + goto err2; + } + + if (cb->wlat) + krping_wlat_test_client(cb); + else if (cb->rlat) + krping_rlat_test_client(cb); + else if (cb->bw) + krping_bw_test_client(cb); + else if (cb->frtest) + krping_fr_test(cb); + else + krping_test_client(cb); + rdma_disconnect(cb->cm_id); +err2: + krping_free_buffers(cb); +err1: + krping_free_qp(cb); +} + +static uint16_t +krping_get_ipv6_scope_id(char *name) +{ + struct ifnet *ifp; + uint16_t retval; + + if (name == NULL) + return (0); + CURVNET_SET_QUIET(TD_TO_VNET(curthread)); + ifp = ifunit_ref(name); + CURVNET_RESTORE(); + if (ifp == NULL) + return (0); + retval = ifp->if_index; + if_rele(ifp); + return (retval); +} + +int krping_doit(char *cmd, void *cookie) +{ + struct krping_cb *cb; + int op; + int ret = 0; + char *optarg; + char *scope; + unsigned long optint; + + cb = kzalloc(sizeof(*cb), GFP_KERNEL); + if (!cb) + return -ENOMEM; + + mutex_lock(&krping_mutex); + list_add_tail(&cb->list, &krping_cbs); + mutex_unlock(&krping_mutex); + + cb->cookie = cookie; + cb->server = -1; + cb->state = IDLE; + cb->size = 64; + cb->txdepth = RPING_SQ_DEPTH; + cb->mem = DMA; + cb->addr_type = AF_INET; + init_waitqueue_head(&cb->sem); + + while ((op = krping_getopt("krping", &cmd, krping_opts, NULL, &optarg, + &optint)) != 0) { + switch (op) { + case 'a': + cb->addr_str = optarg; + cb->addr_type = AF_INET; + DEBUG_LOG(cb, "ipv4addr (%s)\n", optarg); + if (inet_pton(AF_INET, optarg, &cb->addr) != 1) { + PRINTF(cb, "bad addr string %s\n", + optarg); + ret = EINVAL; + } + break; + case 'A': + cb->addr_str = optarg; + cb->addr_type = AF_INET6; + DEBUG_LOG(cb, "ipv6addr (%s)\n", optarg); + scope = strstr(optarg, "%"); + /* extract scope ID, if any */ + if (scope != NULL) + *scope++ = 0; + /* extract IPv6 network address */ + if (inet_pton(AF_INET6, optarg, &cb->addr) != 1) { + PRINTF(cb, "bad addr string %s\n", + optarg); + ret = EINVAL; + } else if (IN6_IS_SCOPE_LINKLOCAL(&cb->addr.v6) || + IN6_IS_ADDR_MC_INTFACELOCAL(&cb->addr.v6)) { + uint16_t scope_id = krping_get_ipv6_scope_id(scope); + DEBUG_LOG(cb, "ipv6 scope ID = %d\n", scope_id); + cb->addr.v6.s6_addr[2] = scope_id >> 8; + cb->addr.v6.s6_addr[3] = scope_id & 0xFF; + } + break; + case 'p': + cb->port = htons(optint); + DEBUG_LOG(cb, "port %d\n", (int)optint); + break; + case 'P': + cb->poll = 1; + DEBUG_LOG(cb, "server\n"); + break; + case 's': + cb->server = 1; + DEBUG_LOG(cb, "server\n"); + break; + case 'c': + cb->server = 0; + DEBUG_LOG(cb, "client\n"); + break; + case 'S': + cb->size = optint; + if ((cb->size < 1) || + (cb->size > RPING_BUFSIZE)) { + PRINTF(cb, "Invalid size %d " + "(valid range is 1 to %d)\n", + cb->size, RPING_BUFSIZE); + ret = EINVAL; + } else + DEBUG_LOG(cb, "size %d\n", (int)optint); + break; + case 'C': + cb->count = optint; + if (cb->count < 0) { + PRINTF(cb, "Invalid count %d\n", + cb->count); + ret = EINVAL; + } else + DEBUG_LOG(cb, "count %d\n", (int) cb->count); + break; + case 'v': + cb->verbose++; + DEBUG_LOG(cb, "verbose\n"); + break; + case 'V': + cb->validate++; + DEBUG_LOG(cb, "validate data\n"); + break; + case 'l': + cb->wlat++; + break; + case 'L': + cb->rlat++; + break; + case 'B': + cb->bw++; + break; + case 'd': + cb->duplex++; + break; + case 'm': + if (!strncmp(optarg, "dma", 3)) + cb->mem = DMA; + else if (!strncmp(optarg, "fastreg", 7)) + cb->mem = FASTREG; + else if (!strncmp(optarg, "mw", 2)) + cb->mem = MW; + else if (!strncmp(optarg, "mr", 2)) + cb->mem = MR; + else { + PRINTF(cb, "unknown mem mode %s. " + "Must be dma, fastreg, mw, or mr\n", + optarg); + ret = -EINVAL; + break; + } + break; + case 'I': + cb->server_invalidate = 1; + break; + case 'T': + cb->txdepth = optint; + DEBUG_LOG(cb, "txdepth %d\n", (int) cb->txdepth); + break; + case 'Z': + cb->local_dma_lkey = 1; + DEBUG_LOG(cb, "using local dma lkey\n"); + break; + case 'R': + cb->read_inv = 1; + DEBUG_LOG(cb, "using read-with-inv\n"); + break; + case 'f': + cb->frtest = 1; + cb->testnum = optint; + DEBUG_LOG(cb, "fast-reg test!\n"); + break; + default: + PRINTF(cb, "unknown opt %s\n", optarg); + ret = -EINVAL; + break; + } + } + if (ret) + goto out; + + if (cb->server == -1) { + PRINTF(cb, "must be either client or server\n"); + ret = -EINVAL; + goto out; + } + + if ((cb->frtest + cb->bw + cb->rlat + cb->wlat) > 1) { + PRINTF(cb, "Pick only one test: fr, bw, rlat, wlat\n"); + ret = -EINVAL; + goto out; + } + if (cb->server_invalidate && cb->mem != FASTREG) { + PRINTF(cb, "server_invalidate only valid with fastreg mem_mode\n"); + ret = -EINVAL; + goto out; + } + + if (cb->read_inv && cb->mem != FASTREG) { + PRINTF(cb, "read_inv only valid with fastreg mem_mode\n"); + ret = -EINVAL; + goto out; + } + + if (cb->mem != MR && (cb->wlat || cb->rlat || cb->bw || cb->frtest)) { + PRINTF(cb, "wlat, rlat, and bw tests only support mem_mode MR\n"); + ret = -EINVAL; + goto out; + } + + cb->cm_id = rdma_create_id(krping_cma_event_handler, cb, RDMA_PS_TCP, IB_QPT_RC); + if (IS_ERR(cb->cm_id)) { + ret = PTR_ERR(cb->cm_id); + PRINTF(cb, "rdma_create_id error %d\n", ret); + goto out; + } + DEBUG_LOG(cb, "created cm_id %p\n", cb->cm_id); + + if (cb->server) + krping_run_server(cb); + else + krping_run_client(cb); + + DEBUG_LOG(cb, "destroy cm_id %p\n", cb->cm_id); + rdma_destroy_id(cb->cm_id); +out: + mutex_lock(&krping_mutex); + list_del(&cb->list); + mutex_unlock(&krping_mutex); + kfree(cb); + return ret; +} + +void +krping_walk_cb_list(void (*f)(struct krping_stats *, void *), void *arg) +{ + struct krping_cb *cb; + + mutex_lock(&krping_mutex); + list_for_each_entry(cb, &krping_cbs, list) + (*f)(cb->pd ? &cb->stats : NULL, arg); + mutex_unlock(&krping_mutex); +} + +void krping_init(void) +{ + + mutex_init(&krping_mutex); +} Property changes on: projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/krping.c ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/krping.h =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/krping.h (nonexistent) +++ projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/krping.h (revision 330503) @@ -0,0 +1,20 @@ +/* + * $FreeBSD$ + */ + +struct krping_stats { + unsigned long long send_bytes; + unsigned long long send_msgs; + unsigned long long recv_bytes; + unsigned long long recv_msgs; + unsigned long long write_bytes; + unsigned long long write_msgs; + unsigned long long read_bytes; + unsigned long long read_msgs; + char name[16]; +}; + +int krping_doit(char *, void *); +void krping_walk_cb_list(void (*)(struct krping_stats *, void *), void *); +void krping_init(void); +int krping_sigpending(void); Property changes on: projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/krping.h ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/krping_dev.c =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/krping_dev.c (nonexistent) +++ projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/krping_dev.c (revision 330503) @@ -0,0 +1,220 @@ +/* + * This code lifted from: + * Simple `echo' pseudo-device KLD + * Murray Stokely + * Converted to 5.X by Søren (Xride) Straarup + */ + +/* + * /bin/echo "server,port=9999,addr=192.168.69.142,validate" > /dev/krping + * /bin/echo "client,port=9999,addr=192.168.69.142,validate" > /dev/krping + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include /* uprintf */ +#include +#include /* defines used in kernel.h */ +#include /* types used in module initialization */ +#include /* cdevsw struct */ +#include /* uio struct */ +#include +#include +#include +#include + +#include "krping.h" + +#define BUFFERSIZE 512 + +SYSCTL_NODE(_dev, OID_AUTO, krping, CTLFLAG_RW, 0, "kernel rping module"); + +int krping_debug = 0; +SYSCTL_INT(_dev_krping, OID_AUTO, debug, CTLFLAG_RW, &krping_debug, 0 , ""); + +/* Function prototypes */ +static d_open_t krping_open; +static d_close_t krping_close; +static d_read_t krping_read; +static d_write_t krping_write; + +/* Character device entry points */ +static struct cdevsw krping_cdevsw = { + .d_version = D_VERSION, + .d_open = krping_open, + .d_close = krping_close, + .d_read = krping_read, + .d_write = krping_write, + .d_name = "krping", +}; + +typedef struct s_krping { + char msg[BUFFERSIZE]; + int len; +} krping_t; + +struct stats_list_entry { + STAILQ_ENTRY(stats_list_entry) link; + struct krping_stats *stats; +}; +STAILQ_HEAD(stats_list, stats_list_entry); + +/* vars */ +static struct cdev *krping_dev; + +static int +krping_loader(struct module *m, int what, void *arg) +{ + int err = 0; + + switch (what) { + case MOD_LOAD: /* kldload */ + krping_init(); + krping_dev = make_dev(&krping_cdevsw, 0, UID_ROOT, GID_WHEEL, + 0600, "krping"); + printf("Krping device loaded.\n"); + break; + case MOD_UNLOAD: + destroy_dev(krping_dev); + printf("Krping device unloaded.\n"); + break; + default: + err = EOPNOTSUPP; + break; + } + + return (err); +} + +static int +krping_open(struct cdev *dev, int oflags, int devtype, struct thread *p) +{ + + return (0); +} + +static int +krping_close(struct cdev *dev, int fflag, int devtype, struct thread *p) +{ + + return 0; +} + +static void +krping_copy_stats(struct krping_stats *stats, void *arg) +{ + struct stats_list_entry *s; + struct stats_list *list = arg; + + s = malloc(sizeof(*s), M_DEVBUF, M_NOWAIT | M_ZERO); + if (s == NULL) + return; + if (stats != NULL) { + s->stats = malloc(sizeof(*stats), M_DEVBUF, M_NOWAIT | M_ZERO); + if (s->stats == NULL) { + free(s, M_DEVBUF); + return; + } + *s->stats = *stats; + } + STAILQ_INSERT_TAIL(list, s, link); +} + +static int +krping_read(struct cdev *dev, struct uio *uio, int ioflag) +{ + int num = 1; + struct stats_list list; + struct stats_list_entry *e; + + STAILQ_INIT(&list); + krping_walk_cb_list(krping_copy_stats, &list); + + if (STAILQ_EMPTY(&list)) + return (0); + + uprintf("krping: %4s %10s %10s %10s %10s %10s %10s %10s %10s %10s\n", + "num", "device", "snd bytes", "snd msgs", "rcv bytes", "rcv msgs", + "wr bytes", "wr msgs", "rd bytes", "rd msgs"); + + while (!STAILQ_EMPTY(&list)) { + e = STAILQ_FIRST(&list); + STAILQ_REMOVE_HEAD(&list, link); + if (e->stats == NULL) + uprintf("krping: %d listen\n", num); + else { + struct krping_stats *stats = e->stats; + + uprintf("krping: %4d %10s %10llu %10llu %10llu %10llu " + "%10llu %10llu %10llu %10llu\n", num, stats->name, + stats->send_bytes, stats->send_msgs, + stats->recv_bytes, stats->recv_msgs, + stats->write_bytes, stats->write_msgs, + stats->read_bytes, stats->read_msgs); + free(stats, M_DEVBUF); + } + num++; + free(e, M_DEVBUF); + } + + return (0); +} + +static int +krping_write(struct cdev *dev, struct uio *uio, int ioflag) +{ + int err = 0; + int amt; + int remain = BUFFERSIZE; + char *cp; + krping_t *krpingmsg; + + krpingmsg = malloc(sizeof *krpingmsg, M_DEVBUF, M_WAITOK|M_ZERO); + if (!krpingmsg) { + uprintf("Could not malloc mem!\n"); + return ENOMEM; + } + + cp = krpingmsg->msg; + while (uio->uio_resid) { + amt = MIN(uio->uio_resid, remain); + if (amt == 0) + break; + + /* Copy the string in from user memory to kernel memory */ + err = uiomove(cp, amt, uio); + if (err) { + uprintf("Write failed: bad address!\n"); + return err; + } + cp += amt; + remain -= amt; + } + + if (uio->uio_resid != 0) { + uprintf("Message too big. max size is %d!\n", BUFFERSIZE); + return EMSGSIZE; + } + + /* null terminate and remove the \n */ + cp--; + *cp = 0; + krpingmsg->len = (unsigned long)(cp - krpingmsg->msg); + uprintf("krping: write string = |%s|\n", krpingmsg->msg); + err = krping_doit(krpingmsg->msg, curproc); + free(krpingmsg, M_DEVBUF); + return(err); +} + +int +krping_sigpending(void) +{ + + return (SIGPENDING(curthread)); +} + +DEV_MODULE(krping, krping_loader, NULL); +MODULE_DEPEND(krping, ibcore, 1, 1, 1); Property changes on: projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/krping_dev.c ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/getopt.c =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/getopt.c (nonexistent) +++ projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/getopt.c (revision 330503) @@ -0,0 +1,78 @@ +/* + * lifted from fs/ncpfs/getopt.c + * + */ +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include + +#include "getopt.h" + +/** + * krping_getopt - option parser + * @caller: name of the caller, for error messages + * @options: the options string + * @opts: an array of &struct option entries controlling parser operations + * @optopt: output; will contain the current option + * @optarg: output; will contain the value (if one exists) + * @flag: output; may be NULL; should point to a long for or'ing flags + * @value: output; may be NULL; will be overwritten with the integer value + * of the current argument. + * + * Helper to parse options on the format used by mount ("a=b,c=d,e,f"). + * Returns opts->val if a matching entry in the 'opts' array is found, + * 0 when no more tokens are found, -1 if an error is encountered. + */ +int krping_getopt(const char *caller, char **options, + const struct krping_option *opts, char **optopt, + char **optarg, unsigned long *value) +{ + char *token; + char *val; + + do { + if ((token = strsep(options, ",")) == NULL) + return 0; + } while (*token == '\0'); + if (optopt) + *optopt = token; + + if ((val = strchr (token, '=')) != NULL) { + *val++ = 0; + } + *optarg = val; + for (; opts->name; opts++) { + if (!strcmp(opts->name, token)) { + if (!val) { + if (opts->has_arg & OPT_NOPARAM) { + return opts->val; + } + printk(KERN_INFO "%s: the %s option requires " + "an argument\n", caller, token); + return -EINVAL; + } + if (opts->has_arg & OPT_INT) { + char* v; + + *value = simple_strtoul(val, &v, 0); + if (!*v) { + return opts->val; + } + printk(KERN_INFO "%s: invalid numeric value " + "in %s=%s\n", caller, token, val); + return -EDOM; + } + if (opts->has_arg & OPT_STRING) { + return opts->val; + } + printk(KERN_INFO "%s: unexpected argument %s to the " + "%s option\n", caller, val, token); + return -EINVAL; + } + } + printk(KERN_INFO "%s: Unrecognized option %s\n", caller, token); + return -EOPNOTSUPP; +} Property changes on: projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/getopt.c ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/getopt.h =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/getopt.h (nonexistent) +++ projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/getopt.h (revision 330503) @@ -0,0 +1,21 @@ +/* + * lifted from fs/ncpfs/getopt.c + * + * $FreeBSD$ + */ +#ifndef _KRPING_GETOPT_H +#define _KRPING_GETOPT_H + +#define OPT_NOPARAM 1 +#define OPT_INT 2 +#define OPT_STRING 4 +struct krping_option { + const char *name; + unsigned int has_arg; + int val; +}; + +extern int krping_getopt(const char *caller, char **options, const struct krping_option *opts, + char **optopt, char **optarg, unsigned long *value); + +#endif /* _KRPING_GETOPT_H */ Property changes on: projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat/getopt.h ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat (nonexistent) +++ projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat (revision 330503) Property changes on: projects/bsd_rdma_4_9_stable_11/sys/contrib/rdma/krping_compat ___________________________________________________________________ Added: svn:mergeinfo ## -0,0 +0,10 ## Merged /projects/netbsd-tests-upstream-01-2017/sys/contrib/rdma/krping:r312125-313435 Merged /projects/clang390-import/sys/contrib/rdma/krping:r303250-309123 Merged /projects/clang391-import/sys/contrib/rdma/krping:r309166-310192 Merged /head/sys/contrib/rdma/krping:r302438-302440,302448-302450,302456-302457,302459-302460,302465,302470-302476,302480-302484,302486-302497,302499-302500,302502-302508,302510-302511,302514-302524,302528-302532,302534,302540-302554,302556-302561,302565,302567,302571-302573,302576-302577,302580-302581,302584-302587,302591-302592,302605-302612,302614,302617-302624,302626,302629-302638,302664,302666-302668,302673-302674,302692-302704,302706-302710,302713,302723,302726,302731,302733,302737,302770-302772,302778,302783-302785,302787-302790,302793-302795,302797-302798,302801-302806,302808-302819,302823-302824,302826-302827,302831,302834-302843,302845,302847-302853,302855-302861,302864,302867-302876,302878-302894,302896-302897,302899-302900,302902,302904,302906-302908,302911,302916-302917,302919-302921,302928-302930,302932-302933,302935-302937,302940-302947,302949-302952,302957,302960-302961,302965-302966,302972-302973,302980-302981,302986,302988,302999-303004,303009-303014,303016-303026,303028,303030-303034,303039-303040,303042-303046,303052,303059,303061,303066-303074,303076,303084,303086-303088,303090-303091,303094,303098,303101,303103-303104,303106,303109,303111,303113,303115,303123,303125-303129,303131-303132,303134,303142,303144-303145,303147,303151,303153-303157,303160,303164,303166,303171,303178,303180,303182,303186,303188,303190,303195,303199,303202,303204-303205,303208,303210-303213,303217,303219,303223,303225-303227,303230,303243-303244,303253,303261,303263-303266,303271-303274,303277,303279,303282-303285,303287-303289,303299,303307-303309,303315,303322,303326-303329,303335,303338-303339,303343,303345-303354,303356,303361-303362,303366-303370,303373-303374,303379,303382-303388,303393,303396,303399-303400,303404-303406,303410,303413-303426,303429,303437-303438,303442,303444,303446,303448,303450,303453-303454,303456-303457,303461,303464-303468,303470-303478,303486,303488,303490-303492,303494,303497,303501,303503,303510,303514-303516,303518-303522,303530,303532,303536,303541-303543,303545-303548,303551,303553-303557,303559,303561-303569,303573,303576,303581,303583-303585,303587,303592-303594,303603-303605,303610-303613,303615,303626-303627,303630-303631,303637-303638,303643,303645,303647,303650,303652,303655,303657,303661,303663,303670,303674,303676-303677,303685,303687-303688,303693,303702-303707,303710,303712-303713,303715-303716,303719-303723,303725-303729,303737-303739,303744,303747,303750,303753-303755,303760,303763-303766,303771,303773,303782-303786,303788,303791-303796,303798-303800,303802,303804,303806,303813,303816,303818-303819,303822-303824,303830-303834,303836-303837,303841-303842,303845,303847,303855,303859-303860,303863,303867,303869-303870,303874,303877,303880-303881,303886-303887,303889,303891-303894,303897,303900-303901,303903-303904,303913-303914,303916,303919,303921-303924,303927-303937,303941-303949,303955,303958,303961-303962,303964,303971,303974,303978,303982,303989-303992,303998,304000-304002,304004-304009,304011-304012,304016,304018,304021,304023,304026,304033-304034,304040-304041,304043,304045-304050,304052-304055,304058-304059,304061,304066,304068,304071,304075-304077,304085,304087,304089,304102,304106,304108-304109,304111,304140,304142,304146,304149,304151-304152,304154,304160,304162,304168-304170,304174,304176,304178,304180,304182,304184-304185,304191-304192,304194,304202,304204-304206,304209,304217,304221-304222,304225-304232,304238-304239,304246-304249,304251-304256,304270-304271,304273,304279,304285-304292,304295,304297,304313,304315-304322,304327,304329-304331,304342,304374,304378,304430-304431,304438,304440-304441,304443-304444,304446-304447,304459-304461,304464,304476,304478-304479,304481-304485,304487,304492,304509,304520-304521,304530,304532,304535,304543,304545,304554-304559,304561,304563-304566,304570-304576,304579,304584,304591,304593-304601,304604-304605,304607-304608,304615,304620,304624-304626,304629,304635-304638,304640-304641,304643,304647,304649,304652,304654,304674,304676-304677,304680,304685,304687-304689,304691,304693-304694,304696-304698,304703,304710,304713,304721-304724,304728,304730,304736,304740,304742-304751,304753-304755,304779-304786,304788,304790-304791,304793,304795-304797,304799,304801-304802,304806-304812,304819,304821,304825-304826,304832-304834,304837-304838,304854-304855,304858,304873-304875,304877,304880,304886,304891-304892,304911,304916,304918,304920,304927-304928,304931,304953,304957-304959,304962,304964,304966,304969-304970,304972-304973,304975-304976,304979,304989,304991,305009,305012-305013,305015-305016,305018-305020,305022-305024,305026,305032-305034,305036,305039-305041,305044-305050,305052-305056,305058,305061,305065-305066,305071,305077-305078,305085-305086,305090,305093,305095,305104,305107-305116,305119,305122-305123,305125,305128-305130,305132-305133,305137,305140-305141,305144-305145,305147-305149,305154-305155,305160,305166-305167,305169-305171,305173-305179,305182,305191,305193-305195,305197-305203,305205-305207,305209-305213,305215-305216,305218-305219,305221-305222,305224,305241,305244,305248,305253-305258,305264-305265,305268-305270,305276-305281,305283-305285,305303-305304,305312,305319-305320,305323-305328,305331-305334,305338-305340,305342-305345,305349,305353,305355-305358,305360,305363,305365,305367,305378-305386,305388-305389,305393,305396,305398,305405-305414,305421-305422,305425,305430,305432-305433,305449,305451,305453-305456,305467-305471,305482-305486,305492,305497,305502,305504,305507-305509,305521,305524-305527,305535-305536,305539,305545-305546,305548-305549,305561,305563,305575,305578-305581,305585-305588,305590-305610,305612-305613,305620,305624,305626,305628-305629,305633,305652,305659,305661,305663,305665,305667,305671,305676-305677,305679,305684-305685,305689,305692,305695-305696,305699,305701-305704,305706-305708,305713,305715,305719,305722,305724-305725,305727-305730,305737,305739-305740,305744,305749,305751-305752,305756,305760-305761,305763,305767,305769-305772,305778,305788-305796,305801,305804,305807,305810-305814,305819,305822,305827,305834,305836,305840-305841,305851-305852,305854-305856,305862-305865,305867-305877,305887,305894-305898,305902-305903,305906,305908,305916-305928,305937-305943,305947,305951,305954,305956,305960-305970,305977-305978,305985,305987,305994-305995,305999-306009,306011-306015,306020,306026,306029-306031,306033,306036,306038,306041,306046,306048,306050,306061-306063,306071-306076,306081-306082,306087-306092,306094-306097,306102,306104,306109,306126,306129,306131-306132,306134,306137-306139,306142,306159-306162,306174,306184-306185,306188,306193,306205-306206,306209,306212,306216,306218,306220,306224,306226,306228,306232,306257,306260-306262,306264,306266-306267,306273-306277,306279,306281,306285-306286,306289-306292,306295,306297,306299-306301,306304,306306,306310-306313,306318,306320,306325,306329-306330,306332-306334,306342,306346,306349-306351,306355-306357,306359-306360,306366,306375-306377,306380,306387-306393,306395-306396,306409-306410,306417,306424-306426,306430-306433,306435,306441-306445,306447,306449,306451,306453-306454,306456-306460,306464-306465,306474,306477-306486,306488,306494-306496,306498,306504-306508,306510-306511,306513,306522,306526-306532,306534,306538,306541,306544,306550-306552,306554-306555,306557,306559-306571,306577-306579,306581-306585,306588-306589,306597-306608,306610-306622,306631,306634,306638,306640-306641,306647,306650,306652,306654,306656,306658,306665-306668,306670,306674,306677,306680,306686,306691,306695-306696,306699,306702-306704,306706,306709-306710,306712,306714-306715,306717-306725,306727-306728,306731-306736,306740,306742-306744,306746-306747,306751,306754-306756,306759,306762,306766-306767,306771,306773,306775,306782-306783,306787,306790,306792-306795,306801,306803,306806-306808,306812,306821,306823,306825-306826,306828-306830,306836-306837,306840,306852-306855,306857,306860,306877,306896-306902,306936-306939,306944,306962,306966,306999-307003,307008,307012-307013,307036-307038,307041-307042,307044,307059-307060,307064,307066-307068,307070-307072,307074,307078,307087-307089,307091-307093,307095,307098,307115,307130-307132,307141,307145,307150-307152,307154,307156,307158-307159,307182,307189-307192,307195-307196,307204-307205,307215-307218,307220,307224,307227,307233,307236,307238-307241,307261-307263,307316,307318,307322-307324,307326-307327,307332-307334,307338-307339,307343,307348,307350,307364,307366,307368-307369,307372,307374,307379-307381,307384,307390-307392,307397-307398,307400-307402,307469,307499,307501,307507,307509,307515,307517-307523,307531,307538,307541,307544-307545,307550-307551,307556-307558,307561,307564,307566-307567,307570,307578,307584,307598,307602,307624,307626,307628-307629,307637-307638,307649-307651,307653,307657,307684-307688,307691-307694,307698,307700-307702,307710-307714,307726-307727,307731,307741,307745-307747,307752,307755-307756,307759,307768-307769,307774,307779,307783,307785-307787,307800-307802,307804-307805,307808,307821,307838-307840,307842-307845,307857,307861,307863,307865-307866,307869,307873,307876,307879-307880,307890-307891,307893,307901-307904,307908,307911,307917,307936-307937,307942,307948,307950-307955,307966,307969,307975-307978,307983,307985-307991,307994,308004-308008,308010-308031,308038-308040,308049-308051,308055-308056,308064-308066,308088-308089,308094-308099,308101,308104,308106,308108-308109,308112-308114,308116-308120,308124-308126,308129,308133-308134,308139,308142,308144,308148,308150,308156-308158,308160-308169,308171-308176,308180,308183,308185-308189,308194-308195,308197,308201,308206,308209-308212,308216-308221,308225,308228,308231-308233,308235,308237,308240,308242,308247,308250,308261,308270,308278,308286-308288,308295,308297-308298,308300-308301,308309,308312,308314,308339-308340,308343,308347-308348,308358,308375,308386,308406-308407,308409,308411-308414,308416,308420,308423-308425,308428,308430-308431,308434,308437,308440-308443,308456,308458-308459,308461-308462,308464,308471,308474,308476-308480,308487,308489,308493,308527-308530,308532,308534-308535,308538,308559,308564-308565,308567,308569-308570,308577,308579-308581,308583,308597-308603,308608,308611-308612,308614-308619,308637,308640,308642-308643,308659,308663-308665,308667-308669,308671,308673,308687-308692,308694,308696,308700,308704-308706,308708,308723-308725,308729-308730,308733,308737,308742-308743,308745,308772-308777,308779-308782,308786-308787,308789,308793-308795,308797-308799,308806,308808-308809,308813,308820-308821,308824,308826-308827,308832,308843,308848-308856,308869,308871,308887,308895,308898,308904-308909,308925-308926,308940-308945,308948,308950,308953,308957-308958,308969,308973,308980,308985,308995-308996,309006,309017,309026-309030,309039,309051,309060,309062-309063,309066-309069,309073,309078,309080-309083,309085,309090,309092-309093,309096-309099,309111-309113,309119-309121,309124,309126-309129,309131-309136,309138-309140,309142-309144,309146-309149,309151,309167,309169,309171,309174,309179,309184,309187,309189,309191-309192,309194-309195,309197-309201,309203,309209,309212,309216,309224-309245,309250-309252,309257,309260,309262,309264,309266-309270,309274,309281-309282,309284-309295,309297-309298,309300,309302,309307-309311,309316,309318-309321,309332,309345-309348,309353,309356-309357,309360-309361,309363-309365,309367,309369,309373-309374,309392-309394,309397,309400,309404-309413,309416,309425,309460-309461,309464,309472,309474,309477-309478,309491,309504,309513,309521,309523,309527-309528,309532,309534,309537-309539,309544,309547,309550-309552,309563,309566,309581-309582,309588-309591,309602,309607,309613,309624-309627,309630,309638-309640,309649-309650,309656-309657,309659-309660,309666,309669-309670,309676,309679,309682-309685,309688,309698-309700,309703-309705,309708-309712,309714,309716-309717,309719-309720,309722-309723,309725-309726,309728,309731-309734,309736-309738,309743-309744,309774-309780,309782-309784,309787,309795-309803,309805,309822-309823,309833,309835-309837,309839-309840,309851,309854,309856,309858,309869,309873-309875,309883,309886-309888,309893,309898,309901-309902,309904-309920,309922-309924,309926,309928-309930,309932,309934-309935,309937-309942,309944-309952,309957-310000,310012-310014,310021,310023,310025,310027-310028,310031,310033,310036,310038,310048-310050,310058,310083-310084,310086,310088-310090,310092,310095-310101,310104,310112,310114,310118,310124,310128,310132,310136-310137,310146,310150-310152,310155,310159-310160,310170-310171,310175,310177,310179-310182,310185-310187,310189,310193-310194,310196,310201-310203,310205,310225,310230,310232,310234,310236,310239,310242,310257-310259,310265-310268,310271-310275,310279,310284-310286,310298,310302,310304,310309,310312-310318,310324,310327,310329,310332,310336,310338-310341,310343-310345,310347-310348,310356,310360-310361,310365-310367,310373,310375,310387-310390,310417-310418,310420-310426,310430,310454-310455,310457-310459,310462,310465-310467,310478,310481,310488-310489,310491-310492,310496-310501,310503,310524,310534,310539,310547,310552-310555,310557,310559-310560,310574-310575,310583,310586-310593,310599,310608-310609,310611-310616,310621,310623,310627,310630,310633-310636,310638,310640,310642-310649,310651-310652,310654-310658,310660-310669,310674-310675,310677-310696,310698-310699,310702-310709,310711-310721,310724,310726-310729,310734,310741-310742,310744-310750,310752-310756,310758,310760,310762-310767,310770,310774,310778-310779,310782-310783,310785-310786,310791,310803,310805-310807,310810-310821,310823,310834,310840,310845-310846,310849-310850,310853-310854,310865-310866,310868,310870,310872-310874,310886-310889,310891-310892,310894,310907,310925,310931,310942,310950,310952,310954,310957-310958,310960-310961,310963,310972,310976-310977,310982-310989,310996,311000-311005,311012,311014,311055,311101-311103,311105,311108,311111-311115,311122,311131,311133,311135-311136,311140,311151,311157,311160,311168-311169,311171-311175,311194,311204,311210,311221-311222,311224-311229,311233,311235-311236,311239-311240,311242-311243,311245-311250,311263,311265,311268-311275,311282-311285,311287-311288,311290-311294,311298,311305,311311,311340,311346,311349,311352,311377-311378,311381-311382,311384,311390,311392-311394,311426,311445-311447,311452-311453,311458-311459,311461-311462,311469-311475,311504-311505,311511,311517,311520,311522-311527,311529-311531,311548,311556,311565,311568-311570,311572,311581,311584-311585,311601,311623,311636,311638-311640,311646-311647,311649-311651,311655,311657-311660,311664,311679-311680,311688,311693,311695,311699-311702,311706,311709-311715,311727,311733-311736,311739-311744,311748,311750,311754,311757-311762,311769,311780-311781,311787,311793-311794,311797,311804,311806-311807,311811,311815,311817,311830-311832,311846,311848,311850,311861,311870-311871,311873,311877,311879-311881,311886,311888,311890-311898,311901-311903,311907,311911-311912,311914,311917,311919,311923-311926,311928-311929,311932-311933,311935,311941-311942,311946-311947,311949-311951,311953-311954,311958,311961-311963,311968-311969,311971-311972,311977,311979,311981,311983-311985,311989,311993,311996,312001,312003,312008-312009,312012,312026,312063,312077-312081,312083,312086-312087,312102,312105,312108,312110-312114,312118-312122,312124,312153,312162,312164,312191,312199,312208-312209,312213-312214,312216,312226,312228,312230-312232,312237,312250-312251,312288,312291-312293,312305,312307,312314,312328-312332,312335,312338,312341,312343,312346-312348,312353-312355,312368-312369,312374,312387,312389-312390,312392,312395-312396,312404,312407-312412,312414,312417-312419,312422-312430,312432,312434-312435,312437-312438,312443,312446,312450-312514,312519-312521,312523,312526-312528,312531-312539,312544,312547,312551,312553,312555,312558-312560,312563,312593,312598-312606,312608,312610,312614,312617,312621-312622,312636-312637,312641,312644-312650,312659,312663-312664,312667,312669,312678-312679,312684-312686,312688-312690,312692,312694,312698-312699,312703,312721-312724,312728,312750,312759-312760,312764-312765,312767,312777,312779-312780,312782,312787,312791-312792,312814-312815,312825,312831-312832,312855,312857,312859,312865,312872,312875-312882,312886,312888-312890,312893,312897,312899,312901-312902,312904,312906,312913-312914,312919,312925,312932-312935,312937,312939,312942-312943,312947,312949,312952,312954,312973-312975,312977,312979-312983,312986-312988,312991-312995,313005-313009,313015-313018,313020-313021,313025,313027,313030-313032,313040-313042,313044-313045,313047-313048,313052,313069,313074,313080-313081,313083,313097,313104-313107,313109,313111,313113,313133,313141,313154,313160,313162-313163,313166,313169,313174,313176-313177,313180,313182,313184-313186,313188,313190-313191,313194,313249-313252,313254,313257-313261,313268-313285,313287,313306-313312,313314-313318,313322,313325,313327-313328,313330-313333,313335,313337,313341,313345-313346,313348-313352,313354,313356,313358-313360,313374,313376-313379,313386,313389-313392,313394-313398,313401,313404,313407,313409,313411-313421,313423,313425-313430,313436-313439,313442,313448-313449,313451,313453-313455,313457-313458,313467,313472,313477,313483,313493-313497,313547,313549,313554-313556,313559,313561-313564,313568,313572-313573,313575,313578,313645,313650,313652-313656,313659-313674,313679,313683-313684,313686-313688,313690,313692-313697,313707,313709-313710,313712-313713,313715,313727,313730-313731,313733-313740,313744-313746,313748-313749,313751-313752,313756-313757,313760-313761,313769-313772,313774-313775,313777-313779,313782,313784,313797-313800,313804-313810,313813,313817,313819-313821,313823,313841,313850-313855,313859,313863-313867,313869-313875,313877-313880,313893,313895,313897-313898,313901-313910,313912-313915,313917-313919,313922,313924-313925,313928,313933-313934,313936-313937,313939-313941,313944-313949,313953,313955,313958-313963,313965-313967,313972-313974,313978-313979,313982,313992-313996,313999,314001,314027,314035,314037-314040,314043-314046,314048,314050,314054-314056,314058-314062,314064-314071,314073-314082,314086-314088,314094,314097,314100-314101,314104-314106,314109,314112-314114,314118,314131-314132,314136,314139,314145,314148,314150-314155,314158,314167,314176,314179,314181,314183,314185,314187,314189-314196,314200,314203-314209,314211,314213-314215,314217-314219,314226,314228-314233,314239-314242,314244-314247,314252-314257,314267-314268,314271-314275,314280,314282,314287,314290-314295,314299-314300,314302,314306-314314,314316,314319-314321,314326,314328,314336-314339,314341-314345,314357-314358,314365,314370,314372,314374-314375,314382,314387,314396-314400,314402-314404,314409,314419-314420,314424,314429-314430,314435,314442,314450-314464,314466-314468,314470,314472,314474-314479,314483-314487,314489-314490,314496-314497,314502,314504-314505,314507,314509,314520,314527-314528,314533,314535,314541-314543,314545,314547-314549,314553-314558,314561-314562,314564,314571-314572,314576,314578-314579,314587,314592,314596,314598,314601,314604,314610-314611,314614-314621,314623,314625-314627,314636-314638,314643-314645,314647-314651,314653-314659,314666,314671,314675-314676,314686,314691,314693-314694,314700-314701,314705,314708-314709,314714-314718,314720,314723,314768-314774,314782-314783,314785-314786,314790,314793-314805,314807-314808,314810-314812,314814,314826,314828,314830-314831,314833,314837,314840-314843,314849-314853,314859-314861,314863-314872,314877-314878,314884-314896,314902-314908,314912-314914,314916,314918-314920,314924-314928,314933-314935,314942-314948,314950-314953,314956,314960,314962-314968,314970-314971,314973-314974,314987-314989,314993-314996,314998,315001-315002,315004-315006,315008-315010,315016,315018-315019,315022,315025,315030-315036,315038-315041,315046-315050,315053-315054,315056,315059,315067,315074-315078,315082-315089,315091,315095-315098,315102,315106,315108,315111-315114,315132-315133,315143,315146,315155-315163,315165,315167,315170,315176-315178,315186-315187,315192,315196,315199-315204,315206,315208,315211-315215,315225,315228-315230,315233-315238,315242,315244,315272-315273,315277-315281,315283,315286-315287,315289,315292,315294,315298,315303,315305,315307,315310,315318,315320,315323,315325-315327,315331,315334-315337,315360-315365,315368,315370,315402,315404-315408,315410,315412,315419-315420,315422-315423,315426-315427,315429-315431,315435,315442-315443,315449,315452-315453,315455,315457-315461,315466,315478,315482,315484-315485,315488-315490,315496-315510,315516,315518,315523-315526,315529-315530,315533-315534,315536,315545,315552,315560-315561,315564,315571-315580,315586-315589,315591,315595,315597-315598,315601,315603,315615,315619,315621,315636,315639-315642,315645-315648,315652-315654,315656,315670,315673-315674,315677-315678,315681-315683,315685-315692,315694,315697-315701,315708-315709,315711,315713-315716,315718-315720,315732,315734,315736,315738,315745,315749,315753-315754,315759-315762,315764-315766,315768,315770-315772,315774-315778,315780,315784,315788-315790,315793,315795-315799,315802-315804,315845,315850-315853,315856-315861,315863-315864,315869-315871,315876,315878,315892,315896,315900,315908,315910,315912-315913,315920-315924,315926,315932-315935,315947,315956-315957,315960-315961,315964,315968,315973-315974,315983,316005,316008-316009,316011-316013,316017,316020,316022-316023,316025,316029-316038,316049-316051,316053,316055-316057,316060-316062,316064-316066,316076-316082,316095,316099-316100,316102-316113,316115,316118-316119,316125,316131-316133,316171-316172,316174-316175,316179-316181,316183-316184,316211-316214,316258,316260,316275,316279-316280,316283-316289,316304,316309-316311,316313-316314,316328-316329,316341-316343,316355,316357-316358,316368,316374,316377,316393-316395,316397,316412-316413,316424,316426-316428,316430,316433-316437,316460-316461,316466-316468,316473,316477,316484-316485,316489-316493,316495,316497,316500-316501,316503-316508,316510-316512,316515,316519-316526,316528-316537,316541-316542,316544,316548-316554,316557-316558,316561-316566,316568,316571,316573,316577-316578,316580,316585-316587,316589-316590,316592-316593,316599-316603,316606-316612,316617,316625,316627-316635,316637-316638,316643-316644,316647-316649,316652-316659,316661,316664-316667,316669-316670,316676-316677,316679,316681-316682,316684-316687,316689-316692,316694-316695,316698-316699,316702,316704,316706,316715-316720,316731-316732,316734,316738-316740,316742-316743,316745-316747,316749-316750,316754-316756,316758-316761,316766,316768,316770-316771,316774,316776-316777,316780,316782,316792-316794,316802,316809-316818,316822-316824,316829,316851-316854,316856,316859,316866-316867,316869,316874,316936-316938,316940-316941,316945-316946,316949,316953,316956,316971-316972,316989,316991,316993-316997,317033-317034,317036,317041,317049-317052,317058,317060,317064,317075-317076,317079-317080,317092,317097,317099-317100,317107,317135,317137-317139,317141,317148,317154-317156,317159-317164,317168-317171,317173,317179-317180,317185-317186,317192-317193,317196-317197,317200-317201,317208,317212,317214-317216,317235-317239,317244,317246-317247,317253-317254,317265,317267,317269-317270,317272-317276,317279-317280,317283-317284,317288-317297,317299-317300,317304-317313,317315-317316,317323-317324,317340,317343-317346,317348-317350,317353,317356,317361,317369-317372,317376,317378,317382,317385,317388-317390,317409-317415,317428,317431,317435-317437,317444-317445,317457-317460,317463-317465,317483,317504-317505,317507,317511-317512,317517-317518,317522-317523,317527,317533,317541-317545,317547,317558,317560-317561,317566,317568,317576,317578,317583-317585,317591-317592,317594,317596-317597,317601,317606,317608,317610-317611,317632,317645,317648,317651-317652,317657-317660,317663,317665-317667,317678-317682,317694,317696,317700-317710,317712-317713,317715,317723,317729-317732,317736-317737,317739-317743,317745,317747-317748,317750-317752,317754-317755,317758-317759,317774-317775,317777,317779,317782-317786,317789-317790,317799,317801-317803,317806,317809-317821,317824,317828,317830,317835-317837,317839-317840,317842-317848,317854,317862-317863,317886-317888,317901,317904-317906,317908-317909,317912,317915,317923,317928,317931,317933-317935,317938,317981-317982,317996,317998,318001-318017,318021,318024-318026,318090-318092,318094,318098-318100,318107,318109,318116,318124-318126,318130,318135-318138,318141-318144,318147,318150,318160-318161,318163,318167,318175-318182,318185,318188-318191,318193-318194,318210-318219,318222-318224,318229-318232,318242-318243,318246,318251-318252,318254-318257,318259,318263,318276,318278,318280-318283,318287,318294,318298-318299,318302-318305,318307,318312-318318,318320,318325-318326,318352-318354,318356,318360,318364,318380,318382-318383,318386,318398-318401,318427,318431-318432,318434,318436-318437,318439,318444-318446,318448,318450,318476,318478-318479,318481,318511-318512,318514-318517,318520,318523,318527,318530-318531,318539,318545-318546,318565,318571,318574,318578,318580-318582,318584-318586,318588,318590-318593,318599,318601,318603,318606,318631-318633,318643,318646-318647,318649,318651,318655,318676-318679,318688-318689,318693-318695,318701-318712,318715,318717-318723,318734,318738-318739,318745,318748-318749,318754-318755,318762,318765,318774,318780-318781,318788,318790,318794-318795,318812,318814,318816,318818-318824,318827-318834,318860-318861,318863,318867,318872,318879,318884,318891,318894-318896,318899-318900,318906-318908,318914,318916-318917,318920-318926,318928,318930,318932,318935,318943,318945,318952-318955,318960,318962,318966,318968,318970,318973-318975,318982,318986-318996,318998-318999,319001-319002,319008,319010,319015,319019-319020,319022,319026-319031,319033-319049,319051-319052,319054,319056-319063,319071-319078,319083-319086,319096,319118,319125,319132,319137-319138,319156-319159,319162,319167,319197-319201,319212,319219,319229,319246,319293-319295,319312,319316-319321,319336-319341,319350,319355,319365,319369,319409-319410,319412-319414,319432,319444,319454-319456,319473,319475,319481,319487-319488,319490-319491,319493,319499-319501,319506,319509,319513,319516-319520,319539-319540,319542-319546,319548,319551,319556-319558,319561-319562,319571,319578-319579,319581-319582,319584,319586,319588-319590,319595,319602-319605,319608-319610,319612,319620-319621,319626-319627,319634,319637,319642,319655-319656,319659-319660,319662,319670-319672,319675-319682,319688-319690,319694,319699,319702,319709,319714,319717-319718,319731,319746-319751,319755-319758,319761-319769,319774-319775,319793,319796,319798,319800,319803,319806,319811,319813-319815,319817-319818,319824-319825,319827,319829-319830,319834,319836,319841-319846,319850-319852,319854-319859,319861-319864,319866-319867,319869,319871-319876,319882,319885,319888,319890,319895,319899-319900,319905,319916,319918,319928,319932-319934,319942,319947,319949,319953-319957,319962,319964,319968-319970,319972,319975,319987,319995-319996,320007-320008,320010-320012,320028,320030,320034-320035,320038,320048-320049,320051-320052,320056,320061-320063,320065,320069-320070,320072,320076-320079,320093,320108,320118,320121,320123-320126,320135,320145-320146,320151-320153,320156,320163,320165-320167,320170,320172-320176,320181,320183-320186,320189,320192-320196,320201-320202,320204,320206-320208,320216,320234-320235,320237-320240,320242,320252,320255-320256,320259,320262-320264,320266-320267,320269-320271,320273,320277,320288,320292,320296,320300-320301,320304,320307-320308,320313-320314,320316-320319,320322-320323,320332-320339,320343-320345,320352-320353,320359-320360,320363-320364,320367,320372,320387,320390,320392,320403,320406,320408-320409,320412,320417,320420-320425,320430,320433,320441-320446,320451-320452,320456,320458,320460-320463,320467-320468,320472,320478-320479,320481-320483,320488-320495,320497-320498,320501,320508-320509,320517,320521-320522,320527,320546-320547,320553,320555,320560-320561,320570,320574-320578,320580,320595,320599,320604,320619-320620,320624-320636,320644,320655-320656,320658-320659,320663,320665,320670-320672,320675,320678,320683,320686,320688,320690,320694,320696,320701-320703,320705,320714,320723-320724,320729-320730,320733-320734,320736-320737,320740-320743,320748,320750,320754-320755,320761-320762,320773-320775,320785,320796,320801,320803-320804,320806-320807,320810-320820,320836-320837,320842-320843,320851-320854,320865,320868,320876,320883,320892-320893,320895-320896,320900-320902,320906,320914,320916,320918,320927-320932,320936-320938,320941,320943-320944,320956,320969,320974-320975,320977,320980,320982,320984,320986,320988-320991,320996-320997,320999,321001-321003,321008,321011,321015,321023,321035-321036,321045-321046,321048,321063,321072,321075,321080-321082,321102-321106,321109,321111,321128,321138,321173,321179,321196,321203,321206-321207,321214,321217-321218,321226,321228,321230,321233,321235,321240,321247-321248,321256,321261,321284,321286,321293-321294,321298,321302,321305-321306,321314,321327-321330,321332,321342,321347-321349,321356,321366,321368-321371,321376-321378,321385-321387,321390,321393,321397-321401,321406-321410,321420,321422-321423,321426,321433,321435-321437,321439,321444,321446,321450,321455-321457,321460-321461,321471,321483-321484,321486,321489-321490,321502,321508-321512,321514,321580-321584,321586,321588-321589,321601,321603,321605-321608,321620-321623,321625,321627-321628,321639-321640,321652,321664,321668-321671,321674-321675,321684-321686,321688-321689,321702-321706,321708-321714,321719-321723,321726-321727,321730,321733-321734,321737,321743-321746,321751,321762,321772-321773,321779-321780,321782,321791-321792,321794-321795,321798-321799,321803,321805-321806,321810,321812,321816,321821,321823,321826,321828,321836-321847,321849,321851-321852,321856,321858-321859,321864,321876,321884,321896,321899,321915,321919,321921-321922,321924,321926,321933-321934,321938,321947-321952,321954,321959,321963,321965,321969,321983-321986,321991,322014-322016,322019,322023,322025-322026,322028,322034,322036-322041,322050,322052,322055-322056,322059,322073-322074,322076-322077,322097,322101,322112-322113,322123-322124,322139,322167,322169-322171,322175,322177-322179,322200-322201,322203,322209-322210,322212-322214,322218,322222,322226,322228,322230,322233-322234,322237,322239,322241,322245,322248,322250-322251,322255-322258,322270-322272,322280,322282,322288,322291,322293,322296-322297,322299,322302,322304-322306,322308-322310,322316,322320-322323,322326,322328-322329,322331,322336,322351,322354-322357,322360,322364,322368,322371,322373-322374,322380,322383,322386,322391-322392,322397,322401,322404-322406,322408,322410-322411,322418,322424-322428,322431,322434-322437,322441,322443,322445,322447-322451,322456,322459,322463-322465,322473-322479,322483,322485-322488,322493-322496,322527,322530,322542,322544,322546-322550,322567,322571,322580-322581,322590,322592-322593,322596,322613,322618,322627-322628,322633,322636,322648,322657,322667-322669,322672,322677-322678,322680,322682-322684,322694-322695,322706,322710,322713-322714,322718-322723,322726,322740,322746,322749-322752,322756-322757,322762-322763,322770-322771,322773-322775,322795-322800,322802-322804,322810,322812,322816,322821,322825-322826,322830-322833,322852,322854-322855,322868,322872,322874-322875,322883,322885,322888,322894,322896-322897,322899,322901-322903,322913-322915,322921,322923,322925-322941,322947-322948,322951,322959-322960,322962,322964-322965,322970-322971,322980-322982,322984-322987,322990,322992,322994-322995,322997-322999,323001-323004,323006,323010-323011,323014-323018,323020-323022,323024-323026,323032,323039-323041,323045-323047,323051,323053-323054,323058-323059,323062-323065,323067-323069,323074,323076,323078,323084,323086,323100,323102,323112,323114,323126-323128,323131-323133,323151,323155,323160,323166,323170,323174-323178,323183-323185,323192-323197,323217,323220-323221,323224-323226,323228-323230,323234-323236,323245,323252-323253,323258,323261,323263,323272,323275,323278,323280,323286,323290,323305,323307-323308,323310,323317,323324-323327,323341,323343,323347,323349-323351,323355,323364,323367-323368,323379,323382,323385-323392,323394,323405,323407,323428,323433,323435-323436,323438-323439,323448,323450,323465,323474,323478-323483,323491,323494,323496-323499,323508-323509,323513-323514,323520,323522,323524-323525,323528,323539-323541,323544,323547,323552-323554,323557-323559,323561-323562,323564,323572-323575,323578-323585,323587-323589,323597-323598,323611-323613,323616,323625,323630-323631,323642,323645,323654-323656,323671,323675-323676,323683,323689,323691-323692,323703-323705,323707,323709-323710,323714-323715,323722,323727-323729,323768-323769,323772,323781-323782,323785-323786,323791,323793,323796-323797,323812-323813,323822,323824,323834,323836,323839-323840,323851-323854,323856,323858-323860,323864,323867-323868,323873,323880,323882,323885-323886,323889,323895-323897,323905-323910,323915-323916,323918,323924,323933-323934,323945,323952,323961-323962,323973,323978,323981,323985,323991,323993-323995,323997-323998,324007,324011-324012,324014,324016,324026,324039,324041,324044-324045,324048-324050,324065-324066,324072-324075,324077-324081,324086-324088,324098-324100,324107-324108,324112-324114,324123,324125,324127,324143,324146,324148,324155-324156,324163-324164,324168-324170,324172-324174,324178-324179,324185,324193-324197,324199-324202,324206,324212,324220-324224,324226,324228,324239-324241,324243,324248-324249,324257-324258,324260-324261,324276-324278,324281,324285-324286,324292,324296,324301-324316,324320-324321,324326-324328,324330,324334-324335,324343-324350,324354-324355,324359-324360,324362,324364,324366-324367,324369,324373,324378-324379,324386,324388,324413,324415,324422-324425,324427,324429-324439,324443,324445,324449-324454,324456-324457,324460,324469-324471,324476,324478,324487-324492,324497,324501-324502,324506,324509,324516-324517,324528,324535-324536,324538,324547,324552-324559,324566,324568,324590,324592-324595,324597,324600,324606,324609-324610,324613,324621-324623,324628-324637,324639,324644,324646-324654,324659-324661,324665,324668-324672,324681,324683,324688-324689,324694,324696,324700-324704,324709,324716-324717,324719,324721,324727,324738,324752,324757,324770,324772,324774,324778,324780,324787,324789,324792-324795,324803-324805,324807,324809,324814,324817,324821,324823-324824,324826,324836,324841-324845,324850,324853,324857,324862-324865,324867-324869,324876-324883,324920,324923,324926,324928-324929,324940-324941,324945,324947,324956-324957,324972,324977-324982,324991-324996,325007-325011,325014,325016-325018,325028,325030,325035,325039,325042,325044-325045,325050-325051,325054-325056,325059-325061,325063,325065-325067,325093-325095,325101,325103,325108,325114-325116,325122-325125,325156-325158,325170-325176,325217,325227-325228,325232,325248,325266,325268-325279,325285-325286,325292,325294-325299,325310,325319,325321,325328,325332,325338-325339,325353,325355,325359-325364,325371,325373,325376-325377,325379-325380,325382-325383,325386-325389,325397,325410,325420,325433,325436,325438,325441,325444,325447,325450-325451,325456,325458,325469,325478-325485,325515,325517,325528,325530,325532-325533,325552-325556,325558-325559,325561,325566-325567,325570-325571,325588,325596,325606,325608,325610,325614-325616,325627,325635,325637-325638,325641,325648-325658,325662,325670-325671,325680-325681,325683,325685-325694,325706-325708,325716-325718,325723-325725,325737,325743-325744,325748,325755,325758-325759,325764,325766-325767,325769,325775-325776,325779-325780,325795,325800,325805-325807,325811-325813,325817,325827,325834-325835,325841,325850-325851,325857,325861,325863,325865,325880,325883-325884,325887-325888,325897,325917-325922,325924-325925,325946-325947,325950,325953,325959-325963,325976,325997,326019-326021,326026,326028,326030,326032,326034,326036,326038-326042,326046,326052,326055,326058,326060-326061,326063,326065,326067-326070,326072,326074,326082,326084,326086,326089-326090,326093-326094,326096,326098,326100-326101,326106-326107,326110-326112,326115-326118,326122,326132,326134-326137,326140,326143-326144,326150,326161,326173,326175-326178,326182,326184-326187,326194-326200,326228-326229,326234-326235,326237,326244,326273,326276,326281,326284,326286,326288-326290,326307,326309,326311,326315,326330-326331,326343,326361-326362,326371,326378,326383-326384,326386,326391-326392,326399,326401,326407,326409-326410,326412,326420-326422,326424,326429,326434,326438,326440-326448,326455-326456,326473,326479-326480,326483-326496,326502,326504,326507,326509-326510,326518,326522,326526-326527,326544,326547,326558,326560-326562,326565-326567,326572,326584-326594,326597,326600,326609-326610,326613,326615-326616,326624,326628-326629,326633,326640,326646,326654-326655,326657,326662,326666,326668-326671,326698,326707-326712,326714,326720,326724,326731-326732,326738,326748,326750,326752-326754,326768-326769,326772,326774,326776,326784,326791-326792,326794,326796-326799,326811-326813,326831,326834-326835,326847,326851,326853-326858,326861-326863,326866,326868-326869,326872-326888,326897-326898,326909,326912,326914,326919,326924-326927,326935,326937,326940,326953-326954,326957,326960-326963,326971,326973,326977,326982-326983,326986,326988-326990,326993-326995,327004,327029,327032,327034,327047-327050,327053,327062,327074,327088,327093-327094,327096-327097,327099,327117-327118,327140,327164,327167-327168,327183,327211,327213,327220-327222,327226,327231-327232,327235-327236,327264,327282-327286,327289,327295,327298-327299,327316,327319,327329,327332-327334,327336,327342,327346-327347,327350-327354,327356,327359,327361,327367,327375-327377,327385-327386,327390,327393-327399,327401-327402,327416,327420,327435,327437-327439,327444,327448-327451,327453-327454,327468-327469,327472,327489,327496-327498,327502-327503,327510,327517,327523-327525,327528,327540,327555,327560-327561,327567,327578,327593,327597,327606,327612,327625,327662,327666,327674-327676,327682-327683,327685,327697-327700,327703-327705,327718,327721-327726,327730,327737,327751-327753,327755,327760,327768,327770,327775,327779-327780,327788,327792,327796,327803,327816-327822,327826,327834,327838,327845,327862,327874-327875,327878-327881,327888,327905,327911-327914,327917,327925,327930,327946-327947,327951,327963-327965,327972-327973,327991,327996,328007-328009,328013,328027,328029-328032,328035,328052,328055,328060-328061,328082-328083,328085,328096,328106-328108,328116,328119-328120,328126,328128,328134-328135,328153,328156-328157,328160-328161,328163,328166,328169,328171,328173,328177,328188-328192,328199-328200,328202,328205-328206,328208-328209,328212,328217,328221-328222,328224,328226,328228,328230,328232,328234,328237,328243,328246,328248,328250,328252,328254,328256,328264,328266,328273,328283-328284,328287-328293,328296-328297,328304-328306,328314-328315,328318,328326,328329,328332,328344,328350,328352,328382,328398-328399,328411-328412,328419-328420,328423,328436-328439,328441,328444,328446-328450,328468,328470,328491,328497,328504-328505,328514,328536,328540-328541,328552,328567,328584,328591,328603,328605,328608,328610-328615,328622-328628,328630,328641-328643,328647,328659,328694-328695,328720,328731,328769-328770,328773-328774,328776,328779-328784,328795,328799,328806,328808,328826,328830,328835,328853,328864-328865,328876,328880-328881,328895,328913,328917,328937-328938,328971,328986-328987,328990,328995-328996,328999-329000,329009,329016,329019,329022,329024,329050,329053-329054,329060,329091-329092,329101,329181,329195,329197-329198,329214-329216,329252,329254,329256,329264,329271,329295,329297,329311,329314,329347,329361-329362,329364-329365,329371-329372,329374,329376-329378,329416,329447,329464-329468,329470-329472,329475-329477,329509-329516,329519,329521,329523-329525,329555-329556,329561,329563,329584,329682,329703,329711,329749,329820,329825,329843,329846,329848,329864,329880,329905,329930,329994,330004,330033,330104,330127,330236,330256,330364 Merged /projects/netbsd-tests-update-12/sys/contrib/rdma/krping:r303985-305029,305031-305318 Merged /releng/11.1/sys/contrib/rdma/krping:r320566 Merged /projects/clang400-import/sys/contrib/rdma/krping:r311132-314524 Merged /projects/clang500-import/sys/contrib/rdma/krping:r316992-321352 Merged /projects/contrib-netbsd-update-12/sys/contrib/rdma/krping:r303899-303984 Merged /stable/11/sys/contrib/rdma/krping:r304545 Index: projects/bsd_rdma_4_9_stable_11/sys/modules/rdma/krping_compat/Makefile =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/modules/rdma/krping_compat/Makefile (nonexistent) +++ projects/bsd_rdma_4_9_stable_11/sys/modules/rdma/krping_compat/Makefile (revision 330503) @@ -0,0 +1,12 @@ +# $FreeBSD$ +.PATH: ${SRCTOP}/sys/contrib/rdma/krping_compat + +KMOD= krping_compat +SRCS= krping.c krping_dev.c getopt.c +SRCS+= bus_if.h device_if.h pci_if.h pcib_if.h vnode_if.h +SRCS+= opt_sched.h opt_inet.h opt_inet6.h +CFLAGS+= -I${SRCTOP}/sys/ofed/include +CFLAGS+= -I${SRCTOP}/sys/ofed/include/uapi +CFLAGS+= -I${SRCTOP}/sys/compat/linuxkpi/common/include + +.include Property changes on: projects/bsd_rdma_4_9_stable_11/sys/modules/rdma/krping_compat/Makefile ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: projects/bsd_rdma_4_9_stable_11/sys/modules/rdma/krping_compat =================================================================== --- projects/bsd_rdma_4_9_stable_11/sys/modules/rdma/krping_compat (nonexistent) +++ projects/bsd_rdma_4_9_stable_11/sys/modules/rdma/krping_compat (revision 330503) Property changes on: projects/bsd_rdma_4_9_stable_11/sys/modules/rdma/krping_compat ___________________________________________________________________ Added: svn:mergeinfo ## -0,0 +0,10 ## Merged /projects/clang400-import/sys/modules/rdma/krping:r311132-314524 Merged /projects/clang500-import/sys/modules/rdma/krping:r316992-321352 Merged /projects/contrib-netbsd-update-12/sys/modules/rdma/krping:r303899-303984 Merged /projects/netbsd-tests-update-12/sys/modules/rdma/krping:r303985-305029,305031-305318 Merged /stable/11/sys/modules/rdma/krping:r304545 Merged /projects/netbsd-tests-upstream-01-2017/sys/modules/rdma/krping:r312125-313435 Merged /releng/11.1/sys/modules/rdma/krping:r320566 Merged /projects/clang390-import/sys/modules/rdma/krping:r303250-309123 Merged /projects/clang391-import/sys/modules/rdma/krping:r309166-310192 Merged /head/sys/modules/rdma/krping:r302438-302440,302448-302450,302456-302457,302459-302460,302465,302470-302476,302480-302484,302486-302497,302499-302500,302502-302508,302510-302511,302514-302524,302528-302532,302534,302540-302554,302556-302561,302565,302567,302571-302573,302576-302577,302580-302581,302584-302587,302591-302592,302605-302612,302614,302617-302624,302626,302629-302638,302664,302666-302668,302673-302674,302692-302704,302706-302710,302713,302723,302726,302731,302733,302737,302770-302772,302778,302783-302785,302787-302790,302793-302795,302797-302798,302801-302806,302808-302819,302823-302824,302826-302827,302831,302834-302843,302845,302847-302853,302855-302861,302864,302867-302876,302878-302894,302896-302897,302899-302900,302902,302904,302906-302908,302911,302916-302917,302919-302921,302928-302930,302932-302933,302935-302937,302940-302947,302949-302952,302957,302960-302961,302965-302966,302972-302973,302980-302981,302986,302988,302999-303004,303009-303014,303016-303026,303028,303030-303034,303039-303040,303042-303046,303052,303059,303061,303066-303074,303076,303084,303086-303088,303090-303091,303094,303098,303101,303103-303104,303106,303109,303111,303113,303115,303123,303125-303129,303131-303132,303134,303142,303144-303145,303147,303151,303153-303157,303160,303164,303166,303171,303178,303180,303182,303186,303188,303190,303195,303199,303202,303204-303205,303208,303210-303213,303217,303219,303223,303225-303227,303230,303243-303244,303253,303261,303263-303266,303271-303274,303277,303279,303282-303285,303287-303289,303299,303307-303309,303315,303322,303326-303329,303335,303338-303339,303343,303345-303354,303356,303361-303362,303366-303370,303373-303374,303379,303382-303388,303393,303396,303399-303400,303404-303406,303410,303413-303426,303429,303437-303438,303442,303444,303446,303448,303450,303453-303454,303456-303457,303461,303464-303468,303470-303478,303486,303488,303490-303492,303494,303497,303501,303503,303510,303514-303516,303518-303522,303530,303532,303536,303541-303543,303545-303548,303551,303553-303557,303559,303561-303569,303573,303576,303581,303583-303585,303587,303592-303594,303603-303605,303610-303613,303615,303626-303627,303630-303631,303637-303638,303643,303645,303647,303650,303652,303655,303657,303661,303663,303670,303674,303676-303677,303685,303687-303688,303693,303702-303707,303710,303712-303713,303715-303716,303719-303723,303725-303729,303737-303739,303744,303747,303750,303753-303755,303760,303763-303766,303771,303773,303782-303786,303788,303791-303796,303798-303800,303802,303804,303806,303813,303816,303818-303819,303822-303824,303830-303834,303836-303837,303841-303842,303845,303847,303855,303859-303860,303863,303867,303869-303870,303874,303877,303880-303881,303886-303887,303889,303891-303894,303897,303900-303901,303903-303904,303913-303914,303916,303919,303921-303924,303927-303937,303941-303949,303955,303958,303961-303962,303964,303971,303974,303978,303982,303989-303992,303998,304000-304002,304004-304009,304011-304012,304016,304018,304021,304023,304026,304033-304034,304040-304041,304043,304045-304050,304052-304055,304058-304059,304061,304066,304068,304071,304075-304077,304085,304087,304089,304102,304106,304108-304109,304111,304140,304142,304146,304149,304151-304152,304154,304160,304162,304168-304170,304174,304176,304178,304180,304182,304184-304185,304191-304192,304194,304202,304204-304206,304209,304217,304221-304222,304225-304232,304238-304239,304246-304249,304251-304256,304270-304271,304273,304279,304285-304292,304295,304297,304313,304315-304322,304327,304329-304331,304342,304374,304378,304430-304431,304438,304440-304441,304443-304444,304446-304447,304459-304461,304464,304476,304478-304479,304481-304485,304487,304492,304509,304520-304521,304530,304532,304535,304543,304545,304554-304559,304561,304563-304566,304570-304576,304579,304584,304591,304593-304601,304604-304605,304607-304608,304615,304620,304624-304626,304629,304635-304638,304640-304641,304643,304647,304649,304652,304654,304674,304676-304677,304680,304685,304687-304689,304691,304693-304694,304696-304698,304703,304710,304713,304721-304724,304728,304730,304736,304740,304742-304751,304753-304755,304779-304786,304788,304790-304791,304793,304795-304797,304799,304801-304802,304806-304812,304819,304821,304825-304826,304832-304834,304837-304838,304854-304855,304858,304873-304875,304877,304880,304886,304891-304892,304911,304916,304918,304920,304927-304928,304931,304953,304957-304959,304962,304964,304966,304969-304970,304972-304973,304975-304976,304979,304989,304991,305009,305012-305013,305015-305016,305018-305020,305022-305024,305026,305032-305034,305036,305039-305041,305044-305050,305052-305056,305058,305061,305065-305066,305071,305077-305078,305085-305086,305090,305093,305095,305104,305107-305116,305119,305122-305123,305125,305128-305130,305132-305133,305137,305140-305141,305144-305145,305147-305149,305154-305155,305160,305166-305167,305169-305171,305173-305179,305182,305191,305193-305195,305197-305203,305205-305207,305209-305213,305215-305216,305218-305219,305221-305222,305224,305241,305244,305248,305253-305258,305264-305265,305268-305270,305276-305281,305283-305285,305303-305304,305312,305319-305320,305323-305328,305331-305334,305338-305340,305342-305345,305349,305353,305355-305358,305360,305363,305365,305367,305378-305386,305388-305389,305393,305396,305398,305405-305414,305421-305422,305425,305430,305432-305433,305449,305451,305453-305456,305467-305471,305482-305486,305492,305497,305502,305504,305507-305509,305521,305524-305527,305535-305536,305539,305545-305546,305548-305549,305561,305563,305575,305578-305581,305585-305588,305590-305610,305612-305613,305620,305624,305626,305628-305629,305633,305652,305659,305661,305663,305665,305667,305671,305676-305677,305679,305684-305685,305689,305692,305695-305696,305699,305701-305704,305706-305708,305713,305715,305719,305722,305724-305725,305727-305730,305737,305739-305740,305744,305749,305751-305752,305756,305760-305761,305763,305767,305769-305772,305778,305788-305796,305801,305804,305807,305810-305814,305819,305822,305827,305834,305836,305840-305841,305851-305852,305854-305856,305862-305865,305867-305877,305887,305894-305898,305902-305903,305906,305908,305916-305928,305937-305943,305947,305951,305954,305956,305960-305970,305977-305978,305985,305987,305994-305995,305999-306009,306011-306015,306020,306026,306029-306031,306033,306036,306038,306041,306046,306048,306050,306061-306063,306071-306076,306081-306082,306087-306092,306094-306097,306102,306104,306109,306126,306129,306131-306132,306134,306137-306139,306142,306159-306162,306174,306184-306185,306188,306193,306205-306206,306209,306212,306216,306218,306220,306224,306226,306228,306232,306257,306260-306262,306264,306266-306267,306273-306277,306279,306281,306285-306286,306289-306292,306295,306297,306299-306301,306304,306306,306310-306313,306318,306320,306325,306329-306330,306332-306334,306342,306346,306349-306351,306355-306357,306359-306360,306366,306375-306377,306380,306387-306393,306395-306396,306409-306410,306417,306424-306426,306430-306433,306435,306441-306445,306447,306449,306451,306453-306454,306456-306460,306464-306465,306474,306477-306486,306488,306494-306496,306498,306504-306508,306510-306511,306513,306522,306526-306532,306534,306538,306541,306544,306550-306552,306554-306555,306557,306559-306571,306577-306579,306581-306585,306588-306589,306597-306608,306610-306622,306631,306634,306638,306640-306641,306647,306650,306652,306654,306656,306658,306665-306668,306670,306674,306677,306680,306686,306691,306695-306696,306699,306702-306704,306706,306709-306710,306712,306714-306715,306717-306725,306727-306728,306731-306736,306740,306742-306744,306746-306747,306751,306754-306756,306759,306762,306766-306767,306771,306773,306775,306782-306783,306787,306790,306792-306795,306801,306803,306806-306808,306812,306821,306823,306825-306826,306828-306830,306836-306837,306840,306852-306855,306857,306860,306877,306896-306902,306936-306939,306944,306962,306966,306999-307003,307008,307012-307013,307036-307038,307041-307042,307044,307059-307060,307064,307066-307068,307070-307072,307074,307078,307087-307089,307091-307093,307095,307098,307115,307130-307132,307141,307145,307150-307152,307154,307156,307158-307159,307182,307189-307192,307195-307196,307204-307205,307215-307218,307220,307224,307227,307233,307236,307238-307241,307261-307263,307316,307318,307322-307324,307326-307327,307332-307334,307338-307339,307343,307348,307350,307364,307366,307368-307369,307372,307374,307379-307381,307384,307390-307392,307397-307398,307400-307402,307469,307499,307501,307507,307509,307515,307517-307523,307531,307538,307541,307544-307545,307550-307551,307556-307558,307561,307564,307566-307567,307570,307578,307584,307598,307602,307624,307626,307628-307629,307637-307638,307649-307651,307653,307657,307684-307688,307691-307694,307698,307700-307702,307710-307714,307726-307727,307731,307741,307745-307747,307752,307755-307756,307759,307768-307769,307774,307779,307783,307785-307787,307800-307802,307804-307805,307808,307821,307838-307840,307842-307845,307857,307861,307863,307865-307866,307869,307873,307876,307879-307880,307890-307891,307893,307901-307904,307908,307911,307917,307936-307937,307942,307948,307950-307955,307966,307969,307975-307978,307983,307985-307991,307994,308004-308008,308010-308031,308038-308040,308049-308051,308055-308056,308064-308066,308088-308089,308094-308099,308101,308104,308106,308108-308109,308112-308114,308116-308120,308124-308126,308129,308133-308134,308139,308142,308144,308148,308150,308156-308158,308160-308169,308171-308176,308180,308183,308185-308189,308194-308195,308197,308201,308206,308209-308212,308216-308221,308225,308228,308231-308233,308235,308237,308240,308242,308247,308250,308261,308270,308278,308286-308288,308295,308297-308298,308300-308301,308309,308312,308314,308339-308340,308343,308347-308348,308358,308375,308386,308406-308407,308409,308411-308414,308416,308420,308423-308425,308428,308430-308431,308434,308437,308440-308443,308456,308458-308459,308461-308462,308464,308471,308474,308476-308480,308487,308489,308493,308527-308530,308532,308534-308535,308538,308559,308564-308565,308567,308569-308570,308577,308579-308581,308583,308597-308603,308608,308611-308612,308614-308619,308637,308640,308642-308643,308659,308663-308665,308667-308669,308671,308673,308687-308692,308694,308696,308700,308704-308706,308708,308723-308725,308729-308730,308733,308737,308742-308743,308745,308772-308777,308779-308782,308786-308787,308789,308793-308795,308797-308799,308806,308808-308809,308813,308820-308821,308824,308826-308827,308832,308843,308848-308856,308869,308871,308887,308895,308898,308904-308909,308925-308926,308940-308945,308948,308950,308953,308957-308958,308969,308973,308980,308985,308995-308996,309006,309017,309026-309030,309039,309051,309060,309062-309063,309066-309069,309073,309078,309080-309083,309085,309090,309092-309093,309096-309099,309111-309113,309119-309121,309124,309126-309129,309131-309136,309138-309140,309142-309144,309146-309149,309151,309167,309169,309171,309174,309179,309184,309187,309189,309191-309192,309194-309195,309197-309201,309203,309209,309212,309216,309224-309245,309250-309252,309257,309260,309262,309264,309266-309270,309274,309281-309282,309284-309295,309297-309298,309300,309302,309307-309311,309316,309318-309321,309332,309345-309348,309353,309356-309357,309360-309361,309363-309365,309367,309369,309373-309374,309392-309394,309397,309400,309404-309413,309416,309425,309460-309461,309464,309472,309474,309477-309478,309491,309504,309513,309521,309523,309527-309528,309532,309534,309537-309539,309544,309547,309550-309552,309563,309566,309581-309582,309588-309591,309602,309607,309613,309624-309627,309630,309638-309640,309649-309650,309656-309657,309659-309660,309666,309669-309670,309676,309679,309682-309685,309688,309698-309700,309703-309705,309708-309712,309714,309716-309717,309719-309720,309722-309723,309725-309726,309728,309731-309734,309736-309738,309743-309744,309774-309780,309782-309784,309787,309795-309803,309805,309822-309823,309833,309835-309837,309839-309840,309851,309854,309856,309858,309869,309873-309875,309883,309886-309888,309893,309898,309901-309902,309904-309920,309922-309924,309926,309928-309930,309932,309934-309935,309937-309942,309944-309952,309957-310000,310012-310014,310021,310023,310025,310027-310028,310031,310033,310036,310038,310048-310050,310058,310083-310084,310086,310088-310090,310092,310095-310101,310104,310112,310114,310118,310124,310128,310132,310136-310137,310146,310150-310152,310155,310159-310160,310170-310171,310175,310177,310179-310182,310185-310187,310189,310193-310194,310196,310201-310203,310205,310225,310230,310232,310234,310236,310239,310242,310257-310259,310265-310268,310271-310275,310279,310284-310286,310298,310302,310304,310309,310312-310318,310324,310327,310329,310332,310336,310338-310341,310343-310345,310347-310348,310356,310360-310361,310365-310367,310373,310375,310387-310390,310417-310418,310420-310426,310430,310454-310455,310457-310459,310462,310465-310467,310478,310481,310488-310489,310491-310492,310496-310501,310503,310524,310534,310539,310547,310552-310555,310557,310559-310560,310574-310575,310583,310586-310593,310599,310608-310609,310611-310616,310621,310623,310627,310630,310633-310636,310638,310640,310642-310649,310651-310652,310654-310658,310660-310669,310674-310675,310677-310696,310698-310699,310702-310709,310711-310721,310724,310726-310729,310734,310741-310742,310744-310750,310752-310756,310758,310760,310762-310767,310770,310774,310778-310779,310782-310783,310785-310786,310791,310803,310805-310807,310810-310821,310823,310834,310840,310845-310846,310849-310850,310853-310854,310865-310866,310868,310870,310872-310874,310886-310889,310891-310892,310894,310907,310925,310931,310942,310950,310952,310954,310957-310958,310960-310961,310963,310972,310976-310977,310982-310989,310996,311000-311005,311012,311014,311055,311101-311103,311105,311108,311111-311115,311122,311131,311133,311135-311136,311140,311151,311157,311160,311168-311169,311171-311175,311194,311204,311210,311221-311222,311224-311229,311233,311235-311236,311239-311240,311242-311243,311245-311250,311263,311265,311268-311275,311282-311285,311287-311288,311290-311294,311298,311305,311311,311340,311346,311349,311352,311377-311378,311381-311382,311384,311390,311392-311394,311426,311445-311447,311452-311453,311458-311459,311461-311462,311469-311475,311504-311505,311511,311517,311520,311522-311527,311529-311531,311548,311556,311565,311568-311570,311572,311581,311584-311585,311601,311623,311636,311638-311640,311646-311647,311649-311651,311655,311657-311660,311664,311679-311680,311688,311693,311695,311699-311702,311706,311709-311715,311727,311733-311736,311739-311744,311748,311750,311754,311757-311762,311769,311780-311781,311787,311793-311794,311797,311804,311806-311807,311811,311815,311817,311830-311832,311846,311848,311850,311861,311870-311871,311873,311877,311879-311881,311886,311888,311890-311898,311901-311903,311907,311911-311912,311914,311917,311919,311923-311926,311928-311929,311932-311933,311935,311941-311942,311946-311947,311949-311951,311953-311954,311958,311961-311963,311968-311969,311971-311972,311977,311979,311981,311983-311985,311989,311993,311996,312001,312003,312008-312009,312012,312026,312063,312077-312081,312083,312086-312087,312102,312105,312108,312110-312114,312118-312122,312124,312153,312162,312164,312191,312199,312208-312209,312213-312214,312216,312226,312228,312230-312232,312237,312250-312251,312288,312291-312293,312305,312307,312314,312328-312332,312335,312338,312341,312343,312346-312348,312353-312355,312368-312369,312374,312387,312389-312390,312392,312395-312396,312404,312407-312412,312414,312417-312419,312422-312430,312432,312434-312435,312437-312438,312443,312446,312450-312514,312519-312521,312523,312526-312528,312531-312539,312544,312547,312551,312553,312555,312558-312560,312563,312593,312598-312606,312608,312610,312614,312617,312621-312622,312636-312637,312641,312644-312650,312659,312663-312664,312667,312669,312678-312679,312684-312686,312688-312690,312692,312694,312698-312699,312703,312721-312724,312728,312750,312759-312760,312764-312765,312767,312777,312779-312780,312782,312787,312791-312792,312814-312815,312825,312831-312832,312855,312857,312859,312865,312872,312875-312882,312886,312888-312890,312893,312897,312899,312901-312902,312904,312906,312913-312914,312919,312925,312932-312935,312937,312939,312942-312943,312947,312949,312952,312954,312973-312975,312977,312979-312983,312986-312988,312991-312995,313005-313009,313015-313018,313020-313021,313025,313027,313030-313032,313040-313042,313044-313045,313047-313048,313052,313069,313074,313080-313081,313083,313097,313104-313107,313109,313111,313113,313133,313141,313154,313160,313162-313163,313166,313169,313174,313176-313177,313180,313182,313184-313186,313188,313190-313191,313194,313249-313252,313254,313257-313261,313268-313285,313287,313306-313312,313314-313318,313322,313325,313327-313328,313330-313333,313335,313337,313341,313345-313346,313348-313352,313354,313356,313358-313360,313374,313376-313379,313386,313389-313392,313394-313398,313401,313404,313407,313409,313411-313421,313423,313425-313430,313436-313439,313442,313448-313449,313451,313453-313455,313457-313458,313467,313472,313477,313483,313493-313497,313547,313549,313554-313556,313559,313561-313564,313568,313572-313573,313575,313578,313645,313650,313652-313656,313659-313674,313679,313683-313684,313686-313688,313690,313692-313697,313707,313709-313710,313712-313713,313715,313727,313730-313731,313733-313740,313744-313746,313748-313749,313751-313752,313756-313757,313760-313761,313769-313772,313774-313775,313777-313779,313782,313784,313797-313800,313804-313810,313813,313817,313819-313821,313823,313841,313850-313855,313859,313863-313867,313869-313875,313877-313880,313893,313895,313897-313898,313901-313910,313912-313915,313917-313919,313922,313924-313925,313928,313933-313934,313936-313937,313939-313941,313944-313949,313953,313955,313958-313963,313965-313967,313972-313974,313978-313979,313982,313992-313996,313999,314001,314027,314035,314037-314040,314043-314046,314048,314050,314054-314056,314058-314062,314064-314071,314073-314082,314086-314088,314094,314097,314100-314101,314104-314106,314109,314112-314114,314118,314131-314132,314136,314139,314145,314148,314150-314155,314158,314167,314176,314179,314181,314183,314185,314187,314189-314196,314200,314203-314209,314211,314213-314215,314217-314219,314226,314228-314233,314239-314242,314244-314247,314252-314257,314267-314268,314271-314275,314280,314282,314287,314290-314295,314299-314300,314302,314306-314314,314316,314319-314321,314326,314328,314336-314339,314341-314345,314357-314358,314365,314370,314372,314374-314375,314382,314387,314396-314400,314402-314404,314409,314419-314420,314424,314429-314430,314435,314442,314450-314464,314466-314468,314470,314472,314474-314479,314483-314487,314489-314490,314496-314497,314502,314504-314505,314507,314509,314520,314527-314528,314533,314535,314541-314543,314545,314547-314549,314553-314558,314561-314562,314564,314571-314572,314576,314578-314579,314587,314592,314596,314598,314601,314604,314610-314611,314614-314621,314623,314625-314627,314636-314638,314643-314645,314647-314651,314653-314659,314666,314671,314675-314676,314686,314691,314693-314694,314700-314701,314705,314708-314709,314714-314718,314720,314723,314768-314774,314782-314783,314785-314786,314790,314793-314805,314807-314808,314810-314812,314814,314826,314828,314830-314831,314833,314837,314840-314843,314849-314853,314859-314861,314863-314872,314877-314878,314884-314896,314902-314908,314912-314914,314916,314918-314920,314924-314928,314933-314935,314942-314948,314950-314953,314956,314960,314962-314968,314970-314971,314973-314974,314987-314989,314993-314996,314998,315001-315002,315004-315006,315008-315010,315016,315018-315019,315022,315025,315030-315036,315038-315041,315046-315050,315053-315054,315056,315059,315067,315074-315078,315082-315089,315091,315095-315098,315102,315106,315108,315111-315114,315132-315133,315143,315146,315155-315163,315165,315167,315170,315176-315178,315186-315187,315192,315196,315199-315204,315206,315208,315211-315215,315225,315228-315230,315233-315238,315242,315244,315272-315273,315277-315281,315283,315286-315287,315289,315292,315294,315298,315303,315305,315307,315310,315318,315320,315323,315325-315327,315331,315334-315337,315360-315365,315368,315370,315402,315404-315408,315410,315412,315419-315420,315422-315423,315426-315427,315429-315431,315435,315442-315443,315449,315452-315453,315455,315457-315461,315466,315478,315482,315484-315485,315488-315490,315496-315510,315516,315518,315523-315526,315529-315530,315533-315534,315536,315545,315552,315560-315561,315564,315571-315580,315586-315589,315591,315595,315597-315598,315601,315603,315615,315619,315621,315636,315639-315642,315645-315648,315652-315654,315656,315670,315673-315674,315677-315678,315681-315683,315685-315692,315694,315697-315701,315708-315709,315711,315713-315716,315718-315720,315732,315734,315736,315738,315745,315749,315753-315754,315759-315762,315764-315766,315768,315770-315772,315774-315778,315780,315784,315788-315790,315793,315795-315799,315802-315804,315845,315850-315853,315856-315861,315863-315864,315869-315871,315876,315878,315892,315896,315900,315908,315910,315912-315913,315920-315924,315926,315932-315935,315947,315956-315957,315960-315961,315964,315968,315973-315974,315983,316005,316008-316009,316011-316013,316017,316020,316022-316023,316025,316029-316038,316049-316051,316053,316055-316057,316060-316062,316064-316066,316076-316082,316095,316099-316100,316102-316113,316115,316118-316119,316125,316131-316133,316171-316172,316174-316175,316179-316181,316183-316184,316211-316214,316258,316260,316275,316279-316280,316283-316289,316304,316309-316311,316313-316314,316328-316329,316341-316343,316355,316357-316358,316368,316374,316377,316393-316395,316397,316412-316413,316424,316426-316428,316430,316433-316437,316460-316461,316466-316468,316473,316477,316484-316485,316489-316493,316495,316497,316500-316501,316503-316508,316510-316512,316515,316519-316526,316528-316537,316541-316542,316544,316548-316554,316557-316558,316561-316566,316568,316571,316573,316577-316578,316580,316585-316587,316589-316590,316592-316593,316599-316603,316606-316612,316617,316625,316627-316635,316637-316638,316643-316644,316647-316649,316652-316659,316661,316664-316667,316669-316670,316676-316677,316679,316681-316682,316684-316687,316689-316692,316694-316695,316698-316699,316702,316704,316706,316715-316720,316731-316732,316734,316738-316740,316742-316743,316745-316747,316749-316750,316754-316756,316758-316761,316766,316768,316770-316771,316774,316776-316777,316780,316782,316792-316794,316802,316809-316818,316822-316824,316829,316851-316854,316856,316859,316866-316867,316869,316874,316936-316938,316940-316941,316945-316946,316949,316953,316956,316971-316972,316989,316991,316993-316997,317033-317034,317036,317041,317049-317052,317058,317060,317064,317075-317076,317079-317080,317092,317097,317099-317100,317107,317135,317137-317139,317141,317148,317154-317156,317159-317164,317168-317171,317173,317179-317180,317185-317186,317192-317193,317196-317197,317200-317201,317208,317212,317214-317216,317235-317239,317244,317246-317247,317253-317254,317265,317267,317269-317270,317272-317276,317279-317280,317283-317284,317288-317297,317299-317300,317304-317313,317315-317316,317323-317324,317340,317343-317346,317348-317350,317353,317356,317361,317369-317372,317376,317378,317382,317385,317388-317390,317409-317415,317428,317431,317435-317437,317444-317445,317457-317460,317463-317465,317483,317504-317505,317507,317511-317512,317517-317518,317522-317523,317527,317533,317541-317545,317547,317558,317560-317561,317566,317568,317576,317578,317583-317585,317591-317592,317594,317596-317597,317601,317606,317608,317610-317611,317632,317645,317648,317651-317652,317657-317660,317663,317665-317667,317678-317682,317694,317696,317700-317710,317712-317713,317715,317723,317729-317732,317736-317737,317739-317743,317745,317747-317748,317750-317752,317754-317755,317758-317759,317774-317775,317777,317779,317782-317786,317789-317790,317799,317801-317803,317806,317809-317821,317824,317828,317830,317835-317837,317839-317840,317842-317848,317854,317862-317863,317886-317888,317901,317904-317906,317908-317909,317912,317915,317923,317928,317931,317933-317935,317938,317981-317982,317996,317998,318001-318017,318021,318024-318026,318090-318092,318094,318098-318100,318107,318109,318116,318124-318126,318130,318135-318138,318141-318144,318147,318150,318160-318161,318163,318167,318175-318182,318185,318188-318191,318193-318194,318210-318219,318222-318224,318229-318232,318242-318243,318246,318251-318252,318254-318257,318259,318263,318276,318278,318280-318283,318287,318294,318298-318299,318302-318305,318307,318312-318318,318320,318325-318326,318352-318354,318356,318360,318364,318380,318382-318383,318386,318398-318401,318427,318431-318432,318434,318436-318437,318439,318444-318446,318448,318450,318476,318478-318479,318481,318511-318512,318514-318517,318520,318523,318527,318530-318531,318539,318545-318546,318565,318571,318574,318578,318580-318582,318584-318586,318588,318590-318593,318599,318601,318603,318606,318631-318633,318643,318646-318647,318649,318651,318655,318676-318679,318688-318689,318693-318695,318701-318712,318715,318717-318723,318734,318738-318739,318745,318748-318749,318754-318755,318762,318765,318774,318780-318781,318788,318790,318794-318795,318812,318814,318816,318818-318824,318827-318834,318860-318861,318863,318867,318872,318879,318884,318891,318894-318896,318899-318900,318906-318908,318914,318916-318917,318920-318926,318928,318930,318932,318935,318943,318945,318952-318955,318960,318962,318966,318968,318970,318973-318975,318982,318986-318996,318998-318999,319001-319002,319008,319010,319015,319019-319020,319022,319026-319031,319033-319049,319051-319052,319054,319056-319063,319071-319078,319083-319086,319096,319118,319125,319132,319137-319138,319156-319159,319162,319167,319197-319201,319212,319219,319229,319246,319293-319295,319312,319316-319321,319336-319341,319350,319355,319365,319369,319409-319410,319412-319414,319432,319444,319454-319456,319473,319475,319481,319487-319488,319490-319491,319493,319499-319501,319506,319509,319513,319516-319520,319539-319540,319542-319546,319548,319551,319556-319558,319561-319562,319571,319578-319579,319581-319582,319584,319586,319588-319590,319595,319602-319605,319608-319610,319612,319620-319621,319626-319627,319634,319637,319642,319655-319656,319659-319660,319662,319670-319672,319675-319682,319688-319690,319694,319699,319702,319709,319714,319717-319718,319731,319746-319751,319755-319758,319761-319769,319774-319775,319793,319796,319798,319800,319803,319806,319811,319813-319815,319817-319818,319824-319825,319827,319829-319830,319834,319836,319841-319846,319850-319852,319854-319859,319861-319864,319866-319867,319869,319871-319876,319882,319885,319888,319890,319895,319899-319900,319905,319916,319918,319928,319932-319934,319942,319947,319949,319953-319957,319962,319964,319968-319970,319972,319975,319987,319995-319996,320007-320008,320010-320012,320028,320030,320034-320035,320038,320048-320049,320051-320052,320056,320061-320063,320065,320069-320070,320072,320076-320079,320093,320108,320118,320121,320123-320126,320135,320145-320146,320151-320153,320156,320163,320165-320167,320170,320172-320176,320181,320183-320186,320189,320192-320196,320201-320202,320204,320206-320208,320216,320234-320235,320237-320240,320242,320252,320255-320256,320259,320262-320264,320266-320267,320269-320271,320273,320277,320288,320292,320296,320300-320301,320304,320307-320308,320313-320314,320316-320319,320322-320323,320332-320339,320343-320345,320352-320353,320359-320360,320363-320364,320367,320372,320387,320390,320392,320403,320406,320408-320409,320412,320417,320420-320425,320430,320433,320441-320446,320451-320452,320456,320458,320460-320463,320467-320468,320472,320478-320479,320481-320483,320488-320495,320497-320498,320501,320508-320509,320517,320521-320522,320527,320546-320547,320553,320555,320560-320561,320570,320574-320578,320580,320595,320599,320604,320619-320620,320624-320636,320644,320655-320656,320658-320659,320663,320665,320670-320672,320675,320678,320683,320686,320688,320690,320694,320696,320701-320703,320705,320714,320723-320724,320729-320730,320733-320734,320736-320737,320740-320743,320748,320750,320754-320755,320761-320762,320773-320775,320785,320796,320801,320803-320804,320806-320807,320810-320820,320836-320837,320842-320843,320851-320854,320865,320868,320876,320883,320892-320893,320895-320896,320900-320902,320906,320914,320916,320918,320927-320932,320936-320938,320941,320943-320944,320956,320969,320974-320975,320977,320980,320982,320984,320986,320988-320991,320996-320997,320999,321001-321003,321008,321011,321015,321023,321035-321036,321045-321046,321048,321063,321072,321075,321080-321082,321102-321106,321109,321111,321128,321138,321173,321179,321196,321203,321206-321207,321214,321217-321218,321226,321228,321230,321233,321235,321240,321247-321248,321256,321261,321284,321286,321293-321294,321298,321302,321305-321306,321314,321327-321330,321332,321342,321347-321349,321356,321366,321368-321371,321376-321378,321385-321387,321390,321393,321397-321401,321406-321410,321420,321422-321423,321426,321433,321435-321437,321439,321444,321446,321450,321455-321457,321460-321461,321471,321483-321484,321486,321489-321490,321502,321508-321512,321514,321580-321584,321586,321588-321589,321601,321603,321605-321608,321620-321623,321625,321627-321628,321639-321640,321652,321664,321668-321671,321674-321675,321684-321686,321688-321689,321702-321706,321708-321714,321719-321723,321726-321727,321730,321733-321734,321737,321743-321746,321751,321762,321772-321773,321779-321780,321782,321791-321792,321794-321795,321798-321799,321803,321805-321806,321810,321812,321816,321821,321823,321826,321828,321836-321847,321849,321851-321852,321856,321858-321859,321864,321876,321884,321896,321899,321915,321919,321921-321922,321924,321926,321933-321934,321938,321947-321952,321954,321959,321963,321965,321969,321983-321986,321991,322014-322016,322019,322023,322025-322026,322028,322034,322036-322041,322050,322052,322055-322056,322059,322073-322074,322076-322077,322097,322101,322112-322113,322123-322124,322139,322167,322169-322171,322175,322177-322179,322200-322201,322203,322209-322210,322212-322214,322218,322222,322226,322228,322230,322233-322234,322237,322239,322241,322245,322248,322250-322251,322255-322258,322270-322272,322280,322282,322288,322291,322293,322296-322297,322299,322302,322304-322306,322308-322310,322316,322320-322323,322326,322328-322329,322331,322336,322351,322354-322357,322360,322364,322368,322371,322373-322374,322380,322383,322386,322391-322392,322397,322401,322404-322406,322408,322410-322411,322418,322424-322428,322431,322434-322437,322441,322443,322445,322447-322451,322456,322459,322463-322465,322473-322479,322483,322485-322488,322493-322496,322527,322530,322542,322544,322546-322550,322567,322571,322580-322581,322590,322592-322593,322596,322613,322618,322627-322628,322633,322636,322648,322657,322667-322669,322672,322677-322678,322680,322682-322684,322694-322695,322706,322710,322713-322714,322718-322723,322726,322740,322746,322749-322752,322756-322757,322762-322763,322770-322771,322773-322775,322795-322800,322802-322804,322810,322812,322816,322821,322825-322826,322830-322833,322852,322854-322855,322868,322872,322874-322875,322883,322885,322888,322894,322896-322897,322899,322901-322903,322913-322915,322921,322923,322925-322941,322947-322948,322951,322959-322960,322962,322964-322965,322970-322971,322980-322982,322984-322987,322990,322992,322994-322995,322997-322999,323001-323004,323006,323010-323011,323014-323018,323020-323022,323024-323026,323032,323039-323041,323045-323047,323051,323053-323054,323058-323059,323062-323065,323067-323069,323074,323076,323078,323084,323086,323100,323102,323112,323114,323126-323128,323131-323133,323151,323155,323160,323166,323170,323174-323178,323183-323185,323192-323197,323217,323220-323221,323224-323226,323228-323230,323234-323236,323245,323252-323253,323258,323261,323263,323272,323275,323278,323280,323286,323290,323305,323307-323308,323310,323317,323324-323327,323341,323343,323347,323349-323351,323355,323364,323367-323368,323379,323382,323385-323392,323394,323405,323407,323428,323433,323435-323436,323438-323439,323448,323450,323465,323474,323478-323483,323491,323494,323496-323499,323508-323509,323513-323514,323520,323522,323524-323525,323528,323539-323541,323544,323547,323552-323554,323557-323559,323561-323562,323564,323572-323575,323578-323585,323587-323589,323597-323598,323611-323613,323616,323625,323630-323631,323642,323645,323654-323656,323671,323675-323676,323683,323689,323691-323692,323703-323705,323707,323709-323710,323714-323715,323722,323727-323729,323768-323769,323772,323781-323782,323785-323786,323791,323793,323796-323797,323812-323813,323822,323824,323834,323836,323839-323840,323851-323854,323856,323858-323860,323864,323867-323868,323873,323880,323882,323885-323886,323889,323895-323897,323905-323910,323915-323916,323918,323924,323933-323934,323945,323952,323961-323962,323973,323978,323981,323985,323991,323993-323995,323997-323998,324007,324011-324012,324014,324016,324026,324039,324041,324044-324045,324048-324050,324065-324066,324072-324075,324077-324081,324086-324088,324098-324100,324107-324108,324112-324114,324123,324125,324127,324143,324146,324148,324155-324156,324163-324164,324168-324170,324172-324174,324178-324179,324185,324193-324197,324199-324202,324206,324212,324220-324224,324226,324228,324239-324241,324243,324248-324249,324257-324258,324260-324261,324276-324278,324281,324285-324286,324292,324296,324301-324316,324320-324321,324326-324328,324330,324334-324335,324343-324350,324354-324355,324359-324360,324362,324364,324366-324367,324369,324373,324378-324379,324386,324388,324413,324415,324422-324425,324427,324429-324439,324443,324445,324449-324454,324456-324457,324460,324469-324471,324476,324478,324487-324492,324497,324501-324502,324506,324509,324516-324517,324528,324535-324536,324538,324547,324552-324559,324566,324568,324590,324592-324595,324597,324600,324606,324609-324610,324613,324621-324623,324628-324637,324639,324644,324646-324654,324659-324661,324665,324668-324672,324681,324683,324688-324689,324694,324696,324700-324704,324709,324716-324717,324719,324721,324727,324738,324752,324757,324770,324772,324774,324778,324780,324787,324789,324792-324795,324803-324805,324807,324809,324814,324817,324821,324823-324824,324826,324836,324841-324845,324850,324853,324857,324862-324865,324867-324869,324876-324883,324920,324923,324926,324928-324929,324940-324941,324945,324947,324956-324957,324972,324977-324982,324991-324996,325007-325011,325014,325016-325018,325028,325030,325035,325039,325042,325044-325045,325050-325051,325054-325056,325059-325061,325063,325065-325067,325093-325095,325101,325103,325108,325114-325116,325122-325125,325156-325158,325170-325176,325217,325227-325228,325232,325248,325266,325268-325279,325285-325286,325292,325294-325299,325310,325319,325321,325328,325332,325338-325339,325353,325355,325359-325364,325371,325373,325376-325377,325379-325380,325382-325383,325386-325389,325397,325410,325420,325433,325436,325438,325441,325444,325447,325450-325451,325456,325458,325469,325478-325485,325515,325517,325528,325530,325532-325533,325552-325556,325558-325559,325561,325566-325567,325570-325571,325588,325596,325606,325608,325610,325614-325616,325627,325635,325637-325638,325641,325648-325658,325662,325670-325671,325680-325681,325683,325685-325694,325706-325708,325716-325718,325723-325725,325737,325743-325744,325748,325755,325758-325759,325764,325766-325767,325769,325775-325776,325779-325780,325795,325800,325805-325807,325811-325813,325817,325827,325834-325835,325841,325850-325851,325857,325861,325863,325865,325880,325883-325884,325887-325888,325897,325917-325922,325924-325925,325946-325947,325950,325953,325959-325963,325976,325997,326019-326021,326026,326028,326030,326032,326034,326036,326038-326042,326046,326052,326055,326058,326060-326061,326063,326065,326067-326070,326072,326074,326082,326084,326086,326089-326090,326093-326094,326096,326098,326100-326101,326106-326107,326110-326112,326115-326118,326122,326132,326134-326137,326140,326143-326144,326150,326161,326173,326175-326178,326182,326184-326187,326194-326200,326228-326229,326234-326235,326237,326244,326273,326276,326281,326284,326286,326288-326290,326307,326309,326311,326315,326330-326331,326343,326361-326362,326371,326378,326383-326384,326386,326391-326392,326399,326401,326407,326409-326410,326412,326420-326422,326424,326429,326434,326438,326440-326448,326455-326456,326473,326479-326480,326483-326496,326502,326504,326507,326509-326510,326518,326522,326526-326527,326544,326547,326558,326560-326562,326565-326567,326572,326584-326594,326597,326600,326609-326610,326613,326615-326616,326624,326628-326629,326633,326640,326646,326654-326655,326657,326662,326666,326668-326671,326698,326707-326712,326714,326720,326724,326731-326732,326738,326748,326750,326752-326754,326768-326769,326772,326774,326776,326784,326791-326792,326794,326796-326799,326811-326813,326831,326834-326835,326847,326851,326853-326858,326861-326863,326866,326868-326869,326872-326888,326897-326898,326909,326912,326914,326919,326924-326927,326935,326937,326940,326953-326954,326957,326960-326963,326971,326973,326977,326982-326983,326986,326988-326990,326993-326995,327004,327029,327032,327034,327047-327050,327053,327062,327074,327088,327093-327094,327096-327097,327099,327117-327118,327140,327164,327167-327168,327183,327211,327213,327220-327222,327226,327231-327232,327235-327236,327264,327282-327286,327289,327295,327298-327299,327316,327319,327329,327332-327334,327336,327342,327346-327347,327350-327354,327356,327359,327361,327367,327375-327377,327385-327386,327390,327393-327399,327401-327402,327416,327420,327435,327437-327439,327444,327448-327451,327453-327454,327468-327469,327472,327489,327496-327498,327502-327503,327510,327517,327523-327525,327528,327540,327555,327560-327561,327567,327578,327593,327597,327606,327612,327625,327662,327666,327674-327676,327682-327683,327685,327697-327700,327703-327705,327718,327721-327726,327730,327737,327751-327753,327755,327760,327768,327770,327775,327779-327780,327788,327792,327796,327803,327816-327822,327826,327834,327838,327845,327862,327874-327875,327878-327881,327888,327905,327911-327914,327917,327925,327930,327946-327947,327951,327963-327965,327972-327973,327991,327996,328007-328009,328013,328027,328029-328032,328035,328052,328055,328060-328061,328082-328083,328085,328096,328106-328108,328116,328119-328120,328126,328128,328134-328135,328153,328156-328157,328160-328161,328163,328166,328169,328171,328173,328177,328188-328192,328199-328200,328202,328205-328206,328208-328209,328212,328217,328221-328222,328224,328226,328228,328230,328232,328234,328237,328243,328246,328248,328250,328252,328254,328256,328264,328266,328273,328283-328284,328287-328293,328296-328297,328304-328306,328314-328315,328318,328326,328329,328332,328344,328350,328352,328382,328398-328399,328411-328412,328419-328420,328423,328436-328439,328441,328444,328446-328450,328468,328470,328491,328497,328504-328505,328514,328536,328540-328541,328552,328567,328584,328591,328603,328605,328608,328610-328615,328622-328628,328630,328641-328643,328647,328659,328694-328695,328720,328731,328769-328770,328773-328774,328776,328779-328784,328795,328799,328806,328808,328826,328830,328835,328853,328864-328865,328876,328880-328881,328895,328913,328917,328937-328938,328971,328986-328987,328990,328995-328996,328999-329000,329009,329016,329019,329022,329024,329050,329053-329054,329060,329091-329092,329101,329181,329195,329197-329198,329214-329216,329252,329254,329256,329264,329271,329295,329297,329311,329314,329347,329361-329362,329364-329365,329371-329372,329374,329376-329378,329416,329447,329464-329468,329470-329472,329475-329477,329509-329516,329519,329521,329523-329525,329555-329556,329561,329563,329584,329682,329703,329711,329749,329820,329825,329843,329846,329848,329864,329880,329905,329930,329994,330004,330033,330104,330127,330236,330256,330364